infrastructure/gitlab-ci
2
0
Fork 0
Code Issues 17 Pull Requests 1 Actions Projects Releases 21 Wiki Activity

refactor: restructure repo #71

Merged
jon_nfc merged 32 commits from restructure into development 2023-05-14 06:32:22 +00:00
Conversation 144 Commits 32 Files Changed 21 +525 -127
21 changed files with 525 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
.gitlab-ci.yml
View File

@ -1,15 +1,15 @@
variables:
JOB_ROOT_DIR: '.'
SECURE_LOG_LEVEL: debug
GIT_SUBMODULE_STRATEGY: recursive
MY_PROJECT_ID: "28543717"
LICENSE_FINDER_CLI_OPTS: '--recursive'
JOB_ROOT_DIR: '.'
SECURE_LOG_LEVEL: debug
GIT_SUBMODULE_STRATEGY: recursive
MY_PROJECT_ID: "28543717"
LICENSE_FINDER_CLI_OPTS: '--recursive'
stages:
- validation
- test
- release
- sync
- validation
- test
- release
- sync
include:
- local: $JOB_ROOT_DIR/yaml_lint/.gitlab-ci.yml
@ -95,33 +95,37 @@ yaml_lint Dependencies:
PyLint:
extends:
- .PyLint
image: python:3.6-slim
extends:
- .PyLint
image: python:3.6-slim
gilab-ci.yml Lint (python 3.6):
variables:
YAML_LINT_PATH: "/*/.gitlab-ci.yml"
extends:
- .yaml_lint_defaults
image: python:3.6-slim
gilab-ci.yml Lint (python 3.11):
variables:
YAML_LINT_PATH: ".gitlab-ci* ./**/*gitlab-ci*"
YAML_LINT_EXTRA_ARGS: "{ extends: yaml_lint/.yamllint.yaml, ignore: [website-template/*] }"
extends:
- .yaml_lint_defaults
image: python:3.11-slim
Markdown Linting:
variables:
#MDLINT_PATHS: '!website_template** !.gitlab** !CHANGELOG.md **/*.md'
MDLINT_PATHS: "**/*.md **/**/*.md **/**/**/*.md **/**/**/**/*.md **/**/**/**/**/**/*.md #**CHANGELOG.md #CHANGELOG.md #website-template/** #.gitlab/**"
extends:
- .Lint_Markdown
Gitlab Release:
variables:
MY_COMMAND: ./sub-folder_changlog.sh
extends:
- .gitlab_release
variables:
MY_COMMAND: ./sub-folder_changlog.sh
extends:
- .gitlab_release
Github (Push --mirror):
variables:
GIT_SYNC_URL: "https://$GITHUB_USERNAME_ROBOT:$GITHUB_TOKEN_ROBOT@github.com/NoFussComputing/gitlab-ci.git"
extends:
- .git_push_mirror
variables:
GIT_SYNC_URL: "https://$GITHUB_USERNAME_ROBOT:$GITHUB_TOKEN_ROBOT@github.com/NoFussComputing/gitlab-ci.git"
extends:
- .git_push_mirror

13
.gitlab-ci_common.yaml Normal file
View File

@ -0,0 +1,13 @@
variables:
JOB_ROOT_DIR: 'gitlab-ci'
GIT_SUBMODULE_STRATEGY: recursive
stages:
- validation
- build
- prepare
- test
- release
- sync
- publish

4
.gitmodules vendored Normal file
View File

@ -0,0 +1,4 @@
[submodule "website-template"]
path = website-template
url = https://gitlab.com/nofusscomputing/infrastructure/website-template.git
branch=development

130
conventional_commits/.gitlab-ci.yml
View File

@ -1,70 +1,76 @@
.conventional_commit:
variables:
DEFAULT_ROOT_DIR: './gitlab-ci'
image: python:3.6-slim
stage: validation
before_script:
- mkdir -p "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME"
- mkdir -p "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests"
- if [ "0$MR_ACCESS_TOKEN" == "0" ]; then MR_ACCESS_TOKEN=$CI_JOB_TOKEN; fi
- echo "[DEBUG] MR_ACCESS_TOKEN[$MR_ACCESS_TOKEN]"
- if [ "0$JOB_ROOT_DIR" == "0" ]; then ROOT_DIR=$DEFAULT_ROOT_DIR; else ROOT_DIR=$JOB_ROOT_DIR ; fi
- echo "[DEBUG] ROOT_DIR[$ROOT_DIR]"
- if [ "0$MY_PROJECT_ID" == "0" ]; then PROJECT_ID=$CI_PROJECT_ID; else PROJECT_ID=$MY_PROJECT_ID ; fi
- echo "[DEBUG] PROJECT_ID[$PROJECT_ID]"
- export PYTHON_VERSION=`python -c 'import sys; version=sys.version_info[:3]; print("{0}.{1}.{2}".format(*version))'`
- apt update
- apt install --no-install-recommends -y git
- python3 -m venv env
- . env/bin/activate
- pip install --upgrade pip
- pip install -r $ROOT_DIR/conventional_commits/requirements.txt
- echo "[DEBUG] CI_PROJECT_ID[$CI_PROJECT_ID]"
- echo "[DEBUG] CI_COMMIT_BRANCH[$CI_COMMIT_BRANCH]"
- git fetch --all
- git checkout --track origin/$CI_COMMIT_BRANCH
- git show-branch -a
- target_branch=$(git show-branch -a | awk 'BEGIN { FS="\n\s+*" } { print $1 }' | awk '{print $2}' | grep '\[' | sed 's/.*\[origin\/\(.*\)\].*/\1/' | grep -v '\[' | grep -v $(git rev-parse --abbrev-ref HEAD) | grep -vi 'HEAD' | awk 'BEGIN{ RS = "" ; FS = "\n" }{print $1}')
- echo "[DEBUG] Target Branch[$target_branch]"
- if [ -d "gitlab-ci" ]; then ls -la gitlab-ci; fi
- first_sha1=$(git log origin/$target_branch..$CI_COMMIT_BRANCH --format=format:%H | tail -1)
- echo "[DEBUG] First Commit SHA[$first_sha1]"
- echo "[DEBUG] artifacts directory [$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME]"
after_script:
- ls -lR "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE"
- cd ..
- rm -Rf check
artifacts:
expire_in: 3 days
when: always
paths:
- "$CI_PROJECT_DIR/artifacts/*"
reports:
junit:
- "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/*.junit.xml"
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
when: never
- if: '$CI_COMMIT_BRANCH'
when: always
- when: never
variables:
DEFAULT_ROOT_DIR: './gitlab-ci'
image: python:3.6-slim
stage: validation
before_script:
- mkdir -p "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME"
- mkdir -p "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests"
- if [ "0$MR_ACCESS_TOKEN" == "0" ]; then MR_ACCESS_TOKEN=$CI_JOB_TOKEN; fi
- echo "[DEBUG] MR_ACCESS_TOKEN[$MR_ACCESS_TOKEN]"
- if [ "0$JOB_ROOT_DIR" == "0" ]; then ROOT_DIR=$DEFAULT_ROOT_DIR; else ROOT_DIR=$JOB_ROOT_DIR ; fi
- echo "[DEBUG] ROOT_DIR[$ROOT_DIR]"
- if [ "0$MY_PROJECT_ID" == "0" ]; then PROJECT_ID=$CI_PROJECT_ID; else PROJECT_ID=$MY_PROJECT_ID ; fi
- echo "[DEBUG] PROJECT_ID[$PROJECT_ID]"
- export PYTHON_VERSION=`python -c 'import sys; version=sys.version_info[:3]; print("{0}.{1}.{2}".format(*version))'`
- apt update
- apt install --no-install-recommends -y git
- python3 -m venv env
- . env/bin/activate
- pip install --upgrade pip
- pip install -r $ROOT_DIR/conventional_commits/requirements.txt
- echo "[DEBUG] CI_PROJECT_ID[$CI_PROJECT_ID]"
- echo "[DEBUG] CI_COMMIT_BRANCH[$CI_COMMIT_BRANCH]"
- git fetch --all
- git checkout --track origin/$CI_COMMIT_BRANCH
- git show-branch -a
- target_branch=$(git show-branch -a | awk 'BEGIN { FS="\n\s+*" } { print $1 }' | awk '{print $2}' | grep '\[' | sed 's/.*\[origin\/\(.*\)\].*/\1/' | grep -v '\[' | grep -v $(git rev-parse --abbrev-ref HEAD) | grep -vi 'HEAD' | awk 'BEGIN{ RS = "" ; FS = "\n" }{print $1}')
- echo "[DEBUG] Target Branch[$target_branch]"
- if [ -d "gitlab-ci" ]; then ls -la gitlab-ci; fi
- first_sha1=$(git log origin/$target_branch..$CI_COMMIT_BRANCH --format=format:%H | tail -1)
- echo "[DEBUG] First Commit SHA[$first_sha1]"
- echo "[DEBUG] artifacts directory [$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME]"
after_script:
- ls -lR "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE"
- cd ..
- rm -Rf check
artifacts:
expire_in: 3 days
when: always
paths:
- "$CI_PROJECT_DIR/artifacts/*"
reports:
junit:
- "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/*.junit.xml"
rules:
- if: '$JOB_STOP_CONVENTIONAL_COMMITS'
when: never
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_TAG'
when: always
- if: '$CI_COMMIT_BRANCH == "master"'
when: always
- if: '$CI_COMMIT_BRANCH'
when: always
- when: never
MR Title:
extends:
- .conventional_commit
script:
- MR_TITLE=$($ROOT_DIR/conventional_commits/scripts/commit.py --token "$MR_ACCESS_TOKEN" --project $PROJECT_ID --title --branch $CI_COMMIT_BRANCH)
- echo "[DEBUG] MR_TITLE[$MR_TITLE]"
- cz_exit=0 && cz check --message "$MR_TITLE" > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/cz_output.log" 2>&1 || cz_exit=$?
- . $ROOT_DIR/conventional_commits/scripts/cz_junit.sh > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/$CI_JOB_NAME-cz.junit.xml"
extends:
- .conventional_commit
script:
- MR_TITLE=$($ROOT_DIR/conventional_commits/scripts/commit.py --token "$MR_ACCESS_TOKEN" --project $PROJECT_ID --title --branch $CI_COMMIT_BRANCH)
- echo "[DEBUG] MR_TITLE[$MR_TITLE]"
- cz_exit=0 && cz check --message "$MR_TITLE" > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/cz_output.log" 2>&1 || cz_exit=$?
- . $ROOT_DIR/conventional_commits/scripts/cz_junit.sh > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/$CI_JOB_NAME-cz.junit.xml"
Commit Messages:
extends:
- .conventional_commit
script:
- if [ "$(git log $first_sha1..HEAD --format=format:%H | wc -l)" -eq 0 ]; then echo "[DEBUG] Single Commit"; cz_exit=0 && cz check -m "$(git log HEAD --format=format:%B -1)" > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/cz_output.log" 2>&1 || cz_exit=$?; fi
- if [ "$(git log $first_sha1..HEAD --format=format:%H | wc -l)" -gt 0 ]; then echo "[DEBUG] Commit range"; cz_exit=0 && cz check --rev-range $first_sha1..HEAD > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/cz_output.log" 2>&1 || cz_exit=$?; fi
- . $ROOT_DIR/conventional_commits/scripts/cz_junit.sh > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/$CI_JOB_NAME-cz.junit.xml"
extends:
- .conventional_commit
script:
- if [ "$(git log $first_sha1..HEAD --format=format:%H | wc -l)" -eq 0 ]; then echo "[DEBUG] Single Commit"; cz_exit=0 && cz check -m "$(git log HEAD --format=format:%B -1)" > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/cz_output.log" 2>&1 || cz_exit=$?; fi
- if [ "$(git log $first_sha1..HEAD --format=format:%H | wc -l)" -gt 0 ]; then echo "[DEBUG] Commit range"; cz_exit=0 && cz check --rev-range $first_sha1..HEAD > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/cz_output.log" 2>&1 || cz_exit=$?; fi
- . $ROOT_DIR/conventional_commits/scripts/cz_junit.sh > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/$CI_JOB_NAME-cz.junit.xml"

2
conventional_commits/requirements.txt
View File

@ -1,3 +1,3 @@
python-gitlab
requests
commitizen
commitizen==2.21.0

9
git_push_mirror/.gitlab-ci.yml
View File

@ -6,6 +6,7 @@
- apk update
- apk add git
script:
- if [ "0$GIT_SYNC_URL" == "0"]; then echo "[ERROR] you must define variable GIT_SYNC_URL for mirroring this repository."; fi
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/$CI_PROJECT_PATH --mirror $CI_PROJECT_NAME
- cd $CI_PROJECT_NAME
- git remote add destination $GIT_SYNC_URL
@ -14,7 +15,11 @@
artifacts:
expire_in: 1 day
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
- if: '$CI_COMMIT_BRANCH == "development"'
- if: '$JOB_STOP_GIT_PUSH_MIRROR'
when: never
- if: '$CI_COMMIT_TAG && $GIT_SYNC_URL != null'
when: always
- if: '$CI_COMMIT_BRANCH == "master" && $GIT_SYNC_URL != null'
- if: '$CI_COMMIT_BRANCH == "development" && $GIT_SYNC_URL != null'
when: always

25
gitlab_release/.gitlab-ci.yml
View File

@ -26,14 +26,15 @@
- "$MY_COMMAND"
- if [ "$CI_COMMIT_BRANCH" == "development" ] ; then RELEASE_CHANGELOG=$(cz -n cz_nfc bump --changelog --changelog-to-stdout --prerelease rc); else RELEASE_CHANGELOG=$(cz -n cz_nfc bump --changelog --changelog-to-stdout); fi
- RELEASE_VERSION_NEW=$(cz -n cz_nfc version --project)
- RELEASE_TAG=v$RELEASE_VERSION_NEW
- if [ "0$RELEASE_VERSION_CURRENT" == "0$RELEASE_VERSION_NEW" ]; then echo "[DEBUG] No tag to delete, version was not bumped"; else git tag -d $RELEASE_TAG; fi
- RELEASE_TAG=$RELEASE_VERSION_NEW
- echo "[DEBUG] RELEASE_VERSION_CURRENT[$RELEASE_VERSION_CURRENT]"
- echo "[DEBUG] RELEASE_CHANGELOG[$RELEASE_CHANGELOG]"
- echo "[DEBUG] RELEASE_VERSION_NEW[$RELEASE_VERSION_NEW]"
- echo "[DEBUG] RELEASE_TAG[$RELEASE_TAG]"
- RELEASE_TAG_SHA1=$(git log -n1 --format=format:"%H")
- echo "[DEBUG] RELEASE_TAG_SHA1[$RELEASE_TAG_SHA1]"
- if [ "0$RELEASE_VERSION_CURRENT" == "0$RELEASE_VERSION_NEW" ]; then echo "[DEBUG] No tag to delete, version was not bumped"; else git tag -d $RELEASE_TAG; fi
- if [ "0$RELEASE_VERSION_CURRENT" == "0$RELEASE_VERSION_NEW" ]; then echo "[DEBUG] No push will be conducted, version was not bumped"; else git push; fi
- if [ "0$RELEASE_VERSION_CURRENT" == "0$RELEASE_VERSION_NEW" ]; then echo "[DEBUG] No release will be created, version was not bumped"; else release-cli create --name "Release $RELEASE_TAG" --tag-name "$RELEASE_TAG" --ref "$RELEASE_TAG_SHA1" --description "$RELEASE_CHANGELOG"; fi
- if [ "$CI_COMMIT_BRANCH" == "master" ] ; then git checkout master; fi
@ -43,14 +44,24 @@
after_script:
- rm -Rf repo
rules:
- if: "$CI_COMMIT_AUTHOR =='NFC CI <CI@nfc-gitlab>'"
- if: '$JOB_STOP_GITLAB_RELEASE'
when: never
- if: '$CI_COMMIT_TAG'
when: never
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
when: never
- if: '$CI_COMMIT_BRANCH == "master"'
when: always
when: on_success
allow_failure: false
- if: '$CI_COMMIT_BRANCH == "development"'
when: manual
allow_failure: true
when: on_success
allow_failure: false
# for testing
# - if: '$CI_COMMIT_BRANCH != "master"'
# when: always
# allow_failure: true
- when: never
@ -85,6 +96,8 @@ commit footer refs:
junit:
- "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/*.junit.xml"
rules:
- if: '$JOB_STOP_CONVENTIONAL_COMMITS'
when: never
- if: '$CI_COMMIT_BRANCH == "development" || $CI_COMMIT_BRANCH == "master"'
when: never
- if: '$CI_COMMIT_BRANCH && $CHANGELOG_FOOTER_REFERENCES != "False"'

2
gitlab_release/requirements.txt
View File

@ -1 +1 @@
commitizen
commitizen==2.21.0

49
mkdocs.yml Normal file
View File

@ -0,0 +1,49 @@
INHERIT: website-template/mkdocs.yml
repo_name: Website
repo_url: https://gitlab.com/nofusscomputing/projects/gitlab-ci
edit_uri: ''
nav:
- Home: index.md
- Articles:
- articles/index.md
- Content Tags: tags.md
- Projects:
- projects/index.md
- Ansible Roles: projects/ansible-roles/README.md
- Gitlab CI:
- projects/gitlab-ci/README.md
- projects/gitlab-ci/ansible_roles.md
- projects/gitlab-ci/conventional_commits.md
- projects/gitlab-ci/docker_container.md
- projects/gitlab-ci/git_mirror.md
- projects/gitlab-ci/gitlab_release.md
- projects/gitlab-ci/python.md
- projects/gitlab-ci/yaml_lint.md
- Python Gitlab Management: projects/python-gitlab-management/README.md
- Operations:
- operations/index.md
- Contact Us: contact.md

0
pages/index.md Normal file
View File

9
ansible/README.md → pages/projects/gitlab-ci/ansible_roles.md
View File

@ -1,4 +1,11 @@
# Ansible
---
title: Ansible
description: How to use No Fuss Computings gitlab-ci job for ansible
date: 2021-08-03
template: manual.html
about: https://gitlab.com/nofusscomputing/projects/gitlab-ci
---
This job does ansible role/playbook linting when any commit is pushed to any branch.

28
conventional_commits/README.md → pages/projects/gitlab-ci/conventional_commits.md
View File

@ -1,4 +1,12 @@
# Conventional Commits User Manual
---
title: Conventional Commits
description: How to use No Fuss Computings gitlab-ci job for Conventional Commits
date: 2021-08-03
template: manual.html
about: https://gitlab.com/nofusscomputing/projects/gitlab-ci
---
## User Manual
Commitizen is used to validate the format of commit messages. we use [Conventional Commit Messages](https://www.conventionalcommits.org/en/v1.0.0/) format for our validation jobs.
This repository may have two CI jobs to do with commitizen:
@ -9,17 +17,17 @@ These CI Jobs output a test report that can be viewed inside of the merge reques
To fix an error please refer to the titled sections below.
## MR Title
### MR Title
Ensure that the merge request title is in the [conventional message](https://www.conventionalcommits.org/en/v1.0.0/) format. NOTE: the title is case sensitive.
## Commit Messages
### Commit Messages
All commit messages that form part of your merge request must be in [conventional message](https://www.conventionalcommits.org/en/v1.0.0/) format.
To fix them go back and edit your commit messages.
### fixing commit messages (suggestion)
#### fixing commit messages (suggestion)
If only the last commit is the commit with an error just use `git commit --amend` and edit your commit message to be in the correct format and save. now push your changes.
@ -51,18 +59,18 @@ Now push your changes upstream.
| *Ensure that all of your commits were exported prior to reseting the branch and when re-applying, that all of your commits were applied correctly* |
# Conventional Commits Admin Manual Manual
## Conventional Commits Admin Manual Manual
This job checks commit messages on a branch and the merge request title for validity against the [conventional commit format](https://www.conventionalcommits.org/en/v1.0.0/)
This job provides the following badge:
- None
## Dependencies
### Dependencies
- None
## your .gitlab-ci.yml changes
### your .gitlab-ci.yml changes
To use this job add the following to your `.gitlab-ci.yml` file
``` yaml
@ -84,11 +92,11 @@ include:
| MR_ACCESS_TOKEN | *only required if you are accessing a private repository.* <br>This token is a user access token that as a minimum requires read-only access to the api to fetch the projects merg requests. |
## Job Workflow
### Job Workflow
## Artifacts
### Artifacts
## License
### License
To view the license for this folder and any sub-folders, refer [here](https://gitlab.com/nofusscomputing/projects/gitlab-ci)

29
pages/projects/gitlab-ci/docker_container.md Normal file
View File

@ -0,0 +1,29 @@
---
title: Docker Container CI Template
description: How to use No Fuss Computings gitlab-ci template for docker containers
date: 2023-05-13
template: manual.html
about: https://gitlab.com/nofusscomputing/projects/gitlab-ci
---
- available tags `dev` for latest dev build
- latest matches latest git tag
- notate available platforms can be viewed in the job
``` yaml
# Available platforms: linux/amd64, linux/amd64/v2, linux/amd64/v3, linux/arm64, linux/riscv64, linux/ppc64, linux/ppc64le, linux/s390x, linux/386, linux/mips64le, linux/mips64, linux/arm/v7, linux/arm/v6
# DOCKER_IMAGE_BUILD_TARGET_PLATFORMS: "linux/amd64,linux/arm64,linux/arm/v7"
DOCKER_IMAGE_BUILD_NAME: $CI_PROJECT_NAME
DOCKER_IMAGE_BUILD_REGISTRY: $CI_REGISTRY_IMAGE
DOCKER_IMAGE_BUILD_TAG: $CI_COMMIT_SHA
# DOCKER_IMAGE_PUBLISH_NAME: $CI_PROJECT_NAME
# DOCKER_IMAGE_PUBLISH_REGISTRY: docker.io/nofusscomputing
# DOCKER_IMAGE_PUBLISH_URL: https://hub.docker.com/r/nofusscomputing/$DOCKER_IMAGE_PUBLISH_NAME
# JOB_STOP_CONVENTIONAL_COMMITS: 'any_value'
# JOB_STOP_GIT_PUSH_MIRROR: 'any_value'
# GIT_SYNC_URL: "https://$GITHUB_USERNAME_ROBOT:$GITHUB_TOKEN_ROBOT@github.com/NoFussComputing/config.git" # Must be defined for job to run
# JOB_STOP_GITLAB_RELEASE: 'any value'
```

9
git_push_mirror/README.md → pages/projects/gitlab-ci/git_mirror.md
View File

@ -1,4 +1,11 @@
# git push mirror Admin Manual Manual
---
title: Git push mirror
description: How to use No Fuss Computings gitlab-ci job for repository mirrororing
date: 2021-08-03
template: manual.html
about: https://gitlab.com/nofusscomputing/projects/gitlab-ci
---
This job does a git push to a remote git repo.

11
gitlab_release/README.md → pages/projects/gitlab-ci/gitlab_release.md
View File

@ -1,4 +1,13 @@
# Commit Message Footer References - User Manual
---
title: Gitlab Release / Commit Footer References
description: How to use No Fuss Computings gitlab-ci job for Gitlab Releases and commit footer messages
date: 2021-08-03
template: manual.html
about: https://gitlab.com/nofusscomputing/projects/gitlab-ci
---
# User Manual
All commit messages must be in [conventional commit format](https://www.conventionalcommits.org/en/v1.0.0/) and have a footer with a gitlab reference. The reference **must** be either a merge request or a gitlab issue. (format i.e. `!1` or `#2` *using the correct reference number*).

10
python/README.md → pages/projects/gitlab-ci/python.md
View File

@ -1,4 +1,12 @@
# Python
---
title: Python
description: How to use No Fuss Computings gitlab-ci job for Python
date: 2021-08-11
template: manual.html
about: https://gitlab.com/nofusscomputing/projects/gitlab-ci
---
This folder (`python`) covers jobs for python development

17
yaml_lint/README.md → pages/projects/gitlab-ci/yaml_lint.md
View File

@ -1,4 +1,11 @@
# Yaml Lint
---
title: YAML Linting
description: How to use No Fuss Computings gitlab-ci job for YAML Linting
date: 2021-08-11
template: manual.html
about: https://gitlab.com/nofusscomputing/projects/gitlab-ci
---
This job does yaml linting when any commit is pushed to any branch.
@ -22,8 +29,8 @@ include:
Yaml Lint (python 3.6):
variables:
YAML_LINT_PATH: "roles/"
YAML_LINT_CONFIG: ".yamllint.yaml"
YAML_LINT_PATH: "."
YAML_LINT_EXTRA_ARGS: "{ extends: $ROOT_DIR/yaml_lint/.yamllint.yaml, ignore: [gitlab-ci/*] }"
extends:
- .yaml_linter_defaults
image: python:3.6-slim
@ -34,8 +41,8 @@ Yaml Lint (python 3.6):
| var name | Description |
|:----:|:----|
| YAML_LINT_PATH | *The path you wish the linter to search for yaml files* |
| YAML_LINT_CONFOG | *The path you have stored the yaml config file in* |
| YAML_LINT_PATH | *The path you wish the linter to search for yaml files, defaults to `.`* |
| YAML_LINT_EXTRA_ARGS | *configuration in yaml format., defaults to `{ extends: $ROOT_DIR/yaml_lint/.yamllint.yaml, ignore: [gitlab-ci/*] }` for further info see the [YAML Lint docs](https://yamllint.readthedocs.io/en/stable/configuration.html?highlight=exclude#custom-configuration-without-a-config-file)* |
## Job Workflow

222
type/docker-image.gitlab-ci.yaml Normal file
View File

@ -0,0 +1,222 @@
# This gitlab-ci file is used for creating docker images.
# by including this file, the jobs will be autocreated.
include:
- local: $JOB_ROOT_DIR/.gitlab-ci_common.yaml
- local: $JOB_ROOT_DIR/conventional_commits/.gitlab-ci.yml
- local: $JOB_ROOT_DIR/git_push_mirror/.gitlab-ci.yml
- local: $JOB_ROOT_DIR/gitlab_release/.gitlab-ci.yml
variables:
# Available platforms: linux/amd64, linux/amd64/v2, linux/amd64/v3, linux/arm64, linux/riscv64, linux/ppc64, linux/ppc64le, linux/s390x, linux/386, linux/mips64le, linux/mips64, linux/arm/v7, linux/arm/v6
# DOCKER_IMAGE_BUILD_TARGET_PLATFORMS: "linux/amd64,linux/arm64,linux/arm/v7"
DOCKER_IMAGE_BUILD_NAME: $CI_PROJECT_NAME
DOCKER_IMAGE_BUILD_REGISTRY: $CI_REGISTRY_IMAGE
DOCKER_IMAGE_BUILD_TAG: $CI_COMMIT_SHA
# DOCKER_IMAGE_PUBLISH_NAME: $CI_PROJECT_NAME
# DOCKER_IMAGE_PUBLISH_REGISTRY: docker.io/nofusscomputing
# DOCKER_IMAGE_PUBLISH_URL: https://hub.docker.com/r/nofusscomputing/$DOCKER_IMAGE_PUBLISH_NAME
# JOB_STOP_CONVENTIONAL_COMMITS: 'any_value'
# JOB_STOP_GIT_PUSH_MIRROR: 'any_value'
# GIT_SYNC_URL: "https://$GITHUB_USERNAME_ROBOT:$GITHUB_TOKEN_ROBOT@github.com/NoFussComputing/config.git" # Must be defined for job to run
# JOB_STOP_GITLAB_RELEASE: 'any value'
.build_docker_container:
stage: build
image: nofusscomputing/docker-buildx-qemu:dev
services:
- name: docker:23-dind
entrypoint: ["env", "-u", "DOCKER_HOST"]
command: ["dockerd-entrypoint.sh"]
variables:
DOCKER_HOST: tcp://docker:2375/
DOCKER_DRIVER: overlay2
# See https://github.com/docker-library/docker/pull/166
DOCKER_TLS_CERTDIR: ""
before_script:
- docker info
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
#- pip3 install -r gitlab-ci/gitlab_release/requirements.txt
- pip3 install setuptools wheel
- pip install -r $ROOT_DIR/conventional_commits/requirements.txt
- pip3 install gitlab-ci/gitlab_release/python-module/cz_nfc/.
- update-binfmts --enable # Important: Ensures execution of other binary formats is enabled in the kernel
- docker buildx create --driver=docker-container --driver-opt image=moby/buildkit:v0.11.6 --use
- docker buildx inspect --bootstrap
# --label org.opencontainers.image.url="$CI_PROJECT_URL/-/releases/$(cz -n cz_nfc version --project)" \
# --label org.opencontainers.image.version="$(cz -n cz_nfc version --project)" \
script: |
if [ "0$DOCKER_IMAGE_BUILD_TARGET_PLATFORMS" != "0" ]; then
echo "[DEBUG] building multiarch/specified arch image";
docker buildx build --platform=$DOCKER_IMAGE_BUILD_TARGET_PLATFORMS . \
--label org.opencontainers.image.created="$(date '+%Y-%m-%d %H:%M:%S%:z')" \
--label org.opencontainers.image.documentation="$CI_PROJECT_URL" \
--label org.opencontainers.image.source="$CI_PROJECT_URL" \
--label org.opencontainers.image.revision="$CI_COMMIT_SHA" \
--no-cache \
--push \
--tag $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
docker buildx imagetools inspect $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
# during docker multi platform build there are >=3 additional unknown images added to gitlab container registry. cleanup
DOCKER_MULTI_ARCH_IMAGES=$(docker buildx imagetools inspect "$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG" --format "{{ range .Manifest.Manifests }}{{ if ne (print .Platform) \"&{unknown unknown [] }\" }}$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG@{{ println .Digest }}{{end}} {{end}}");
docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
docker buildx imagetools inspect $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
else
echo "[DEBUG] building image";
docker build . \
--label org.opencontainers.image.created="$(date '+%Y-%m-%d %H:%M:%S%:z')" \
--label org.opencontainers.image.documentation="$CI_PROJECT_URL" \
--label org.opencontainers.image.source="$CI_PROJECT_URL" \
--label org.opencontainers.image.url="$CI_PROJECT_URL/-/releases/$(cz -n cz_nfc version --project)" \
--label org.opencontainers.image.version="$(cz -n cz_nfc version --project)" \
--label org.opencontainers.image.revision="$CI_COMMIT_SHA" \
--no-cache \
--tag $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
docker push $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
fi
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: $CI_COMMIT_TAG
when: on_success
- if: '$CI_COMMIT_BRANCH == "development"'
when: on_success
- if: '$CI_COMMIT_BRANCH != "master"'
when: always
- when: never
.publish-docker-hub:
stage: publish
image: docker:23-dind
services:
- docker:23-dind
before_script:
- |
docker login $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD;
if [ "0$DOCKER_IMAGE_BUILD_TARGET_PLATFORMS" != "0" ]; then
for i in ${DOCKER_IMAGE_BUILD_TARGET_PLATFORMS//,/ }
do
docker buildx imagetools inspect $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
#DOCKER_MULTI_ARCH_IMAGES=$(docker buildx imagetools inspect "$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG" --format "{{ range .Manifest.Manifests }}{{ if ne (print .Platform) \"&{unknown unknown [] }\" }}$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG@{{ println .Digest }}{{end}} {{end}}")
DOCKER_MULTI_ARCH_IMAGES=$(docker buildx imagetools inspect "$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG" --format "{{ range .Manifest.Manifests }}$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG@{{ println .Digest }} {{end}}")
echo "[DEBUG] DOCKER_MULTI_ARCH_IMAGES=$DOCKER_MULTI_ARCH_IMAGES";
done;
else
docker pull $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
fi
#docker logout $CI_REGISTRY; # commented out to test if for private repo remaining logged in works to allow pulling image
script:
- docker login docker.io -u $NFC_DOCKERHUB_USERNAME -p $NFC_DOCKERHUB_TOKEN
- docker image ls
- |
DOCKER_HUB_TAG=dev
if [ "0$CI_COMMIT_TAG" != "0" ]; then
DOCKER_HUB_TAG=latest
if [ "0$CI_COMMIT_TAG" == *"rc"* ]; then
DOCKER_HUB_TAG=dev
fi
fi
echo "[DEBUG] DOCKER_IMAGE_PUBLISH_NAME=$DOCKER_IMAGE_PUBLISH_NAME";
echo "[DEBUG] DOCKER_HUB_TAG=$DOCKER_HUB_TAG";
if [ "0$DOCKER_IMAGE_BUILD_TARGET_PLATFORMS" != "0" ]; then
echo "[DEBUG] DOCKER_MULTI_ARCH_IMAGES=$DOCKER_MULTI_ARCH_IMAGES";
docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:$DOCKER_HUB_TAG;
if [ "0$CI_COMMIT_TAG" != "0" ]; then
docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:$CI_COMMIT_TAG;
fi
else
docker image tag $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:$DOCKER_HUB_TAG;
docker push $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_HUB_TAG;
if [ "0$CI_COMMIT_TAG" != "0" ]; then
docker image tag $DOCKER_IMAGE_BUILD_NAME/$DOCKER_IMAGE_BUILD_REGISTRY:$DOCKER_IMAGE_BUILD_TAG $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:$CI_COMMIT_TAG;
docker push $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:$CI_COMMIT_TAG;
fi
fi
- docker logout docker.io
needs: [ "Docker Container" ]
environment:
name: DockerHub
url: $DOCKER_IMAGE_PUBLISH_URL
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: $CI_COMMIT_TAG
when: on_success
- if: '$CI_COMMIT_BRANCH == "development"'
when: on_success
- if: '$CI_COMMIT_BRANCH != "master"'
allow_failure: true
when: manual
- when: never
Docker Container:
extends: .build_docker_container
Docker Hub:
extends: .publish-docker-hub
Gitlab Release:
extends:
- .gitlab_release
Github (Push --mirror):
extends:
- .git_push_mirror

9
validation/.gitlab-ci.yml
View File

@ -2,9 +2,8 @@
image: node:alpine3.14
stage: validation
variables:
MDLINT_PATH: '**/*.md'
MDLINT_EXCLUDE_PATHS: '!gitlab-ci'
DEFAULT_ROOT_DIR: './gitlab-ci'
MDLINT_PATHS: "**/*.md **/**/*.md **/**/**/*.md **/**/**/**/*.md **/**/**/**/**/**/*.md #CHANGELOG.md !gitlab-ci"
DEFAULT_ROOT_DIR: "./gitlab-ci"
before_script:
- mkdir -p "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/lint_markdown"
- if [ "0$JOB_ROOT_DIR" == "0" ]; then ROOT_DIR=$DEFAULT_ROOT_DIR; else ROOT_DIR=$JOB_ROOT_DIR ; fi
@ -13,7 +12,8 @@
- npm install markdownlint-cli2 --global
- npm install markdownlint-cli2-formatter-junit --global
script:
- markdownlint-cli2 "$MDLINT_PATH" "$MDLINT_EXCLUDE_PATHS" 1>&1 || EXITCODE=$?
- echo "[DEBUG] MDLINT_PATHS=$MDLINT_PATHS"
- markdownlint-cli2 $MDLINT_PATHS 1>&1 || EXITCODE=$?
- echo DEBUG EXITCODE[$EXITCODE]
- rm "$CI_PROJECT_DIR/.markdownlint-cli2.jsonc"
- mv *.junit.xml "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/lint_markdown/markdown_lint.junit.xml"
@ -29,5 +29,6 @@
- if: '$CI_COMMIT_BRANCH'
exists:
- "**.md"
- "**/**.md"
- when: never

1
website-template Submodule

Submodule website-template added at 969912bcb7

17
yaml_lint/.gitlab-ci.yml
View File

@ -1,8 +1,8 @@
.yaml_lint_defaults:
variables:
YAML_LINT_PATH: "/*.yml"
YAML_LINT_CONFIG: "/yaml_lint/.yamllint.yaml"
YAML_LINT_PATH: "."
YAML_LINT_EXTRA_ARGS: "{ extends: $ROOT_DIR/yaml_lint/.yamllint.yaml, ignore: [gitlab-ci/*] }"
stage: validation
before_script:
- if [ "0$JOB_ROOT_DIR" == "0" ]; then ROOT_DIR=gitlab-ci; else ROOT_DIR=$JOB_ROOT_DIR ; fi
@ -18,11 +18,9 @@
- pip install -r $ROOT_DIR/yaml_lint/requirements.txt
- mkdir $PYTHON_VERSION
script:
- YAML_LINT_PATH=$ROOT_DIR$YAML_LINT_PATH
- echo "[DEBUG] YAML_LINT_PATH[$YAML_LINT_PATH]"
- YAML_LINT_CONFIG=$ROOT_DIR$YAML_LINT_CONFIG
- echo "[DEBUG] YAML_LINT_CONFIG[$YAML_LINT_CONFIG]"
- yamllint $YAML_LINT_PATH -f parsable -c $ROOT_DIR/yaml_lint/.yamllint.yaml > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/$PYTHON_VERSION-yaml-lint.log" 1>&1 || YAML_LINT=$?
- yamllint $YAML_LINT_PATH -f parsable -d "$(echo $YAML_LINT_EXTRA_ARGS)" > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/$PYTHON_VERSION-yaml-lint.log" 1>&1 || YAML_LINT=$?
- cat "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/$PYTHON_VERSION-yaml-lint.log" | ansible-lint-to-junit-xml > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/$PYTHON_VERSION-yaml-lint.junit.xml"
- if [ $YAML_LINT > 0 ]; then echo "YAML lint failed with $YAML_LINT"; exit $YAML_LINT; fi
after_script:
@ -36,7 +34,14 @@
junit:
- "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/*.junit.xml"
rules:
- if: '$JOB_STOP_YAML_LINT'
when: never
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_TAG'
when: always
- if: '$CI_COMMIT_BRANCH == "master"'
when: always
- if: '$CI_COMMIT_BRANCH'
when: always
- when: never