.gitlab-ci.yml

@@ -1,10 +1,13 @@
 variables:
     JOB_ROOT_DIR: '.'
+    SECURE_LOG_LEVEL: debug
     GIT_SUBMODULE_STRATEGY: recursive
     MY_PROJECT_ID: "28543717"
+    LICENSE_FINDER_CLI_OPTS: '--recursive'
 
 stages:
     - validation
+    - test
     - release
     - sync
 
@@ -13,7 +16,83 @@ include:
   - local: $JOB_ROOT_DIR/conventional_commits/.gitlab-ci.yml
   - local: $JOB_ROOT_DIR/git_push_mirror/.gitlab-ci.yml
   - local: $JOB_ROOT_DIR/gitlab_release/.gitlab-ci.yml
+  - local: $JOB_ROOT_DIR/validation/.gitlab-ci.yml
   - local: $JOB_ROOT_DIR/python/.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml
+  - template: Security/License-Scanning.gitlab-ci.yml
+
+# Scanner doesn't Pickup multiple pip files. Disable and specify jobs with pip file.
+gemnasium-python-dependency_scanning:
+  rules:
+    - when: never
+
+# source: https://gitlab.com/gitlab-org/gitlab/-/blob/2f33a8cb4dcea7b875e360d4cd9e016e027d2973/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+.gemnasium-python-dependency_scanning:
+  extends: .ds-analyzer
+  image:
+    name: "$DS_ANALYZER_IMAGE"
+  variables:
+    # Python 3.7 not available
+#    DS_PYTHON_VERSION: "3.7"
+    # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to
+    # override the analyzer image with a custom value. This may be subject to change or
+    # breakage across GitLab releases.
+    DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION"
+    # Stop reporting Pipenv and Setuptools as "pip".
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252
+    DS_REPORT_PACKAGE_MANAGER_PIP_WHEN_PYTHON: "false"
+  # Can't find pillow 9.0, check python version. 3.6 only available, 3.7 not.
+  before_script:
+    - python --version
+    # Bug reported at gitlab-org/gitlab#350949. 
+    # workaround, delete all other pip files except the one to be scanned
+    # Delete all existing pipfiles except the one to be scanned.
+    - for found_file in $(find . -type f -name 'requirements.txt'); do if [ $found_file != "./$PIP_REQUIREMENTS_FILE" ]; then echo "[DEBUG] Removing '$found_file' due to bug reported at gitlab-org/gitlab#350949"; rm -f $found_file; fi done;
+  rules:
+    - if: $DEPENDENCY_SCANNING_DISABLED
+      when: never
+        # Support passing of $PIP_REQUIREMENTS_FILE
+        # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning
+    - if: $CI_COMMIT_BRANCH &&
+          $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
+          $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ &&
+          $PIP_REQUIREMENTS_FILE
+
+Ansible Dependencies:
+  extends: .gemnasium-python-dependency_scanning
+  variables:
+    PIP_REQUIREMENTS_FILE: ansible/requirements.txt
+
+
+conventional_commits Dependencies:
+  extends: .gemnasium-python-dependency_scanning
+  variables:
+    PIP_REQUIREMENTS_FILE: conventional_commits/requirements.txt
+
+
+gitlab_release Dependencies:
+  extends: .gemnasium-python-dependency_scanning
+  variables:
+    PIP_REQUIREMENTS_FILE: gitlab_release/requirements.txt
+
+
+mkdocs Dependencies:
+  extends: .gemnasium-python-dependency_scanning
+  variables:
+    PIP_REQUIREMENTS_FILE: mkdocs/requirements.txt
+
+
+python Dependencies:
+  extends: .gemnasium-python-dependency_scanning
+  variables:
+    PIP_REQUIREMENTS_FILE: python/requirements.txt
+
+
+yaml_lint Dependencies:
+  extends: .gemnasium-python-dependency_scanning
+  variables:
+    PIP_REQUIREMENTS_FILE: yaml_lint/requirements.txt
+
 
 PyLint:
     extends:
@@ -29,6 +108,11 @@ gilab-ci.yml Lint (python 3.6):
     image: python:3.6-slim
 
 
+Markdown Linting:
+  extends:
+    - .Lint_Markdown
+
+
 Gitlab Release:
     variables:
         MY_COMMAND: ./sub-folder_changlog.sh

.markdownlint.json

@@ -0,0 +1,9 @@
+{
+  "line-length": false,
+  "MD007": {
+    "indent": 4
+  },
+  "MD033": {
+    "allowed_elements": [ "div", "s", "span", "u" ]
+  }
+}

ansible/requirements.txt

@@ -1,3 +1,11 @@
+# Dependency scanning only has Python 3.6 available.
+
+# Python 3.7
+#   Pillow==9.0.0
+#   numpy==1.22.0
+
+# Python 3.6
+pipenv==2022.1.8
 wheel 
 ansible
 setuptools_rust 

mkdocs/.gitlab-ci.yml

@@ -0,0 +1,29 @@
+.MKDocs_Build:
+  image: python:3.7.5-buster
+  stage: build
+  variables:
+    GIT_DEPTH: 0
+    MKDOCS_BUILD_PATH: build
+    MKDOCS_SOURCE_PATH: pages
+    DEFAULT_ROOT_DIR: './gitlab-ci'
+  before_script:
+    - mkdir -p "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME"
+    - if [ "0$JOB_ROOT_DIR" == "0" ]; then ROOT_DIR=$DEFAULT_ROOT_DIR; else ROOT_DIR=$JOB_ROOT_DIR ; fi
+    - echo "[DEBUG] ROOT_DIR[$ROOT_DIR]"
+    - if [ ! -f "requirements.txt" ]; then pip install --upgrade pip -r $ROOT_DIR/mkdocs/requirements.txt; fi
+    - if [ -f "requirements.txt" ]; then pip install --upgrade pip -r $ROOT_DIR/mkdocs/requirements.txt; pip install -r requirements.txt; fi
+  script:
+    - mkdocs build --clean --strict
+    - mv "$MKDOCS_BUILD_PATH" "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/$MKDOCS_BUILD_PATH"
+    - if [ "0$MKDOCS_INCLUDE_SOURCE" != "0" ]; then cp $MKDOCS_SOURCE_PATH "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/$MKDOCS_SOURCE_PATH"; fi
+    - ls -laR "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME"
+  artifacts:
+    expire_in: 24 hrs
+    paths:
+      - "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME"
+  rules:
+    - if: '$CI_COMMIT_BRANCH'
+      exists:
+        - "mkdocs.yml"
+    - when: never
+

mkdocs/build.md

@@ -0,0 +1,58 @@
+# MKDocs Static Site Build
+Build a MKDocs site from the config specified in `mkdocs.yml`. _Only runs if `mkdocs.yml` exists in the repository root directory._ This job is designated to run on all branchs so that you can use the artifacts for deployment to `staging` and/or `production` as required.
+
+This job provides the following badge:
+
+- _None_
+
+## Dependencies
+
+- **Mandatory** file `mkdocs.yml` in the repository root directory with your MKDocs configuration
+
+## your .gitlab-ci.yml changes
+To use this job add the following to your `.gitlab-ci.yml` file
+
+``` yaml
+stages:
+    - build
+
+include:
+    - local: CI/build/.gitlab-ci.yml
+
+MKDocs build:
+  variables:
+    MKDOCS_BUILD_PATH: "build"
+  extends:
+    - .MKDocs_Build
+
+```
+
+## CI/CD Variables required
+
+| var name | Description |
+|:----:|:----|
+| MKDOCS_BUILD_PATH | **Mandatory, if different from default** The path where MKDocs places the build files. Defaults to `build` |
+| MKDOCS_INCLUDE_SOURCE | **Optional** Include the build source files in the artifacts. Default is Not set. Any value in this variable, will include the source files. |
+| MKDOCS_SOURCE_PATH | **Optional, if source files are not to be included** Set to the path where mkdocs uses to build the static html. |
+
+## Job Workflow
+
+1. install mkdocs
+
+    1. if file `requirements.txt` exists in the repository root directory, use this fill to also install additional dependencies.
+
+    1. if no `requirements.txt` file exists, only install mkdocs.
+
+1. run mkdocs to build the static pages
+
+1. if variable `$MKDOCS_INCLUDE_SOURCE` is set, then copy `$MKDOCS_SOURCE_PATH` to the artifacts location.
+
+1. copy directory `$MKDOCS_BUILD_PATH` to the artifacts location.
+
+
+## Artifacts
+
+- files in `"$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME"`
+
+## License
+To view the license for this folder and any sub-folders, refer [here](https://gitlab.com/nofusscomputing/projects/gitlab-ci)

mkdocs/requirements.txt

@@ -0,0 +1 @@
+mkdocs==1.2.3

validation/.gitlab-ci.yml

@@ -0,0 +1,34 @@
+.Lint_Markdown:
+  image: node:alpine3.14
+  stage: validation
+  variables:
+    MDLINT_PATH: '"**/*.md"'
+    MDLINT_EXCLUDE_PATHS: '"!gitlab-ci"'
+    DEFAULT_ROOT_DIR: './gitlab-ci'
+  before_script:
+    - mkdir -p "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME"
+    - mkdir -p "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests"
+    - if [ "0$JOB_ROOT_DIR" == "0" ]; then ROOT_DIR=$DEFAULT_ROOT_DIR; else ROOT_DIR=$JOB_ROOT_DIR ; fi
+    - echo "[DEBUG] ROOT_DIR[$ROOT_DIR]"
+    - cp -f "$CI_PROJECT_DIR/validation/.markdownlint-cli2.jsonc" "$CI_PROJECT_DIR/.markdownlint-cli2.jsonc"
+    - npm install markdownlint-cli2 --global
+    - npm install markdownlint-cli2-formatter-junit --global
+  script:
+    - markdownlint-cli2 $MDLINT_PATH $MDLINT_EXCLUDE_PATHS 1>&1 || EXITCODE=$?
+    - echo DEBUG EXITCODE[$EXITCODE]
+    - rm "$CI_PROJECT_DIR/.markdownlint-cli2.jsonc"
+    - mv *.junit.xml $CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/markdown.junit.xml
+  artifacts:
+    expire_in: 24 hrs
+    when: always
+    paths:
+      - "$CI_PROJECT_DIR/artifacts/*"
+    reports:
+      junit:
+        - "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/*.junit.xml"
+  rules:
+    - if: '$CI_COMMIT_BRANCH'
+      exists:
+        - "**.md"
+    - when: never
+

validation/.markdownlint-cli2.jsonc

@@ -0,0 +1,5 @@
+{
+  "outputFormatters": [
+    [ "markdownlint-cli2-formatter-junit", { "name": "markdown.junit.xml" } ]
+  ]
+}

validation/markdown_lint.md

@@ -0,0 +1,49 @@
+# Markdown File Linting
+This job lints markdown files as part of the validation CI stage. It is designated to run on all branches. If any errors are found, the generated JUnit test report will let you know what errors were found.
+
+You can include your linting rules in `.markdownlint.json` which should be within the root of your repository. for the available rules please see the [docs](https://github.com/DavidAnson/markdownlint/blob/main/README.md#rules--aliases).
+
+This job provides the following badge:
+
+- _None_
+
+## Dependencies
+
+- **Optional** file `.markdownlint.json` in repository root with any rules you wish to specify
+
+## your .gitlab-ci.yml changes
+To use this job add the following to your `.gitlab-ci.yml` file
+
+``` yaml
+
+stages:
+    - validation
+
+include:
+    - local: CI/validation/.gitlab-ci.yml
+
+Markdown Linting:
+  extends:
+    - .Lint_Markdown
+
+```
+
+## CI/CD Variables required
+
+| var name | Description |
+|:----:|:----|
+| MDLINT_PATH | **Optional** specifies the path to lint. defaults to `"**/*.md"` |
+| MDLINT_EXCLUDE_PATHS | **optional** Specifies the paths to exclude from linting. Defaults to `"!gitlab-ci"` |
+
+## Job Workflow
+
+1. installs the required job dependencies
+
+1. Lints any markdow file found in `$MDLINT_PATH`, excluding paths `$MDLINT_EXCLUDE_PATHS`
+
+## Artifacts
+
+- JUnit test report located at `$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/*.junit.xml`
+
+## License
+To view the license for this folder and any sub-folders, refer [here](https://gitlab.com/nofusscomputing/projects/gitlab-ci)