From d61056243804728e059b99fce1644a8cc37230bb Mon Sep 17 00:00:00 2001 From: Jon Lockwood Date: Wed, 11 Aug 2021 14:17:29 +0930 Subject: [PATCH] feat(python_linting): added ci job, python linting, code quality and scoring Added a job to lint python files and ouput a code quality report for gitlab MR. A html report is also available within the artifacts. if the badge is used, the badge links to the html code quality report. Changes to be committed: modified: .gitlab-ci.yml new file: python/.gitlab-ci.yml new file: python/README.md new file: python/requirements.txt issue #1 --- .gitlab-ci.yml | 6 +++ python/.gitlab-ci.yml | 49 +++++++++++++++++++++++ python/README.md | 88 +++++++++++++++++++++++++++++++++++++++++ python/requirements.txt | 2 + 4 files changed, 145 insertions(+) create mode 100644 python/.gitlab-ci.yml create mode 100644 python/README.md create mode 100644 python/requirements.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e364bc..d3545c7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,12 @@ include: - local: $JOB_ROOT_DIR/conventional_commits/.gitlab-ci.yml - local: $JOB_ROOT_DIR/git_push_mirror/.gitlab-ci.yml - local: $JOB_ROOT_DIR/gitlab_release/.gitlab-ci.yml + - local: $JOB_ROOT_DIR/python/.gitlab-ci.yml + +PyLint: + extends: + - .PyLint + image: python:3.6-slim gilab-ci.yml Lint (python 3.6): diff --git a/python/.gitlab-ci.yml b/python/.gitlab-ci.yml new file mode 100644 index 0000000..824025b --- /dev/null +++ b/python/.gitlab-ci.yml @@ -0,0 +1,49 @@ + +.pylint_badge: &pylint_badge | + echo "{ + \"PyLintScore\": \"$PyPIScore\" + }" > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME/badge_pylint.json" + +.PyLint: + variables: + PYLINT_PATH: "/*/python-module/*/*.py" + stage: validation + before_script: + - if [ "0$JOB_ROOT_DIR" == "0" ]; then ROOT_DIR=gitlab-ci; else ROOT_DIR=$JOB_ROOT_DIR ; fi + - echo "[DEBUG] ROOT_DIR[$ROOT_DIR]" + - mkdir -p "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/$CI_JOB_NAME" + - mkdir -p "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests" + - export PYTHON_VERSION=`python -c 'import sys; version=sys.version_info[:3]; print("{0}.{1}.{2}".format(*version))'` + - apt update + - apt install --no-install-recommends -y git + - python3 -m venv env + - . env/bin/activate + - pip install --upgrade pip + - pip install -r $ROOT_DIR/python/requirements.txt + - mkdir $PYTHON_VERSION + script: + - PYLINT_PATH=$ROOT_DIR$PYLINT_PATH + - echo "[DEBUG] PYLINT_PATH[$PYLINT_PATH]" + - python3 -m pylint --exit-zero --output-format=pylint_gitlab.GitlabCodeClimateReporter $PYLINT_PATH > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/gl-code-quality-report.json" + + - python3 -m pylint --exit-zero --output-format=pylint_gitlab.GitlabPagesHtmlReporter $PYLINT_PATH > "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/gl-code-quality-report.html" + + - PyPIScore=$(python3 -m pylint --exit-zero $PYLINT_PATH | sed -n 's/^Your code has been rated at \([-0-9./]*\).*/\1/p') + + - *pylint_badge + + + after_script: + - echo deactivate + artifacts: + expire_in: 1 days + when: always + paths: + - "artifacts/*" + reports: + codequality: "$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/gl-code-quality-report.json" + rules: + - if: '$CI_COMMIT_BRANCH' + when: always + - when: never + diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..33802ab --- /dev/null +++ b/python/README.md @@ -0,0 +1,88 @@ +# Python +This folder (`python`) covers jobs for python development + + +These jobs provides the following badge: + +- `PyLint` - code quality [![PyLint Score](https://img.shields.io/badge/dynamic/json?&style=plastic&logo=python&label=PyLint%20Score&query=%24.PyLintScore&url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fgitlab-ci%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fraw%2Fartifacts%2Fvalidation%2FPyLint%2Fbadge_pylint.json%3Fjob%3DPyLint)](https://gitlab.com/nofusscomputing/projects/gitlab-ci/-/jobs/artifacts/development/file/artifacts/validation/tests/gl-code-quality-report.html?job=PyLint) + +Use the following MD to add a badge adjusting the variables and ensuring everything is on one line. +``` md +[![PyLint Score](https://img.shields.io/badge/dynamic/json?&style=plastic&logo=python&label=PyLint%20Score&query=%24.PyLintScore&url=https%3A%2F%2Fgitlab.com%2F + +{project path} + +%2F-%2Fjobs%2Fartifacts%2F + +{branch} + +%2Fraw%2Fartifacts%2Fvalidation%2FPyLint%2Fbadge_pylint.json%3Fjob%3D + +{Job Name} + +)](https://gitlab.com/ + +{project path} + +/-/jobs/artifacts/ + +{branch} + +/file/ + +artifacts/validation/tests/gl-code-quality-report.html + +?job= + +{Job Name} +) +``` +| Variable | Description | +|:----|:----| +| `{project path}` | *project path, what's after gitlab.com/* | +| `{branch}` | *git branch to fetch the score from* | +| `{Job Name}` | *name of the gitlab-ci job for the linting* | + + +## Dependencies + +- None + +## your .gitlab-ci.yml changes +To add the `PyLint` job, add the following to your `.gitlab-ci.yml` file + +``` yaml +stages: + - validation + +include: + - remote: https://gitlab.com/nofusscomputing/projects/gitlab-ci/-/raw/master/python/.gitlab-ci.yml + +PyLint: + variables: + PYLINT_PATH: "/*/*.py" + extends: + - .PyLint + image: python:3.6-slim +``` + + +## CI/CD Variables required + +| var name | Description | +|:----:|:----| +| PYLINT_PATH | *The path you wish the linter to search for python files* | + + +## Job Workflow + + - This job will lint any yaml file in the specified directory using the specified rules. + +## Artifacts + + - `$CI_PROJECT_DIR/artifacts` - Root artifact directory + - `$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/gl-code-quality-report.json` - Gitlab code quality report (displays in merge request) + - `$CI_PROJECT_DIR/artifacts/$CI_JOB_STAGE/tests/gl-code-quality-report.html` - html code quality report + +## License +To view the license for this folder and any sub-folders, refer [here](https://gitlab.com/nofusscomputing/projects/gitlab-ci) diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 0000000..bc478a8 --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,2 @@ +pylint==2.9.6 +pylint-gitlab==0.3.0