infrastructure/python-gitlab-management
2
0
Fork 0
Code Issues Pull Requests Actions Projects Releases 29 Wiki Activity

#6 ~Feature Docker Image with the module pre-installed. Image based off of Alpine.

Browse Source
This commit is contained in:
Jon Lockwood
2020-06-07 15:51:42 +00:00
parent 5e3cd9ba1f
commit ee011f14c7
18 changed files with 418 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
.dockerignore Normal file
View File

@ -0,0 +1,22 @@
# Directories
**/.git
**/.gitlab
bin/
build/
docs/
gitlab_management/
*.egg-info/
include/
lib/
test/
# Files
**/*.md
**/*.py
**/*.pyc
**/*.yml
**/.gitignore
**/pyvenv.cfg

5
.gitignore vendored
View File

@ -1,6 +1,10 @@
config.yml
*.pyc
#Dont include files generated by buildinit.py
dockerfile
gitlab_management/__init__.py
#Dont include build files
build/
_build/
@ -11,6 +15,7 @@ dist/
bin/
lib/
include/
pyvenv.cfg
#Don't include the autogenerated Module docs

194
.gitlab-ci.yml
View File

@ -3,6 +3,7 @@
stages:
- package
- build
- test
- validate
@ -10,45 +11,28 @@ stages:
- publish
Create_Package:
stage: build
gitlab-management_package:
stage: package
image: python:3.6.9-slim
variables:
GIT_PYTHON_GIT_EXECUTABLE: /bin/git
before_script:
- apt-get update && apt-get install -y --no-install-recommends git
- git --version
- GIT_PYTHON_GIT_EXECUTABLE=$(which git)
- echo $GIT_PYTHON_GIT_EXECUTABLE
- echo $PATH
- PATH=$PATH:$GIT_PYTHON_GIT_EXECUTABLE
- python3 -m pip install --user --upgrade setuptools wheel
- python3 -m pip install -r requirements.txt
script:
- python3 setup.py sdist bdist_wheel
- echo $(python3 -B ./GetVersion.py) > dist/version
- python3 setup.py egg_info sdist bdist_wheel
rules:
- if: $CI_COMMIT_TAG
when: never
- if: $CI_MERGE_REQUEST_IID
when: never
- if: '$CI_COMMIT_BRANCH == "master"'
when: always
artifacts:
expire_in: 3 days
when: on_success
paths:
- dist/*
Create_Pre-Release-Package:
stage: build
image: python:3.6.9-slim
before_script:
- python3 -m pip install --user --upgrade setuptools wheel
script:
- Tag_Build=.rc$(date +%y%m%d%H%M)
- python3 setup.py egg_info --tag-build=$Tag_Build sdist bdist_wheel
- echo $(python3 -B ./GetVersion.py)$Tag_Build > dist/version
rules:
- if: $CI_COMMIT_TAG
when: never
- if: $CI_MERGE_REQUEST_IID
when: never
- if: '$CI_COMMIT_BRANCH == "master"'
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
- if: '$CI_COMMIT_BRANCH'
changes:
- gitlab_management/*.py
- setup.py
@ -60,22 +44,142 @@ Create_Pre-Release-Package:
when: on_success
paths:
- dist/*
- gitlab_management/__init__.py
- dockerfile
Docker_Build-Alpine:
stage: build
image: docker:latest
services:
- docker:19.03.11-dind
before_script:
- docker info
- docker login $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD
script:
- docker build . --no-cache --tag $CI_REGISTRY_IMAGE/gitlab-management:$CI_COMMIT_SHA
after_script:
- docker push $CI_REGISTRY_IMAGE/gitlab-management:$CI_COMMIT_SHA
rules:
- if: $CI_COMMIT_TAG
when: never
- if: $CI_MERGE_REQUEST_IID
when: never
- if: '$CI_COMMIT_BRANCH'
changes:
- gitlab_management/*.py
- setup.py
- README.md
- CONTRIBUTING.md
when: on_success
dependencies:
- gitlab-management_package
tags:
- docker
include:
- template: Dependency-Scanning.gitlab-ci.yml
- template: Container-Scanning.gitlab-ci.yml
- template: License-Scanning.gitlab-ci.yml
- template: SAST.gitlab-ci.yml
variables:
SAST_DEFAULT_ANALYZERS: "bandit"
# to activate security MR approvals: https://docs.gitlab.com/ee/user/application_security/index.html#enabling-security-approvals-within-a-project
bandit-sast:
dependencies:
- gitlab-management_package
rules:
- if: $CI_COMMIT_TAG
when: never
- if: $CI_MERGE_REQUEST_IID
when: never
- if: $CI_COMMIT_BRANCH
changes:
- gitlab_management/*.py
- setup.py
when: on_success
allow_failure: false
Scan gitlab-management-Alpine:
extends: container_scanning
variables:
CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/gitlab-management
dependencies:
- gitlab-management_package
- Docker_Build-Alpine
rules:
- if: $CI_COMMIT_TAG
when: never
- if: $CI_MERGE_REQUEST_IID
when: never
- if: $CI_COMMIT_BRANCH
changes:
- gitlab_management/*.py
- setup.py
when: on_success
allow_failure: false
container_scanning:
rules:
- if: $CI_COMMIT_TAG
when: never
- if: $CI_MERGE_REQUEST_IID
when: never
- if: $CI_COMMIT_BRANCH
when: never
gemnasium-python-dependency_scanning:
variables:
DS_PYTHON_VERSION: 3
PIP_REQUIREMENTS_FILE: requirements.txt
DS_PYTHON_VERSION: 3
PIP_REQUIREMENTS_FILE: requirements.txt
dependencies:
- gitlab-management_package
- Docker_Build-Alpine
rules:
- if: $CI_COMMIT_TAG
when: never
- if: $CI_MERGE_REQUEST_IID
when: never
- if: $CI_COMMIT_BRANCH
changes:
- gitlab_management/*.py
- setup.py
when: on_success
allow_failure: false
# to activate licence approvals: https://docs.gitlab.com/ee/user/application_security/#enabling-license-approvals-within-a-project
license_scanning:
variables:
LM_PYTHON_VERSION: 3
dependencies:
- gitlab-management_package
- Docker_Build-Alpine
rules:
- if: $CI_COMMIT_TAG
when: never
- if: $CI_MERGE_REQUEST_IID
when: never
- if: $CI_COMMIT_BRANCH
changes:
- gitlab_management/*.py
- setup.py
when: on_success
allow_failure: false
Documentation:
stage: validate
image: readthedocs/build:latest
script:
- bash $CI_PROJECT_DIR/test/validation-build-docs.sh
dependencies:
- gitlab-management_package
rules:
- if: $CI_COMMIT_TAG
when: never
@ -90,7 +194,7 @@ Documentation:
- docs/*
- README.md
- CONTRIBUTING.md
when: always
when: on_success
@ -117,17 +221,18 @@ GitLab-Release:
script:
- Release_TAG=$(cat $CI_PROJECT_DIR/dist/version)
- *ReleaseScript
#dependencies:
# - Create_Package
dependencies:
- gitlab-management_package
- Docker_Build-Alpine
rules:
- if: $CI_COMMIT_TAG
when: never
- if: $CI_MERGE_REQUEST_IID
when: never
- if: '$CI_COMMIT_BRANCH == "master"'
when: always
when: on_success
- if: '$CI_COMMIT_BRANCH == "development"'
when: always
when: on_success
allow_failure: false
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
@ -143,8 +248,9 @@ Test_Publish:
script:
- rm $CI_PROJECT_DIR/dist/version
- python3 -m twine upload --verbose --username __token__ --password $NFC_TOKEN_TESTPYPI --repository testpypi dist/*
# dependencies:
# - Create_Package
dependencies:
- gitlab-management_package
- Docker_Build-Alpine
rules:
- if: $CI_COMMIT_TAG
when: never
@ -172,14 +278,16 @@ Publish:
script:
- rm $CI_PROJECT_DIR/dist/version
- python3 -m twine upload --verbose --username __token__ --password $NFC_TOKEN_PYPI dist/*
#dependencies:
# - GitLab-Release
dependencies:
- gitlab-management_package
- Docker_Build-Alpine
rules:
- if: $CI_COMMIT_TAG
when: never
- if: $CI_MERGE_REQUEST_IID
when: never
- if: '$CI_COMMIT_BRANCH == "master"'
when: always
when: on_success
- if: '$CI_COMMIT_BRANCH == "development"'
when: always
when: manual
allow_failure: true

37
CONTRIBUTING.md
View File

@ -11,7 +11,7 @@ our development workflow is designed to be easy to setup. For this to occur, the
## Module
Setup the environment to use the local module baing developed.
Setup the environment to use the local module being developed.
>**Note:** all commands run from the root of the repository directory.
@ -27,18 +27,34 @@ bin/python -m pip install --upgrade --no-cache-dir pip
bin/python -m pip install --upgrade --no-cache-dir -r requirements.txt
bin/python setup.py develop install
python3 ./buildinit.py
```
test the module being developed
``` bash
bin/python -B -m gitlab_management -h
bin/gitlab_management -h
```
Clean-up the environment
``` bash
rm -Rf bin dist gitlab_management.egg-info lib include
rm -Rf build bin docs/_build gitlab_management.egg-info lib include pyvenv.cfg
```
>**Note:** you must increment the version in the build script (`buildinit.py`) prior to committing your final changes to the repo.
### Version Changes
Every change, prior to being committed to the `development` branch, must have it's version incremented. In most cases only the `{newfeature}` number will need to be incremented.
if the version is less than `1.0.0` the version number layout is `0.{newfeature}.{bugfix}`
if the version number is more than `1.0.0` the version number layout is `{majorchange}.{newfeature}.{bugfix}`.
`{majorchange}` should only be incremented if the end user would be required to make a manual adjustments to use the version.
To adjust the version, open `buildinit.py` and edit the `__version__` property at the top of the script.
Regardless of the version until the changes are pushed to the `development` branch, it will be adjusted to be a release candidate. the appended version information will be `.rc` with a formated date i.e. `.rcYYMMDDHHMMSS`.
## Documentation
We use Sphinx for ducumentation. All documentation for the project lives in the `docs/` directory within the repo. Documentation can be written in either RestructureText `.rst` or MarkDown `.md` files. Documentation for the modules however, is written within the code using `NumPy` style `DocBook` code comments. These comments are then rendered to `.rst` via the `sphinx.ext.autodoc` extension.
@ -136,11 +152,18 @@ Buiding of the docs is part of the *validation* stage of the build pipeline. The
Please build the docs locally before pushing to the repo, this will ensure that they are working befor being committed. After a local build the docs can be found in the repository directory under `docs/_build/html/index.html`
Clean-up the environment prior to building the documentation, with:
``` bash
cd {Repo Directory}
docker run -e CI_PROJECT_DIR=/Repository -w /Repository -v $PWD:/Repository readthedocs/build:latest bash test/validation-build-docs.sh
rm -Rf build bin docs/_build gitlab_management.egg-info lib include pyvenv.cfg
```
>**.. Note::**
Substitute `{Repo Directory}` with the actual directory the repository was cloned to. However, you can also miss the `cd` command and adjust `$PWD` to be the `{Repo Directory}`.
To build the documentation run:
``` bash
cd {Repo Directory}
CI_PROJECT_DIR=/Repository && docker run -e CI_PROJECT_DIR=$CI_PROJECT_DIR -w $CI_PROJECT_DIR -v $PWD:$CI_PROJECT_DIR readthedocs/build:latest bash test/validation-build-docs.sh
```
>**Tip:**
The above command contains the setting of the environmental variable `CI_PROJECT_DIR`, this is the directory where the repository is to be mounted to within the container. *don't include a trailing `/` slash if you change this.* The variable `$PWD` is to mount the current directory, if you are not running this command from the repo root dir, please set this to the full path of the repo.

7
GetVersion.py
View File

@ -1,7 +0,0 @@
#!/usr/bin/env python3
#-*- coding: utf-8 -*-
with open("gitlab_management/base.py") as f:
for line in f:
if line.startswith("__version__"):
print(eval(line.split("=")[-1]))

2
README.md
View File

@ -21,7 +21,7 @@ For please refer to the docs.
To install run `pip install gitlab-management`. from there you will be able to use and extend the module as you see fit.
### CLI
Gitlab-management can be used via cli with command `python3 -m gitlab-management -H {GITLAB_URL} -T {GITLAB_PRIVATE_TOKEN}` replacing `{GITLAB_URL}` with the gitlab url and `{GITLAB_PRIVATE_TOKEN}` with your gitlab private token. to view all available options, use switch `-h` or `--help`
Gitlab-management can be used via cli with `gitlab-management`, there is no requirement to prefix with `python3 -m`, as the command is registered on install. To view all available options, use switch `-h` or `--help`
### Config File
The configuration file for this module (`config.yml`) is a yml formated file that is required to be in the directory that the command is ran from.

153
buildinit.py Normal file
View File

@ -0,0 +1,153 @@
#!/usr/bin/env python3
#-*- coding: utf-8 -*-
import sys, git
import os.path
from datetime import datetime
CurrentRepo = None
GIT_COMMIT = ''
GIT_BRANCH = ''
__author__ = "No Fuss Computing"
__email__ = "helpdesk@nofusscomputing.com"
__license__ = "GNU LGPLv3"
__copyright__ = "(C) All Rights reserved"
__source__ = ''
__title__ = "gitlab-management"
__version__ = "0.1.3"
__doc__ = "https://gitlab.com/nofusscomputing/projects/python-gitlab-management"
CurrentDirectory = './'
DistributionDirectory = 'dist'
ModuleDirectory = CurrentDirectory + 'gitlab_management/'
VersionFile = CurrentDirectory + DistributionDirectory + '/version'
OutputPrefix = '[BuildInitScript] '
def init() -> bool:
init = True
global __source__, __version__, CurrentRepo, GIT_BRANCH, GIT_COMMIT
if os.path.isdir('./.git'):
CurrentRepo = git.Repo(search_parent_directories=True)
GIT_COMMIT = CurrentRepo.head.object.hexsha
# Check if running within a GitLab Runner
if os.environ.get('CI_COMMIT_BRANCH') is None and os.environ.get('CI_JOB_ID') is None:
GIT_BRANCH = CurrentRepo.head.reference.name
else:
GIT_BRANCH=os.environ.get('CI_COMMIT_BRANCH')
print(OutputPrefix + 'Git Commit: ' + GIT_COMMIT)
print(OutputPrefix + 'Git Branch: ' + GIT_BRANCH)
__source__ = 'https://gitlab.com/nofusscomputing/projects/python-gitlab-management/-/tree/' + GIT_COMMIT
if GIT_BRANCH != 'master':
__version__ = __version__ + '.rc' + datetime.utcnow().strftime('%y%m%d%H%M%S')
if not os.path.isdir(CurrentDirectory + DistributionDirectory):
os.makedirs(CurrentDirectory + DistributionDirectory)
if not os.path.isdir(CurrentDirectory + DistributionDirectory):
print(OutputPrefix + 'couldnt Make Directory:' + CurrentDirectory + DistributionDirectory)
init = False
if not WriteFile(VersionFile, __version__):
print(OutputPrefix + 'Could not create version file')
init = False
print(OutputPrefix + 'init success')
return init
def BuildModuleInit() -> str:
BuildModuleInit = None
BuildModuleInit='''
#-*- coding: utf-8 -*-
"""
Todo
----
insert module comments in this space.
"""
__title__ = "''' + __title__ + '''"
__version__ = "''' + __version__ + '''"
__doc__ = "''' + __doc__ + '''"
__author__ = "''' + __author__ + '''"
__email__ = "''' + __email__ + '''"
__license__ = "''' + __license__ + '''"
__copyright__ = "''' + __copyright__ + '''"
__source__ = "''' + __source__ + '''"
'''
print(OutputPrefix + "__init__.py generated contents:\n" + BuildModuleInit)
return BuildModuleInit
def BuildDockerfile() -> str:
BuildDockerfile = None
BuildDockerfile = '''
FROM python:3.6.10-alpine3.12
LABEL \\
maintainer="''' + __author__ + ''' <http://nofusscomputing.com> + " \\
version="''' + __version__ + '''" \\
licence="''' + __license__ + '''" \\
source="''' + __source__ + '''" \\
commit="''' + GIT_COMMIT + '''"
ADD dist/ /tmp/
RUN pip --version \\
&& pip install \\
/tmp/$(ls tmp/ | grep -e ^gitlab_management.*\.whl) \\
&& gitlab-management -h
ENTRYPOINT [ "gitlab-management" ]
CMD [ "-h" ]'''
print (OutputPrefix + "dockerfile generated contents:\n" + BuildDockerfile)
return BuildDockerfile
def WriteFile(FileName:str, FileContent:str) -> bool:
WriteFile = False
with open(FileName, "w") as text_file:
text_file.write(FileContent)
if os.path.isfile(FileName):
WriteFile = True
else:
print(OutputPrefix + 'Faile to write file: ' + FileName)
return WriteFile
print(OutputPrefix + 'Starting init python script')
if init():
if not WriteFile(CurrentDirectory + 'dockerfile', BuildDockerfile()):
sys.exit(753)
if not WriteFile(ModuleDirectory + '__init__.py', BuildModuleInit()):
sys.exit(754)
else:
print(OutputPrefix + 'Init Failure')
sys.exit(755)

12
docs/contents.rst
View File

@ -3,12 +3,20 @@ Contents
==========
.. toctree::
:maxdepth: 1
:maxdepth: 3
:caption: Contents:
index
pages/pages
module/modules
includes/Contributing
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

16
docs/index.rst
View File

@ -10,19 +10,3 @@
.. toctree::
:maxdepth: 1
module/modules
includes/Contributing
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

15
docs/pages/docker.md Normal file
View File

@ -0,0 +1,15 @@
# Docker
The module has been built into a docker container that can be found on docker hub. by design the container does not need a command to be specified at runtime, as the `ENTRYPOINT` has been set to `gitlab-management`. Therefore you only need to specify the arguments for `gitlab-management`. If you fail to specify any arguments in the `docker run` command, gitlab-management help will be displayed
## Building
**To Build, execute the following:**
``` bash
docker build . --no-cache -t nofusscomputing/gitlab-management:$(cat dist/version)-alpine
```
## Running
**To run the docker image, execute to following:**
``` bash
WORKING_DIR=/Config && docker run -v $PWD:$WORKING_DIR -w $WORKING_DIR nofusscomputing/gitlab-management:$(cat dist/version)-alpine {Arguments}
```
>**Note:** substitute `{Arguments}` with the switches for gitlab-management, i.e. `-h` for help.

9
docs/pages/pages.rst Normal file
View File

@ -0,0 +1,9 @@
Help
=====
For assistance in using this module please see the sections below.
.. toctree::
:maxdepth: 2
docker

2
docs/requirements.txt
View File

@ -1,3 +1,3 @@
recommonmark
m2r
python-gitlab
python-gitlab

12
gitlab_management/__init__.py
View File

@ -1,12 +0,0 @@
#-*- coding: utf-8 -*-
"""
Todo
----
insert module comments in this space.
"""
__author__ = "No Fuss Computing"
__email__ = "helpdesk@nofusscomputing.com"
__license__ = "GNU LGPLv3"
__copyright__ = "(C) All Rights reserved"

1
gitlab_management/__main__.py
View File

@ -4,3 +4,4 @@ from gitlab_management.cli import main
if __name__ == '__main__':
main()

10
gitlab_management/base.py
View File

@ -3,14 +3,9 @@
import sys, getopt
__title__ = "gitlab-management"
__version__ = "0.1.2"
__doc__ = "https://gitlab.com/nofusscomputing/projects/python-gitlab-management"
class GitlabManagement:
"""
No Fuss Computing's Gitlab Config Management python module. (version: """ + __version__ + """)
No Fuss Computing's Gitlab Config Management python module.
"""
@ -23,8 +18,7 @@ class GitlabManagement:
GitlabSession:GitLabAPIWrapper.Gitlab = None
ModuleVersion = __version__
DesiredOutputLevel:int = None
Config:dict = None

7
gitlab_management/cli.py
View File

@ -1,17 +1,18 @@
#!/usr/bin/env python3
#-*- coding: utf-8 -*-
import sys, getopt
import sys, getopt, logging, traceback
from gitlab_management.base import GitlabManagement
help = """GitLab-Management Help (Version: """ + GitlabManagement.ModuleVersion + """)
help = """GitLab-Management Help
To run this module the following options are available. There is a short and long version of each option.
Example: python3 gitlab-management -T {GitlabAuthToken}
-H --Host (Optional) GitLab host to connect to. include http(s)://. Default: https://gitlab.com
-T --Token (Mandatory) GitLab Private token for authentication
-T --Token (Mandatory) GitLab Private token for authentication.
-l --labels Process configuration group.labels.
-v --verbose (Optional) Verbose command output.

1
requirements.txt
View File

@ -1 +1,2 @@
python-gitlab
gitpython

24
setup.py
View File

@ -3,6 +3,8 @@
import setuptools
import buildinit
with open("README.md", "r") as fh:
long_description = fh.read()
@ -10,18 +12,11 @@ def get_detail(ItemName:str):
with open("gitlab_management/__init__.py") as f:
for line in f:
if line.startswith("__" + ItemName + "__"):
return eval(line.split("=")[-1])
def get_base_detail(ItemName:str):
with open("gitlab_management/base.py") as f:
for line in f:
if line.startswith("__" + ItemName + "__"):
return eval(line.split("=")[-1])
return(str(line.split("=")[-1]).replace('"', '').replace("\n", '').replace(" ", ''))
setuptools.setup(
name=get_base_detail('title'),
version=get_base_detail('version'),
name=get_detail('title'),
version=get_detail('version'),
author=get_detail('author'),
author_email=get_detail('email'),
description="GitLab group configuration as code",
@ -32,9 +27,16 @@ setuptools.setup(
project_urls = {
"Bug Tracker": "https://gitlab.com/nofusscomputing/projects/python-gitlab-management/-/issues",
"Documentation": "https://python-gitlab-management.readthedocs.io/",
"Source Code": "https://gitlab.com/nofusscomputing/projects/python-gitlab-management"
"Source Code": get_detail('source')
},
packages=setuptools.find_packages(),
install_requires=[
'python-gitlab',
'PyYAML'
],
entry_points = {
'console_scripts': ['gitlab-management=gitlab_management.cli:main'],
},
classifiers=[
"Programming Language :: Python :: 3",
"Operating System :: OS Independent",