From f9af64e7a1bcc4a37d18d857b0772e2bbe284d0b Mon Sep 17 00:00:00 2001 From: Jon Date: Wed, 21 Feb 2024 00:26:40 +0930 Subject: [PATCH] feat(docker): container that launches server and scanner in one !1 --- .dockerignore | 21 ++ .gitlab-ci.yml | 257 +++++++++++++++++++ .vscode/extensions.json | 3 +- .vscode/settings.json | 15 +- dockerfile | 126 +++++++++ includes/etc/cron.d/scanner | 5 + includes/etc/supervisor/conf.d/cron.conf | 8 + includes/etc/supervisor/conf.d/rulebook.conf | 8 + includes/etc/supervisor/supervisord.conf | 27 ++ includes/root/hosts.yaml | 6 + 10 files changed, 474 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 dockerfile create mode 100644 includes/etc/cron.d/scanner create mode 100644 includes/etc/supervisor/conf.d/cron.conf create mode 100644 includes/etc/supervisor/conf.d/rulebook.conf create mode 100644 includes/etc/supervisor/supervisord.conf create mode 100644 includes/root/hosts.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e1cf0ed --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +.vscode/* +artifacts/* +cache/* +docs/* +extensions/* +gitlab-ci/* +meta/* +playbooks/* +plugins/* +roles/* +website-template/* +.ansible-lint-ignore +.cz.yaml +.gitignore +.gitlab* +.gitmodules +.nfc_automation.yaml +*.md +galaxy.yml +LICENSE +mkdocs.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e87ed02..685fd5f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,20 @@ variables: GIT_SYNC_URL: "https://$GITHUB_USERNAME_ROBOT:$GITHUB_TOKEN_ROBOT@github.com/NoFussComputing/docker-bind.git" GIT_SUBMODULE_PATHS: gitlab-ci + # Docs NFC + PAGES_ENVIRONMENT_PATH: "projects/ansible/collection/$CI_PROJECT_NAME" + + # Docker Build / Publish + DOCKER_IMAGE_BUILD_TARGET_PLATFORMS: "linux/amd64,linux/arm64" + DOCKER_IMAGE_BUILD_NAME: phpipam-scan-agent + DOCKER_IMAGE_BUILD_REGISTRY: $CI_REGISTRY_IMAGE + DOCKER_IMAGE_BUILD_TAG: $CI_COMMIT_SHA + + # Docker Publish + DOCKER_IMAGE_PUBLISH_NAME: phpipam-scan-agent + DOCKER_IMAGE_PUBLISH_REGISTRY: docker.io/nofusscomputing + DOCKER_IMAGE_PUBLISH_URL: https://hub.docker.com/r/nofusscomputing/$DOCKER_IMAGE_PUBLISH_NAME + include: - project: nofusscomputing/projects/gitlab-ci @@ -13,3 +27,246 @@ include: - template/ansible-collection.gitlab-ci.yaml # ToDo: update gitlabCI jobs for collections workflow - git_push_mirror/.gitlab-ci.yml + + + +.build_docker_container: + stage: prepare + image: + name: nofusscomputing/docker-buildx-qemu:dev + pull_policy: always + services: + - name: docker:23-dind + entrypoint: ["env", "-u", "DOCKER_HOST"] + command: ["dockerd-entrypoint.sh"] + variables: + DOCKER_HOST: tcp://docker:2375/ + DOCKER_DRIVER: overlay2 + DOCKER_DOCKERFILE: dockerfile + # See https://github.com/docker-library/docker/pull/166 + DOCKER_TLS_CERTDIR: "" + # DOCKER_BUILD_ARGS: # Optional + before_script: + - git submodule foreach git submodule update --init + - docker info + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - update-binfmts --display + - update-binfmts --enable # Important: Ensures execution of other binary formats is enabled in the kernel + - docker buildx create --driver=docker-container --driver-opt image=moby/buildkit:v0.11.6 --use + - docker buildx inspect --bootstrap + script: + - update-binfmts --display + - | + echo "[DEBUG] building multiarch/specified arch image"; + + if [ ${CI_COMMIT_TAG} ]; then + + export DOCKER_BUILD_ARGS="$DOCKER_BUILD_ARGS --build-arg COLLECTION_PACKAGE=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${CI_PROJECT_NAME}/${CI_COMMIT_TAG}/${ANSIBLE_GALAXY_NAMESPACE}-${ANSIBLE_GALAXY_PACKAGE_NAME}-${CI_COMMIT_TAG}.tar.gz" + + echo "Trace Updated DOCKER_BUILD_ARGS[$DOCKER_BUILD_ARGS]"; + + fi; + + docker buildx build --platform=$DOCKER_IMAGE_BUILD_TARGET_PLATFORMS . \ + --label org.opencontainers.image.created="$(date '+%Y-%m-%d %H:%M:%S%:z')" \ + --label org.opencontainers.image.documentation="https://nofusscomputing/$PAGES_ENVIRONMENT_PATH/" \ + --label org.opencontainers.image.source="$CI_PROJECT_URL" \ + --label org.opencontainers.image.revision="$CI_COMMIT_SHA" \ + --push \ + --build-arg COLLECTION_BRANCH=$CI_COMMIT_BRANCH --build-arg COLLECTION_COMMIT=$CI_COMMIT_SHA \ + $DOCKER_BUILD_ARGS \ + --build-arg CI_JOB_TOKEN=$CI_JOB_TOKEN --build-arg CI_PROJECT_ID=$CI_PROJECT_ID --build-arg CI_API_V4_URL=$CI_API_V4_URL \ + --file $DOCKER_DOCKERFILE \ + --tag $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG; + + docker buildx imagetools inspect $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG; + + # during docker multi platform build there are >=3 additional unknown images added to gitlab container registry. cleanup + + DOCKER_MULTI_ARCH_IMAGES=$(docker buildx imagetools inspect "$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG" --format "{{ range .Manifest.Manifests }}{{ if ne (print .Platform) \"&{unknown unknown [] }\" }}$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG@{{ println .Digest }}{{end}} {{end}}"); + + docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG; + + docker buildx imagetools inspect $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG; + + # Declare rules with job + # rules: + + # - if: $CI_COMMIT_TAG + # when: on_success + + # - if: "$CI_COMMIT_AUTHOR =='nfc_bot '" + # when: never + + # - if: # Occur on merge + # $CI_COMMIT_BRANCH == "development" + # && + # $CI_PIPELINE_SOURCE == "push" + # when: always + + # - if: + # $CI_COMMIT_BRANCH != "development" + # && + # $CI_COMMIT_BRANCH != "master" + # && + # $CI_PIPELINE_SOURCE == "push" + # when: always + + + # - when: never + + +.publish-docker-hub: + stage: publish + image: docker:23-dind + services: + - docker:23-dind + variables: + GIT_STRATEGY: none + before_script: + - | + + docker login $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD; + + for i in ${DOCKER_IMAGE_BUILD_TARGET_PLATFORMS//,/ } + do + + docker buildx imagetools inspect $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG; + + DOCKER_MULTI_ARCH_IMAGES=$(docker buildx imagetools inspect "$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG" --format "{{ range .Manifest.Manifests }}$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG@{{ println .Digest }} {{end}}") + + echo "[DEBUG] DOCKER_MULTI_ARCH_IMAGES=$DOCKER_MULTI_ARCH_IMAGES"; + + done; + + script: + - docker login docker.io -u $NFC_DOCKERHUB_USERNAME -p $NFC_DOCKERHUB_TOKEN + - docker image ls + - | + DOCKER_HUB_TAG=dev + + echo "[DEBUG] default: DOCKER_HUB_TAG=$DOCKER_HUB_TAG"; + + if [ "$CI_COMMIT_BRANCH" != "master" ]; then + + DOCKER_HUB_TAG=latest + + echo "[DEBUG] stable: DOCKER_HUB_TAG=$DOCKER_HUB_TAG"; + + elif [ "$CI_COMMIT_BRANCH" == "development" ]; then + + DOCKER_HUB_TAG=dev + + echo "[DEBUG] dev: DOCKER_HUB_TAG=$DOCKER_HUB_TAG"; + + fi; + + echo "[DEBUG] DOCKER_IMAGE_PUBLISH_NAME=$DOCKER_IMAGE_PUBLISH_NAME"; + + echo "[DEBUG] final: DOCKER_HUB_TAG=$DOCKER_HUB_TAG"; + + echo "[DEBUG] DOCKER_MULTI_ARCH_IMAGES=$DOCKER_MULTI_ARCH_IMAGES"; + + docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:$DOCKER_HUB_TAG; + + docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:$CI_COMMIT_TAG; + + after_script: + - docker logout docker.io + + environment: + name: DockerHub + url: $DOCKER_IMAGE_PUBLISH_URL + rules: + + - if: $CI_COMMIT_TAG + when: on_success + + - when: never + + + + + + + + + +Docker Container (dev): + extends: .build_docker_container + resource_group: docker-build + needs: + - Build Collection + rules: + - if: $CI_COMMIT_TAG + when: never + + - if: "$CI_COMMIT_AUTHOR =='nfc_bot '" + when: never + + - if: + $CI_COMMIT_BRANCH != "development" + && + $CI_COMMIT_BRANCH != "master" + && + $CI_PIPELINE_SOURCE == "push" + when: always + + - when: never + + +Docker Hub (dev): + extends: .publish-docker-hub + needs: + - "Docker Container (dev)" + resource_group: docker-build + rules: + - if: $CI_COMMIT_TAG + when: never + + - if: "$CI_COMMIT_AUTHOR =='nfc_bot '" + when: never + + - if: + $CI_COMMIT_BRANCH == "development" + && + $CI_PIPELINE_SOURCE == "push" + when: always + + - when: never + + +Docker Container: + extends: .build_docker_container + resource_group: docker-build + needs: + - Stage Collection + rules: + - if: $CI_COMMIT_TAG + when: on_success + + - if: "$CI_COMMIT_AUTHOR =='nfc_bot '" + when: never + + - if: # Occur on merge + $CI_COMMIT_BRANCH == "development" + && + $CI_PIPELINE_SOURCE == "push" + when: always + + - when: never + + +Docker Hub: + extends: .publish-docker-hub + needs: + - "Docker Container" + - "Gitlab Release" + resource_group: docker-build + rules: + + - if: $CI_COMMIT_TAG + when: on_success + + - when: never \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 6097c9d..b60062e 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -6,6 +6,7 @@ "streetsidesoftware.code-spell-checker", "streetsidesoftware.code-spell-checker-australian-english", "redhat.ansible", - "ms-azuretools.vscode-docker" + "ms-azuretools.vscode-docker", + "ms-python.python" ] } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index b83abee..9183fba 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,18 @@ { "gitlab.aiAssistedCodeSuggestions.enabled": false, "gitlab.duoChat.enabled": false, - "cSpell.language": "en-AU,en" + "cSpell.language": "en-AU,en", + "yaml.schemas": { + "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/tasks": [ + "file://${workspaceFolder}/playbooks/tasks/**", + "file://${workspaceFolder}/playbooks/tasks/**/**", + ], + "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/inventory.json": [ + "file://${workspaceFolder}/includes/root/hosts.yaml" + ], + }, + "files.associations": { + "includes/etc/supervisor/supervisord.conf": "ini", + "includes/etc/supervisor/conf.d/*.conf": "ini", + } } \ No newline at end of file diff --git a/dockerfile b/dockerfile new file mode 100644 index 0000000..db63184 --- /dev/null +++ b/dockerfile @@ -0,0 +1,126 @@ + +FROM python:3.11-alpine3.19 as build + + +RUN apk update; \ + apk add gcc + + +RUN pip install --upgrade \ + setuptools \ + wheel + + +RUN apk add openjdk21-jdk; + + +RUN apk add \ + alpine-sdk \ + libffi-dev \ + maven; + + +ENV JAVA_HOME /usr/lib/jvm/java-21-openjdk + + +COPY requirements.txt /tmp/requirements.txt + + +RUN mkdir -p /tmp/python_modules; \ + cd /tmp/python_modules; \ + pip download --dest . --check-build-dependencies \ + supervisor==4.2.5 \ + -r /tmp/requirements.txt + +RUN mkdir -p /tmp/python_builds; + + +RUN cd /tmp/python_modules; \ + mkdir -p /tmp/python_builds; \ + echo "[DEBUG] PATH=$PATH"; \ + pip wheel --wheel-dir /tmp/python_builds --find-links . *.whl; \ + pip wheel --wheel-dir /tmp/python_builds --find-links . *.tar.gz; + + +RUN echo $(date)\ + cd /tmp; \ + ls -laR /tmp + + + + +FROM python:3.11-alpine3.19 + +LABEL \ + # org.opencontainers.image.authors="{contributor url}" \ + # org.opencontainers.image.url="{dockerhub url}" \ + # org.opencontainers.image.documentation="{docs url}" \ + # org.opencontainers.image.source="{repo url}" \ + # org.opencontainers.image.revision="{git commit sha at time of build}" \ + org.opencontainers.image.title="No Fuss Computings phpIPAM Scan Agent" \ + org.opencontainers.image.description="A phpIPAM Scan agent for local and remote networks" \ + org.opencontainers.image.vendor="No Fuss Computing" + # org.opencontainers.image.version="{git tag}" + +RUN apk --no-cache update; \ + apk add \ + openjdk21-jdk \ + git \ + dcron \ + nmap; + + +ENV JAVA_HOME /usr/lib/jvm/java-21-openjdk + + +COPY --from=build /tmp/python_builds /tmp/python_builds + + +RUN chmod 644 -R /etc/cron.d; \ + pip install /tmp/python_builds/*; \ + rm -R /tmp/python_builds + + +ARG COLLECTION_COMMIT=none + +ARG COLLECTION_BRANCH=development + +ARG COLLECTION_PACKAGE=dev + + +COPY includes/ / + +RUN mkdir -p /tmp/collection; \ + if [ "$COLLECTION_PACKAGE" != "dev" ]; then \ + echo "specified"; \ + ansible-galaxy collection install --force-with-deps --pre \ + $COLLECTION; \ + elif [ "$COLLECTION_PACKAGE" == "dev" ]; then \ + git clone \ + --depth=1 \ + -b $COLLECTION_BRANCH \ + https://gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent.git \ + /tmp/collection; \ + if [ "${COLLECTION_COMMIT}" != "none" ]; then git switch $COLLECTION_COMMIT; fi; \ + ansible-galaxy collection install --force-with-deps --pre \ + /tmp/collection/.; \ + rm -Rf /tmp/collection; \ + fi; \ + chmod +x /etc/cron.d/*; + + +WORKDIR /root + + +HEALTHCHECK --interval=10s --timeout=10s --start-period=5s --retries=3 CMD \ + supervisorctl status || exit 1 + + +ENV HTTP_PORT 5000 + +ENV ANSIBLE_FORCE_COLOR 'true' + +ENV ANSIBLE_LOG_PATH /var/log/ansible.log + + +CMD [ "/usr/local/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf" ] diff --git a/includes/etc/cron.d/scanner b/includes/etc/cron.d/scanner new file mode 100644 index 0000000..d191fdb --- /dev/null +++ b/includes/etc/cron.d/scanner @@ -0,0 +1,5 @@ +# +# Default Scanner Scheduled Job +# + +*/1 * * * * ansible-playbook nofusscomputing.phpipam_scan_agent.agent -v diff --git a/includes/etc/supervisor/conf.d/cron.conf b/includes/etc/supervisor/conf.d/cron.conf new file mode 100644 index 0000000..d677dd1 --- /dev/null +++ b/includes/etc/supervisor/conf.d/cron.conf @@ -0,0 +1,8 @@ +[program:cron] +startsecs=0 +stopwaitsecs=55 +autostart=true +autorestart=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +command=/usr/sbin/crond -f -L /var/log/crond.log -l info diff --git a/includes/etc/supervisor/conf.d/rulebook.conf b/includes/etc/supervisor/conf.d/rulebook.conf new file mode 100644 index 0000000..c63aed3 --- /dev/null +++ b/includes/etc/supervisor/conf.d/rulebook.conf @@ -0,0 +1,8 @@ +[program:rulebook] +startsecs=0 +stopwaitsecs=55 +command=ansible-rulebook -r nofusscomputing.phpipam_scan_agent.agent_receive --env-vars "HTTP_PORT" -i /root/hosts.yaml -v +autorestart=true +autostart=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 diff --git a/includes/etc/supervisor/supervisord.conf b/includes/etc/supervisor/supervisord.conf new file mode 100644 index 0000000..a19d9be --- /dev/null +++ b/includes/etc/supervisor/supervisord.conf @@ -0,0 +1,27 @@ + +[unix_http_server] +file=/var/run/supervisor.sock +chmod=0700 + +;[inet_http_server] +;port = :9001 +; username = user +; password = 123 + +[supervisord] +logfile=/var/log/supervisord.log +pidfile=/var/run/supervisord.pid +nodaemon = true +user=root + + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock + + +[include] +files = /etc/supervisor/conf.d/*.conf diff --git a/includes/root/hosts.yaml b/includes/root/hosts.yaml new file mode 100644 index 0000000..c6b01de --- /dev/null +++ b/includes/root/hosts.yaml @@ -0,0 +1,6 @@ +all: + hosts: + localhost: + vars: + ansible_connection: local +