feat(docker): container that launches server and scanner in one

!1
2024-02-21 00:26:40 +09:30
parent add0b2390b
commit f9af64e7a1
10 changed files with 474 additions and 2 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -0,0 +1,21 @@
+.vscode/*
+artifacts/*
+cache/*
+docs/*
+extensions/*
+gitlab-ci/*
+meta/*
+playbooks/*
+plugins/*
+roles/*
+website-template/*
+.ansible-lint-ignore
+.cz.yaml
+.gitignore
+.gitlab*
+.gitmodules
+.nfc_automation.yaml
+*.md
+galaxy.yml
+LICENSE
+mkdocs.yml
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -4,6 +4,20 @@ variables:
  GIT_SYNC_URL: "https://$GITHUB_USERNAME_ROBOT:$GITHUB_TOKEN_ROBOT@github.com/NoFussComputing/docker-bind.git"
  GIT_SUBMODULE_PATHS: gitlab-ci

+  # Docs NFC
+  PAGES_ENVIRONMENT_PATH: "projects/ansible/collection/$CI_PROJECT_NAME"
+
+  # Docker Build / Publish
+  DOCKER_IMAGE_BUILD_TARGET_PLATFORMS: "linux/amd64,linux/arm64"
+  DOCKER_IMAGE_BUILD_NAME: phpipam-scan-agent
+  DOCKER_IMAGE_BUILD_REGISTRY: $CI_REGISTRY_IMAGE
+  DOCKER_IMAGE_BUILD_TAG: $CI_COMMIT_SHA
+
+  # Docker Publish
+  DOCKER_IMAGE_PUBLISH_NAME: phpipam-scan-agent
+  DOCKER_IMAGE_PUBLISH_REGISTRY: docker.io/nofusscomputing
+  DOCKER_IMAGE_PUBLISH_URL: https://hub.docker.com/r/nofusscomputing/$DOCKER_IMAGE_PUBLISH_NAME
+

 include:
  - project: nofusscomputing/projects/gitlab-ci
@ -13,3 +27,246 @@ include:
      - template/ansible-collection.gitlab-ci.yaml
      # ToDo: update gitlabCI jobs for collections workflow
      - git_push_mirror/.gitlab-ci.yml
+
+
+
+.build_docker_container:
+  stage: prepare
+  image: 
+    name: nofusscomputing/docker-buildx-qemu:dev
+    pull_policy: always
+  services:
+    - name: docker:23-dind
+      entrypoint: ["env", "-u", "DOCKER_HOST"]
+      command: ["dockerd-entrypoint.sh"]
+  variables:
+    DOCKER_HOST: tcp://docker:2375/
+    DOCKER_DRIVER: overlay2
+    DOCKER_DOCKERFILE: dockerfile
+    # See https://github.com/docker-library/docker/pull/166
+    DOCKER_TLS_CERTDIR: ""
+    # DOCKER_BUILD_ARGS: # Optional
+  before_script:
+    - git submodule foreach git submodule update --init
+    - docker info
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    - update-binfmts --display
+    - update-binfmts --enable # Important: Ensures execution of other binary formats is enabled in the kernel
+    - docker buildx create --driver=docker-container --driver-opt image=moby/buildkit:v0.11.6 --use
+    - docker buildx inspect --bootstrap
+  script: 
+    - update-binfmts --display
+    - |
+        echo "[DEBUG] building multiarch/specified arch image";
+
+        if [ ${CI_COMMIT_TAG} ]; then
+
+          export DOCKER_BUILD_ARGS="$DOCKER_BUILD_ARGS --build-arg COLLECTION_PACKAGE=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${CI_PROJECT_NAME}/${CI_COMMIT_TAG}/${ANSIBLE_GALAXY_NAMESPACE}-${ANSIBLE_GALAXY_PACKAGE_NAME}-${CI_COMMIT_TAG}.tar.gz"
+
+          echo "Trace Updated DOCKER_BUILD_ARGS[$DOCKER_BUILD_ARGS]";
+
+        fi;
+
+        docker buildx build --platform=$DOCKER_IMAGE_BUILD_TARGET_PLATFORMS . \
+          --label org.opencontainers.image.created="$(date '+%Y-%m-%d %H:%M:%S%:z')" \
+          --label org.opencontainers.image.documentation="https://nofusscomputing/$PAGES_ENVIRONMENT_PATH/" \
+          --label org.opencontainers.image.source="$CI_PROJECT_URL" \
+          --label org.opencontainers.image.revision="$CI_COMMIT_SHA" \
+        --push \
+        --build-arg COLLECTION_BRANCH=$CI_COMMIT_BRANCH --build-arg COLLECTION_COMMIT=$CI_COMMIT_SHA \
+        $DOCKER_BUILD_ARGS \
+        --build-arg CI_JOB_TOKEN=$CI_JOB_TOKEN --build-arg CI_PROJECT_ID=$CI_PROJECT_ID --build-arg CI_API_V4_URL=$CI_API_V4_URL \
+        --file $DOCKER_DOCKERFILE \
+        --tag $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
+
+        docker buildx imagetools inspect $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
+
+        # during docker multi platform build there are >=3 additional unknown images added to gitlab container registry. cleanup
+        
+        DOCKER_MULTI_ARCH_IMAGES=$(docker buildx imagetools inspect "$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG" --format "{{ range .Manifest.Manifests }}{{ if ne (print .Platform) \"&{unknown unknown  [] }\" }}$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG@{{ println .Digest }}{{end}} {{end}}");
+
+        docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
+
+        docker buildx imagetools inspect $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
+
+  # Declare rules with job
+  # rules:
+
+  #   - if: $CI_COMMIT_TAG
+  #     when: on_success
+
+  #   - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+  #     when: never
+
+  #   - if: # Occur on merge
+  #       $CI_COMMIT_BRANCH == "development"
+  #         &&
+  #       $CI_PIPELINE_SOURCE == "push"
+  #     when: always
+
+  #   - if:
+  #       $CI_COMMIT_BRANCH != "development"
+  #         &&
+  #       $CI_COMMIT_BRANCH != "master"
+  #         &&
+  #       $CI_PIPELINE_SOURCE == "push"
+  #     when: always
+
+
+  #   - when: never
+
+
+.publish-docker-hub:
+  stage: publish
+  image: docker:23-dind
+  services:
+    - docker:23-dind
+  variables:
+    GIT_STRATEGY: none
+  before_script:
+    - |
+
+      docker login $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD;
+
+      for i in ${DOCKER_IMAGE_BUILD_TARGET_PLATFORMS//,/ }
+      do
+
+        docker buildx imagetools inspect $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
+
+        DOCKER_MULTI_ARCH_IMAGES=$(docker buildx imagetools inspect "$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG" --format "{{ range .Manifest.Manifests }}$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG@{{ println .Digest }} {{end}}")
+
+        echo "[DEBUG] DOCKER_MULTI_ARCH_IMAGES=$DOCKER_MULTI_ARCH_IMAGES";
+
+      done;
+
+  script:
+    - docker login docker.io -u $NFC_DOCKERHUB_USERNAME -p $NFC_DOCKERHUB_TOKEN
+    - docker image ls
+    - |
+      DOCKER_HUB_TAG=dev
+
+      echo "[DEBUG] default: DOCKER_HUB_TAG=$DOCKER_HUB_TAG";
+
+      if [ "$CI_COMMIT_BRANCH" != "master" ]; then
+
+        DOCKER_HUB_TAG=latest
+
+        echo "[DEBUG] stable: DOCKER_HUB_TAG=$DOCKER_HUB_TAG";
+
+      elif [ "$CI_COMMIT_BRANCH" == "development" ]; then
+
+          DOCKER_HUB_TAG=dev
+
+          echo "[DEBUG] dev: DOCKER_HUB_TAG=$DOCKER_HUB_TAG";
+
+      fi;
+
+      echo "[DEBUG] DOCKER_IMAGE_PUBLISH_NAME=$DOCKER_IMAGE_PUBLISH_NAME";
+
+      echo "[DEBUG] final: DOCKER_HUB_TAG=$DOCKER_HUB_TAG";
+
+      echo "[DEBUG] DOCKER_MULTI_ARCH_IMAGES=$DOCKER_MULTI_ARCH_IMAGES";
+      
+      docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:$DOCKER_HUB_TAG;
+
+      docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:$CI_COMMIT_TAG;
+
+  after_script:
+    - docker logout docker.io
+
+  environment:
+    name: DockerHub
+    url: $DOCKER_IMAGE_PUBLISH_URL
+  rules:
+
+    - if: $CI_COMMIT_TAG
+      when: on_success
+
+    - when: never
+
+
+
+
+
+
+
+
+
+Docker Container (dev):
+  extends: .build_docker_container
+  resource_group: docker-build
+  needs:
+    - Build Collection
+  rules:
+    - if: $CI_COMMIT_TAG
+      when: never
+
+    - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+      when: never
+
+    - if:
+        $CI_COMMIT_BRANCH != "development"
+          &&
+        $CI_COMMIT_BRANCH != "master"
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      when: always
+
+    - when: never
+
+
+Docker Hub (dev):
+  extends: .publish-docker-hub
+  needs: 
+    - "Docker Container (dev)"
+  resource_group: docker-build
+  rules:
+    - if: $CI_COMMIT_TAG
+      when: never
+
+    - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+      when: never
+
+    - if:
+        $CI_COMMIT_BRANCH == "development"
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      when: always
+
+    - when: never
+
+
+Docker Container:
+  extends: .build_docker_container
+  resource_group: docker-build
+  needs:
+    - Stage Collection
+  rules:
+    - if: $CI_COMMIT_TAG
+      when: on_success
+
+    - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+      when: never
+
+    - if: # Occur on merge
+        $CI_COMMIT_BRANCH == "development"
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      when: always
+
+    - when: never
+
+
+Docker Hub:
+  extends: .publish-docker-hub
+  needs: 
+    - "Docker Container"
+    - "Gitlab Release"
+  resource_group: docker-build
+  rules:
+
+    - if: $CI_COMMIT_TAG
+      when: on_success
+
+    - when: never
--- a/.vscode/extensions.json
+++ b/.vscode/extensions.json
@ -6,6 +6,7 @@
        "streetsidesoftware.code-spell-checker",
        "streetsidesoftware.code-spell-checker-australian-english",
        "redhat.ansible",
-        "ms-azuretools.vscode-docker"
+        "ms-azuretools.vscode-docker",
+        "ms-python.python"
    ]
 }
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@ -1,5 +1,18 @@
 {
    "gitlab.aiAssistedCodeSuggestions.enabled": false,
    "gitlab.duoChat.enabled": false,
-    "cSpell.language": "en-AU,en"
+    "cSpell.language": "en-AU,en",
+    "yaml.schemas": {
+        "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/tasks": [
+            "file://${workspaceFolder}/playbooks/tasks/**",
+            "file://${workspaceFolder}/playbooks/tasks/**/**",
+        ],
+        "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/inventory.json": [
+            "file://${workspaceFolder}/includes/root/hosts.yaml"
+        ],
+    },
+    "files.associations": {
+        "includes/etc/supervisor/supervisord.conf": "ini",
+        "includes/etc/supervisor/conf.d/*.conf": "ini",
+    }
 }
--- a/126
+++ b/126
@ -0,0 +1,126 @@
+
+FROM python:3.11-alpine3.19 as build
+
+
+RUN apk update; \
+  apk add gcc
+
+
+RUN pip install --upgrade \
+    setuptools \
+    wheel
+
+
+RUN apk add openjdk21-jdk;
+
+
+RUN apk add \ 
+    alpine-sdk \
+    libffi-dev \
+    maven;
+
+
+ENV JAVA_HOME /usr/lib/jvm/java-21-openjdk
+
+
+COPY requirements.txt /tmp/requirements.txt
+
+
+RUN mkdir -p /tmp/python_modules; \
+  cd /tmp/python_modules; \
+  pip download --dest . --check-build-dependencies \
+    supervisor==4.2.5 \
+    -r /tmp/requirements.txt
+
+RUN mkdir -p /tmp/python_builds;
+
+
+RUN cd /tmp/python_modules; \
+  mkdir -p /tmp/python_builds; \
+  echo "[DEBUG] PATH=$PATH"; \
+  pip wheel --wheel-dir /tmp/python_builds --find-links . *.whl; \
+  pip wheel --wheel-dir /tmp/python_builds --find-links . *.tar.gz;
+
+
+RUN echo $(date)\
+  cd /tmp; \
+  ls -laR /tmp
+
+
+
+
+FROM python:3.11-alpine3.19
+
+LABEL \
+  # org.opencontainers.image.authors="{contributor url}" \
+  # org.opencontainers.image.url="{dockerhub url}" \
+  # org.opencontainers.image.documentation="{docs url}" \
+  # org.opencontainers.image.source="{repo url}" \
+  # org.opencontainers.image.revision="{git commit sha at time of build}" \
+  org.opencontainers.image.title="No Fuss Computings phpIPAM Scan Agent" \
+  org.opencontainers.image.description="A phpIPAM Scan agent for local and remote networks" \
+  org.opencontainers.image.vendor="No Fuss Computing"
+  # org.opencontainers.image.version="{git tag}"
+
+RUN apk --no-cache update; \
+  apk add \
+    openjdk21-jdk \
+    git \
+    dcron \
+    nmap;
+
+
+ENV JAVA_HOME /usr/lib/jvm/java-21-openjdk
+
+
+COPY --from=build /tmp/python_builds /tmp/python_builds
+
+
+RUN chmod 644 -R /etc/cron.d; \
+  pip install /tmp/python_builds/*; \
+  rm -R /tmp/python_builds
+
+
+ARG COLLECTION_COMMIT=none
+
+ARG COLLECTION_BRANCH=development
+
+ARG COLLECTION_PACKAGE=dev
+
+
+COPY includes/ /
+
+RUN mkdir -p /tmp/collection; \
+  if [ "$COLLECTION_PACKAGE" != "dev" ]; then \
+    echo "specified"; \
+    ansible-galaxy collection install --force-with-deps --pre \
+      $COLLECTION; \
+  elif [ "$COLLECTION_PACKAGE" == "dev" ]; then \
+    git clone \
+      --depth=1 \
+      -b $COLLECTION_BRANCH \ 
+      https://gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent.git \
+      /tmp/collection; \
+    if [ "${COLLECTION_COMMIT}" != "none" ]; then git switch $COLLECTION_COMMIT; fi; \
+    ansible-galaxy collection install --force-with-deps --pre \
+    /tmp/collection/.; \
+    rm -Rf /tmp/collection; \
+  fi; \
+  chmod +x /etc/cron.d/*;
+
+
+WORKDIR /root
+
+
+HEALTHCHECK --interval=10s --timeout=10s --start-period=5s --retries=3 CMD \
+  supervisorctl status || exit 1
+
+
+ENV HTTP_PORT 5000
+
+ENV ANSIBLE_FORCE_COLOR 'true'
+
+ENV ANSIBLE_LOG_PATH /var/log/ansible.log
+
+
+CMD [ "/usr/local/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf" ]
--- a/includes/etc/cron.d/scanner
+++ b/includes/etc/cron.d/scanner
@ -0,0 +1,5 @@
+#
+# Default Scanner Scheduled Job
+#
+
+*/1 * * * * ansible-playbook nofusscomputing.phpipam_scan_agent.agent -v
--- a/includes/etc/supervisor/conf.d/cron.conf
+++ b/includes/etc/supervisor/conf.d/cron.conf
@ -0,0 +1,8 @@
+[program:cron]
+startsecs=0
+stopwaitsecs=55
+autostart=true
+autorestart=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+command=/usr/sbin/crond -f -L /var/log/crond.log -l info
--- a/includes/etc/supervisor/conf.d/rulebook.conf
+++ b/includes/etc/supervisor/conf.d/rulebook.conf
@ -0,0 +1,8 @@
+[program:rulebook]
+startsecs=0
+stopwaitsecs=55
+command=ansible-rulebook -r nofusscomputing.phpipam_scan_agent.agent_receive --env-vars "HTTP_PORT" -i /root/hosts.yaml -v
+autorestart=true
+autostart=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
--- a/includes/etc/supervisor/supervisord.conf
+++ b/includes/etc/supervisor/supervisord.conf
@ -0,0 +1,27 @@
+
+[unix_http_server]
+file=/var/run/supervisor.sock
+chmod=0700
+
+;[inet_http_server]
+;port = :9001
+; username = user
+; password = 123
+
+[supervisord]
+logfile=/var/log/supervisord.log
+pidfile=/var/run/supervisord.pid
+nodaemon = true
+user=root
+
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+
+[supervisorctl]
+serverurl=unix:///var/run/supervisor.sock
+
+
+[include]
+files = /etc/supervisor/conf.d/*.conf
--- a/includes/root/hosts.yaml
+++ b/includes/root/hosts.yaml
@ -0,0 +1,6 @@
+all:
+  hosts:
+    localhost:
+      vars:
+        ansible_connection: local
+