From 26df54dfa2ff2ee416cb824252900d7335533724 Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 14 Jan 2024 23:45:59 +0930 Subject: [PATCH 01/16] fix(docker_cache): build image must not be slim !184 --- dockerfile-build_cache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile-build_cache b/dockerfile-build_cache index e35d9a6..b4e0295 100644 --- a/dockerfile-build_cache +++ b/dockerfile-build_cache @@ -1,4 +1,4 @@ -FROM --platform=$TARGETPLATFORM python:3.11-slim-bookworm as wheelbuild +FROM --platform=$TARGETPLATFORM python:3.11-bookworm as wheelbuild ARG CI_JOB_TOKEN ARG CI_API_V4_URL From 6b78eca4563376d2346e3cd75e0887e8cb6bc15a Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 14 Jan 2024 23:47:13 +0930 Subject: [PATCH 02/16] fix(docker_cache): build image binaries to match deployed binaries these install dependencies required to build the python modules !184 --- dockerfile-build_cache | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dockerfile-build_cache b/dockerfile-build_cache index b4e0295..09c12fa 100644 --- a/dockerfile-build_cache +++ b/dockerfile-build_cache @@ -36,7 +36,10 @@ RUN export DEBIAN_FRONTEND=noninteractive \ && dpkg --purge --force-all libc-bin \ && apt-get install -y \ openssh-client \ - git + git \ + postgresql-common \ + postgresql-client \ + mariadb-client RUN export DEBIAN_FRONTEND=noninteractive \ From 4fcb5ee5ea75840768fe81009d6bf761ddb0d0b6 Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 14 Jan 2024 23:48:13 +0930 Subject: [PATCH 03/16] feat(docker): use python slim for base image !184 #23 --- dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile b/dockerfile index 56ab4f6..e55fbb9 100644 --- a/dockerfile +++ b/dockerfile @@ -2,7 +2,7 @@ FROM --platform=$TARGETPLATFORM quay.io/ansible/receptor:devel as receptor -FROM --platform=$TARGETPLATFORM python:3.11-bullseye +FROM --platform=$TARGETPLATFORM python:3.11-slim-bullseye # Ansible chucks a wobbler without. see: https://github.com/ansible/ansible/issues/78283 ENV LC_ALL en_US.UTF-8 From a39be61aac7fce8404f4c60cc7752ec9ca8af0fe Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 14 Jan 2024 23:56:06 +0930 Subject: [PATCH 04/16] feat(collections): update all modules to current HEAD versions available !184 --- dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dockerfile b/dockerfile index e55fbb9..e6acb26 100644 --- a/dockerfile +++ b/dockerfile @@ -83,15 +83,15 @@ RUN pip install --index-url https://gitlab.com/api/v4/projects/45741845/packages RUN ansible-galaxy collection install \ - awx.awx==23.5.0 \ + awx.awx==23.6.0 \ # ansible.posix.authorized_key for SSH ansible.posix==1.5.4 \ ansible.utils==3.0.0 \ - community.dns==2.6.4 \ + community.dns==2.7.0 \ # docker managment - community.docker==3.4.11 \ + community.docker==3.5.0 \ # community.general.gitlab_* - community.general==8.1.0 \ + community.general==8.2.0 \ community.mysql==3.8.0 \ - community.postgresql==3.2.0 \ + community.postgresql==3.3.0 \ kubernetes.core==3.0.0 From 2b3693e957c6e3578ed1ade75574b260638dea4c Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 00:04:24 +0930 Subject: [PATCH 05/16] fix(docker_cache): migrated ldconfig fixes from ee container file required to fix ldconfig missing errors !184 --- dockerfile-build_cache | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dockerfile-build_cache b/dockerfile-build_cache index 09c12fa..a453780 100644 --- a/dockerfile-build_cache +++ b/dockerfile-build_cache @@ -12,6 +12,18 @@ RUN export DEBIAN_FRONTEND=noninteractive \ && dpkg-reconfigure debconf -f noninteractive RUN apt update \ + # SoF fixing dpkg ldconfig not found error + && cd /tmp \ + && apt-get download \ + libc-bin \ + && dpkg --extract ./libc-bin_*.deb /tmp/deb \ + && cp /tmp/deb/sbin/ldconfig /sbin/ \ + && rm -Rf /tmp/deb \ + && rm libc-bin_*.deb \ + && apt-get install --reinstall \ + libc-bin \ + # EoF fixing dpkg ldconfig not found error + # Set Locale to en_US as ansible requires a locale for it to function without chucking a tantrum!! && apt install --reinstall -yq \ git \ curl \ From cefc16d145c931aeacd94e7a89f36a184074f50b Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 00:05:36 +0930 Subject: [PATCH 06/16] ci(docker_cache): start straight away as no deps for building !184 --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 70ccb31..eec941d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,6 +27,7 @@ Build/Cache Python Packages: - name: docker:23-dind entrypoint: ["env", "-u", "DOCKER_HOST"] command: ["dockerd-entrypoint.sh"] + needs: [] variables: DOCKER_HOST: tcp://docker:2375/ DOCKER_DRIVER: overlay2 From c5f9ae33a556e9ac5391505bf3d2cb184f7b69f7 Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 00:15:19 +0930 Subject: [PATCH 07/16] ci(docker_cache): require compilation tools !184 #23 --- dockerfile-build_cache | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockerfile-build_cache b/dockerfile-build_cache index a453780..ac76162 100644 --- a/dockerfile-build_cache +++ b/dockerfile-build_cache @@ -51,7 +51,8 @@ RUN export DEBIAN_FRONTEND=noninteractive \ git \ postgresql-common \ postgresql-client \ - mariadb-client + mariadb-client \ + build-essential RUN export DEBIAN_FRONTEND=noninteractive \ From 9730657d1ec8bfdf1d780f88ae7513f0211540fb Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 00:17:08 +0930 Subject: [PATCH 08/16] feat(python): updated awxkit, python_gitlab and dns to current head versions !184 --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index cc27f24..cb1a207 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ ansible-core==2.16.1 ansible-lint==6.15.0 ansible-runner==2.3.4 -awxkit==23.5.1 +awxkit==23.6.0 # Required for community.general.gitlab_* -python_gitlab==3.14.0 +python_gitlab==4.3.0 # community.postgresql psycopg2==2.9.9 # community.mysql PyMySQL==1.1.0 -dnspython +dnspython==2.4.2 -xmltodict==0.13.0 \ No newline at end of file +xmltodict==0.13.0 From 0ae0fadfd220bd71d86c7096b114b95b7e04216d Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 00:36:49 +0930 Subject: [PATCH 09/16] refactor(docker_cache): cleanup commands and place items in alphanumerical order !184 --- .gitlab-ci.yml | 3 +- dockerfile-build_cache | 115 +++++++++++++++++++++++------------------ 2 files changed, 67 insertions(+), 51 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eec941d..8966c6c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,8 @@ Build/Cache Python Packages: --platform=$DOCKER_IMAGE_BUILD_TARGET_PLATFORMS . \ --build-arg CI_JOB_TOKEN=$CI_JOB_TOKEN \ --build-arg CI_API_V4_URL=$CI_API_V4_URL \ - --build-arg CI_PROJECT_ID=$CI_PROJECT_ID + --build-arg CI_PROJECT_ID=$CI_PROJECT_ID \ + --progress plain rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: never diff --git a/dockerfile-build_cache b/dockerfile-build_cache index ac76162..107a6bd 100644 --- a/dockerfile-build_cache +++ b/dockerfile-build_cache @@ -6,78 +6,90 @@ ARG CI_PROJECT_ID ENV LC_ALL en_US.UTF-8 + ENV PATH /usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.cargo/bin::~/.cargo/bin -RUN export DEBIAN_FRONTEND=noninteractive \ - && dpkg-reconfigure debconf -f noninteractive +ENV DEBIAN_FRONTEND noninteractive -RUN apt update \ - # SoF fixing dpkg ldconfig not found error - && cd /tmp \ + +RUN dpkg-reconfigure debconf -f noninteractive + + +RUN apt update + +# SoF fixing dpkg ldconfig not found error +# Download Binary +RUN cd /tmp \ && apt-get download \ - libc-bin \ - && dpkg --extract ./libc-bin_*.deb /tmp/deb \ - && cp /tmp/deb/sbin/ldconfig /sbin/ \ - && rm -Rf /tmp/deb \ - && rm libc-bin_*.deb \ - && apt-get install --reinstall \ - libc-bin \ - # EoF fixing dpkg ldconfig not found error - # Set Locale to en_US as ansible requires a locale for it to function without chucking a tantrum!! - && apt install --reinstall -yq \ - git \ + libc-bin + +RUN ls -la + +# Extract Binary +RUN cd /tmp; \ + dpkg --extract $(ls | grep libc-bin_ | grep -a '.deb') /tmp/deb; ls -laR + +# # Extracted? +# RUN ls -laR + +# Copy to home location +RUN cp /tmp/deb/sbin/ldconfig /sbin/ + +# Confirm present +RUN ls -l /sbin/ldconfig + +# Confirm can be found +RUN which ldconfig + +# Cleanup +# RUN rm -Rf /tmp/deb \ +# && rm libc-bin_*.deb + +# Should re-install without issue +RUN apt-get install --reinstall \ + libc-bin +# EoF fixing dpkg ldconfig not found error +# Set Locale to en_US as ansible requires a locale for it to function without chucking a tantrum!! + + +RUN apt install -yq \ + build-essential \ + cargo \ curl \ - rustc cargo \ - libpq-dev || true - - -RUN mkdir -p /tmp/python_modules /tmp/python_builds + git \ + libffi-dev \ + libpq-dev \ + libssl-dev \ + mariadb-client \ + openssh-client \ + pkg-config \ + postgresql-client \ + postgresql-common \ + python3-dev \ + rustc RUN rustc --version \ && cargo --version -RUN export DEBIAN_FRONTEND=noninteractive \ - && apt update \ - && apt --fix-broken install \ - && apt install -y libc-bin locales-all \ - && apt update \ - && apt install --reinstall -yq \ - openssh-client \ - git || true \ - && dpkg --purge --force-all libc-bin \ - && apt-get install -y \ - openssh-client \ - git \ - postgresql-common \ - postgresql-client \ - mariadb-client \ - build-essential - - -RUN export DEBIAN_FRONTEND=noninteractive \ - && apt update \ - && apt install --no-install-recommends -yq libc-bin locales-all \ - && apt-get install --reinstall -yq \ - build-essential \ - libssl-dev \ - libffi-dev \ - python3-dev \ - cargo \ - pkg-config \ - libpq-dev || true RUN pip install --upgrade pip + RUN pip install --upgrade \ setuptools \ wheel \ setuptools-rust \ twine + COPY requirements.txt /tmp/requirements.txt + +RUN mkdir -p /tmp/python_modules /tmp/python_builds + + RUN cd /tmp/python_modules \ && pip download --dest . --check-build-dependencies \ -r /tmp/requirements.txt @@ -88,5 +100,8 @@ RUN cd /tmp/python_modules \ && echo "[DEBUG] PATH=$PATH" \ && pip wheel --wheel-dir /tmp/python_builds --find-links . *.whl +RUN cd /tmp; \ + ls -laR + RUN TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --skip-existing --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi /tmp/python_builds/* From d48901bab9f7a48f56b774245c399f1b079219e7 Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 01:36:21 +0930 Subject: [PATCH 10/16] fix(docker_cache): ensure .tar.gz are built to wheels !184 --- dockerfile-build_cache | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockerfile-build_cache b/dockerfile-build_cache index 107a6bd..3cceb30 100644 --- a/dockerfile-build_cache +++ b/dockerfile-build_cache @@ -98,7 +98,8 @@ RUN cd /tmp/python_modules \ RUN cd /tmp/python_modules \ # && export PATH=$PATH:~/.cargo/bin \ && echo "[DEBUG] PATH=$PATH" \ - && pip wheel --wheel-dir /tmp/python_builds --find-links . *.whl + && pip wheel --wheel-dir /tmp/python_builds --find-links . *.whl; \ + pip wheel --wheel-dir /tmp/python_builds --find-links . *.tar.gz; RUN cd /tmp; \ ls -laR From e4feaaa69dd6c1dd0ca2f7a8a827ca032d2b706a Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 01:36:39 +0930 Subject: [PATCH 11/16] fix: depreciated warning "ANSIBLE_COLLECTIONS_PATHS option, does not fit var naming standard, use the singular form ANSIBLE_COLLECTIONS_PATH instead" !184 --- dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile b/dockerfile index e6acb26..adcc631 100644 --- a/dockerfile +++ b/dockerfile @@ -11,7 +11,7 @@ ENV DEBIAN_FRONTEND noninteractive ENV ANSIBLE_PLAYBOOK_DIR=/etc/ansible/playbooks -ENV ANSIBLE_COLLECTIONS_PATHS=/etc/ansible/collections +ENV ANSIBLE_COLLECTIONS_PATH=/etc/ansible/collections COPY includes / From b037996a4dc9631f2395f5b14390415f943139ac Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 01:37:13 +0930 Subject: [PATCH 12/16] refactor(docker): get file name with ls and grep !184 --- dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfile b/dockerfile index adcc631..9110abb 100644 --- a/dockerfile +++ b/dockerfile @@ -36,10 +36,10 @@ RUN apt update \ && cd /tmp \ && apt-get download \ libc-bin \ - && dpkg --extract ./libc-bin_*.deb /tmp/deb \ + && dpkg --extract $(ls | grep libc-bin_ | grep -a '.deb') /tmp/deb \ && cp /tmp/deb/sbin/ldconfig /sbin/ \ && rm -Rf /tmp/deb \ - && rm libc-bin_*.deb \ + && rm $(ls | grep libc-bin_ | grep -a '.deb') \ && apt-get install --reinstall \ libc-bin \ # EoF fixing dpkg ldconfig not found error From 4c69d8e2e56cb63d8890a2a70d53bac782ce3df3 Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 01:42:14 +0930 Subject: [PATCH 13/16] feat(docker): update base image from buster to bookworm !184 #23 --- dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile b/dockerfile index 9110abb..9ee6b9a 100644 --- a/dockerfile +++ b/dockerfile @@ -2,7 +2,7 @@ FROM --platform=$TARGETPLATFORM quay.io/ansible/receptor:devel as receptor -FROM --platform=$TARGETPLATFORM python:3.11-slim-bullseye +FROM --platform=$TARGETPLATFORM python:3.11-slim-bookworm # Ansible chucks a wobbler without. see: https://github.com/ansible/ansible/issues/78283 ENV LC_ALL en_US.UTF-8 From 5fe7a206f0c7741986dffb6af82e97f9554c75ff Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 02:30:12 +0930 Subject: [PATCH 14/16] feat(docker): add required deps for mysql ansible module !184 fixes #25 --- dockerfile | 1 + dockerfile-build_cache | 1 + 2 files changed, 2 insertions(+) diff --git a/dockerfile b/dockerfile index 9ee6b9a..f2ca644 100644 --- a/dockerfile +++ b/dockerfile @@ -57,6 +57,7 @@ RUN apt update \ postgresql-common \ postgresql-client \ mariadb-client \ + mariadb-client-core \ && mkdir -p /etc/ansible/roles \ && mkdir -p /etc/ansible/collections \ && mkdir -p /workdir \ diff --git a/dockerfile-build_cache b/dockerfile-build_cache index 3cceb30..7f8a3ff 100644 --- a/dockerfile-build_cache +++ b/dockerfile-build_cache @@ -61,6 +61,7 @@ RUN apt install -yq \ libpq-dev \ libssl-dev \ mariadb-client \ + mariadb-client-core \ openssh-client \ pkg-config \ postgresql-client \ From 4a722374a8561d301c82839dca0ab2f812f55b07 Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 02:35:15 +0930 Subject: [PATCH 15/16] docs: update to current structure !184 --- .../{ => ansible}/execution_environment/index.md | 4 ++-- docs/projects/{ansible-roles => ansible}/index.md | 0 .../{ansible_playbooks => ansible/playbooks}/index.md | 0 docs/projects/ansible/roles/index.md | 0 mkdocs.yml | 11 +++++++++-- 5 files changed, 11 insertions(+), 4 deletions(-) rename docs/projects/{ => ansible}/execution_environment/index.md (96%) rename docs/projects/{ansible-roles => ansible}/index.md (100%) rename docs/projects/{ansible_playbooks => ansible/playbooks}/index.md (100%) create mode 100644 docs/projects/ansible/roles/index.md diff --git a/docs/projects/execution_environment/index.md b/docs/projects/ansible/execution_environment/index.md similarity index 96% rename from docs/projects/execution_environment/index.md rename to docs/projects/ansible/execution_environment/index.md index 9a162ae..c5c9777 100644 --- a/docs/projects/execution_environment/index.md +++ b/docs/projects/ansible/execution_environment/index.md @@ -8,9 +8,9 @@ about: https://gitlab.com/nofusscomputing/projects/ansible/execution_environment This docker container is designed to be used with Ansible AWX/Tower or from within the Gitlab CI/CD Environment. Included in the container are the following projects from us: -- [Ansible Playbooks](../ansible_playbooks/index.md) +- [Ansible Playbooks](../playbooks/index.md) -- [Ansible Roles](../ansible-roles/index.md) +- [Ansible Roles](../roles/index.md) This container can be found at dockerhub at and for use from docker with `docker pull nofusscomputing/ansible-ee`. versioning and git tags are in accordance with semver. diff --git a/docs/projects/ansible-roles/index.md b/docs/projects/ansible/index.md similarity index 100% rename from docs/projects/ansible-roles/index.md rename to docs/projects/ansible/index.md diff --git a/docs/projects/ansible_playbooks/index.md b/docs/projects/ansible/playbooks/index.md similarity index 100% rename from docs/projects/ansible_playbooks/index.md rename to docs/projects/ansible/playbooks/index.md diff --git a/docs/projects/ansible/roles/index.md b/docs/projects/ansible/roles/index.md new file mode 100644 index 0000000..e69de29 diff --git a/mkdocs.yml b/mkdocs.yml index d0d7fb0..100dc4f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -17,11 +17,18 @@ nav: - projects/index.md - - Ansible Roles: + - Ansible: + + - projects/ansible/index.md - Execution Environment: - - projects/execution_environment/index.md + - projects/ansible/execution_environment/index.md + + - Playbooks: projects/ansible/playbooks/index.md + + - Roles: projects/ansible/roles/index.md + - Operations: From 18d15f2978ba9f3b7fd8802ed4d06d024d174a77 Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 Jan 2024 02:57:54 +0930 Subject: [PATCH 16/16] ci: update pages path !184 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8966c6c..2ebbe27 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: DOCKER_IMAGE_PUBLISH_NAME: 'ansible-ee' DOCKER_IMAGE_PUBLISH_REGISTRY: docker.io/nofusscomputing DOCKER_IMAGE_PUBLISH_URL: https://hub.docker.com/r/nofusscomputing/$DOCKER_IMAGE_PUBLISH_NAME - PAGES_ENVIRONMENT_PATH: projects/execution_environment/ + PAGES_ENVIRONMENT_PATH: projects/ansible/execution_environment/ include: - project: nofusscomputing/projects/gitlab-ci