feat(python_build): compile and cache pip packages

!3

dockerfile

@@ -1,24 +1,31 @@
-# docker pull python:3.9.16-slim-bullseye
+FROM --platform=$BUILDPLATFORM python:3.11-bullseye as fetch-ansible-roles
-# docker pull python:3.10.11-slim-bullseye
-
-FROM python:3.9.16-slim-bullseye as scratchpad
 
 
-RUN apt update && \
+ENV LC_ALL en_US.UTF-8
-  apt install --no-install-recommends -y \
+
-    git
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+  && dpkg-reconfigure debconf -f noninteractive
+
+
+RUN apt update \
+  && apt install --reinstall -yq \
+    git || true
 
 
 RUN git clone -b development --depth 1 https://gitlab.com/nofusscomputing/projects/ansible-roles.git /tmp/ansible-roles 
 
 
 
-FROM python:3.9.16-slim-bullseye
+FROM --platform=$TARGETPLATFORM python:3.11-bullseye
 
-COPY --from=scratchpad /tmp/ansible-roles/roles /etc/ansible/roles
+# Ansible chucks a wobbler without. see: https://github.com/ansible/ansible/issues/78283
+ENV LC_ALL en_US.UTF-8
+
+
+COPY --from=fetch-ansible-roles /tmp/ansible-roles/roles /etc/ansible/roles
 
 # Ref: https://github.com/opencontainers/image-spec/blob/d86384efdb8c30770a92415c100f57a9bffbb64e/annotations.md
-
 LABEL \
   # org.opencontainers.image.authors="{contributor url}" \
   org.opencontainers.image.vendor="No Fuss Computing" \
@@ -32,24 +39,43 @@ LABEL \
   # org.opencontainers.image.version="{git tag}"
 
 
-RUN apt update && \
+
-  apt install --no-install-recommends -y \
+# This Black Magic exists as libc-bin was being a turd and returning errors when trying to install git, ssh.
+# see: https://askubuntu.com/questions/1339558/cant-build-dockerfile-for-arm64-due-to-libc-bin-segmentation-fault
+# see: https://github.com/dcycle/prepare-docker-buildx/blob/09057fe4879e31ee780b9e69b87f41327ca8cd8e/example/Dockerfile#L8-L10
+RUN export DEBIAN_FRONTEND=noninteractive \
+  && apt update \
+  && apt --fix-broken install \
+  && apt install -y libc-bin locales-all \
+  && apt update \
+  && apt install --reinstall --no-install-recommends -yq \
+    openssh-client \
+    git || true \
+  && dpkg --purge --force-all libc-bin \
+  && apt-get install --no-install-recommends -y \
+    openssh-client \
     git \
-    ssh && \
+  # End of Black Magic
-  rm -rf /var/lib/apt/lists/* && \
+  && rm -rf /var/lib/apt/lists/* \
-  mkdir -p /etc/ansible/roles && \
+  && mkdir -p /etc/ansible/roles \
-  mkdir -p /etc/ansible/collections && \
+  && mkdir -p /etc/ansible/collections \
-  mkdir -p /workdir
+  && mkdir -p /workdir \
+  && apt list --installed
+
 
 WORKDIR /workdir
 
+
 COPY ansible.cfg /etc/ansible/ansible.cfg
 
 
-RUN pip install --upgrade pip \
+COPY requirements.txt /tmp/requirements.txt
-  && pip install \
+
-    ansible \
+
-    ansible-lint
+RUN pip install --upgrade pip
+
+RUN pip install --index-url https://gitlab.com/api/v4/projects/45741845/packages/pypi/simple -r /tmp/requirements.txt
+
 
 RUN ansible-galaxy collection install \
     awx.awx \

dockerfile-build_cache

@@ -0,0 +1,74 @@
+FROM --platform=$TARGETPLATFORM python:3.11-bullseye as wheelbuild
+
+ARG CI_JOB_TOKEN
+ARG CI_API_V4_URL
+ARG CI_PROJECT_ID
+
+
+ENV LC_ALL en_US.UTF-8
+ENV PATH /usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.cargo/bin::~/.cargo/bin
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+  && dpkg-reconfigure debconf -f noninteractive
+
+RUN apt update \
+  && apt install --reinstall -yq \
+    git \
+    curl \
+    rustc cargo || true
+
+
+RUN mkdir -p /tmp/python_modules /tmp/python_builds
+
+
+RUN rustc --version \
+  && cargo --version
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+  && apt update \
+  && apt --fix-broken install \
+  && apt install -y libc-bin locales-all \
+  && apt update \
+  && apt install --reinstall -yq \
+    openssh-client \
+    git || true \
+  && dpkg --purge --force-all libc-bin \
+  && apt-get install -y \
+    openssh-client \
+    git
+
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+  && apt update \
+  && apt install --no-install-recommends -yq libc-bin locales-all \
+  && apt-get install --reinstall -yq \
+    build-essential \
+    libssl-dev \
+    libffi-dev \
+    python3-dev \
+    cargo \
+    pkg-config || true
+
+
+RUN pip install --upgrade pip
+
+RUN pip install --upgrade \
+    setuptools \
+    wheel \
+    setuptools-rust \
+    twine
+
+COPY requirements.txt /tmp/requirements.txt
+
+RUN cd /tmp/python_modules \
+  && pip download --dest . --check-build-dependencies \
+    -r /tmp/requirements.txt
+
+
+RUN cd /tmp/python_modules \
+  # && export PATH=$PATH:~/.cargo/bin \
+  && echo "[DEBUG] PATH=$PATH" \
+  && pip wheel --wheel-dir /tmp/python_builds --find-links . *.whl
+
+
+RUN TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --skip-existing --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi /tmp/python_builds/*

requirements.txt

@@ -0,0 +1,2 @@
+ansible-core==2.14.5
+ansible-lint==6.15.0