From 3f6e77b0ed651a1f0418ada7d8849b5f048880f1 Mon Sep 17 00:00:00 2001 From: Jon Date: Sat, 13 May 2023 14:32:15 +0930 Subject: [PATCH] feat(python_build): compile and cache pip packages !3 --- dockerfile | 68 ++++++++++++++++++++++++++------------ dockerfile-build_cache | 74 ++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 ++ 3 files changed, 123 insertions(+), 21 deletions(-) create mode 100644 dockerfile-build_cache create mode 100644 requirements.txt diff --git a/dockerfile b/dockerfile index 9d5c061..62b190e 100644 --- a/dockerfile +++ b/dockerfile @@ -1,24 +1,31 @@ -# docker pull python:3.9.16-slim-bullseye -# docker pull python:3.10.11-slim-bullseye - -FROM python:3.9.16-slim-bullseye as scratchpad +FROM --platform=$BUILDPLATFORM python:3.11-bullseye as fetch-ansible-roles -RUN apt update && \ - apt install --no-install-recommends -y \ - git +ENV LC_ALL en_US.UTF-8 + + +RUN export DEBIAN_FRONTEND=noninteractive \ + && dpkg-reconfigure debconf -f noninteractive + + +RUN apt update \ + && apt install --reinstall -yq \ + git || true RUN git clone -b development --depth 1 https://gitlab.com/nofusscomputing/projects/ansible-roles.git /tmp/ansible-roles -FROM python:3.9.16-slim-bullseye +FROM --platform=$TARGETPLATFORM python:3.11-bullseye -COPY --from=scratchpad /tmp/ansible-roles/roles /etc/ansible/roles +# Ansible chucks a wobbler without. see: https://github.com/ansible/ansible/issues/78283 +ENV LC_ALL en_US.UTF-8 + + +COPY --from=fetch-ansible-roles /tmp/ansible-roles/roles /etc/ansible/roles # Ref: https://github.com/opencontainers/image-spec/blob/d86384efdb8c30770a92415c100f57a9bffbb64e/annotations.md - LABEL \ # org.opencontainers.image.authors="{contributor url}" \ org.opencontainers.image.vendor="No Fuss Computing" \ @@ -32,24 +39,43 @@ LABEL \ # org.opencontainers.image.version="{git tag}" -RUN apt update && \ - apt install --no-install-recommends -y \ + +# This Black Magic exists as libc-bin was being a turd and returning errors when trying to install git, ssh. +# see: https://askubuntu.com/questions/1339558/cant-build-dockerfile-for-arm64-due-to-libc-bin-segmentation-fault +# see: https://github.com/dcycle/prepare-docker-buildx/blob/09057fe4879e31ee780b9e69b87f41327ca8cd8e/example/Dockerfile#L8-L10 +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt update \ + && apt --fix-broken install \ + && apt install -y libc-bin locales-all \ + && apt update \ + && apt install --reinstall --no-install-recommends -yq \ + openssh-client \ + git || true \ + && dpkg --purge --force-all libc-bin \ + && apt-get install --no-install-recommends -y \ + openssh-client \ git \ - ssh && \ - rm -rf /var/lib/apt/lists/* && \ - mkdir -p /etc/ansible/roles && \ - mkdir -p /etc/ansible/collections && \ - mkdir -p /workdir + # End of Black Magic + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /etc/ansible/roles \ + && mkdir -p /etc/ansible/collections \ + && mkdir -p /workdir \ + && apt list --installed + WORKDIR /workdir + COPY ansible.cfg /etc/ansible/ansible.cfg -RUN pip install --upgrade pip \ - && pip install \ - ansible \ - ansible-lint +COPY requirements.txt /tmp/requirements.txt + + +RUN pip install --upgrade pip + +RUN pip install --index-url https://gitlab.com/api/v4/projects/45741845/packages/pypi/simple -r /tmp/requirements.txt + RUN ansible-galaxy collection install \ awx.awx \ diff --git a/dockerfile-build_cache b/dockerfile-build_cache new file mode 100644 index 0000000..53c99ca --- /dev/null +++ b/dockerfile-build_cache @@ -0,0 +1,74 @@ +FROM --platform=$TARGETPLATFORM python:3.11-bullseye as wheelbuild + +ARG CI_JOB_TOKEN +ARG CI_API_V4_URL +ARG CI_PROJECT_ID + + +ENV LC_ALL en_US.UTF-8 +ENV PATH /usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.cargo/bin::~/.cargo/bin + +RUN export DEBIAN_FRONTEND=noninteractive \ + && dpkg-reconfigure debconf -f noninteractive + +RUN apt update \ + && apt install --reinstall -yq \ + git \ + curl \ + rustc cargo || true + + +RUN mkdir -p /tmp/python_modules /tmp/python_builds + + +RUN rustc --version \ + && cargo --version + +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt update \ + && apt --fix-broken install \ + && apt install -y libc-bin locales-all \ + && apt update \ + && apt install --reinstall -yq \ + openssh-client \ + git || true \ + && dpkg --purge --force-all libc-bin \ + && apt-get install -y \ + openssh-client \ + git + + +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt update \ + && apt install --no-install-recommends -yq libc-bin locales-all \ + && apt-get install --reinstall -yq \ + build-essential \ + libssl-dev \ + libffi-dev \ + python3-dev \ + cargo \ + pkg-config || true + + +RUN pip install --upgrade pip + +RUN pip install --upgrade \ + setuptools \ + wheel \ + setuptools-rust \ + twine + +COPY requirements.txt /tmp/requirements.txt + +RUN cd /tmp/python_modules \ + && pip download --dest . --check-build-dependencies \ + -r /tmp/requirements.txt + + +RUN cd /tmp/python_modules \ + # && export PATH=$PATH:~/.cargo/bin \ + && echo "[DEBUG] PATH=$PATH" \ + && pip wheel --wheel-dir /tmp/python_builds --find-links . *.whl + + +RUN TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --skip-existing --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi /tmp/python_builds/* diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..aee9e30 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +ansible-core==2.14.5 +ansible-lint==6.15.0