From 1c517d41575b838e61561f03fcf1d118df163f55 Mon Sep 17 00:00:00 2001 From: Jon Date: Fri, 5 May 2023 11:29:09 +0930 Subject: [PATCH 01/16] ci(github): add github sync job !1 --- .gitlab-ci.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..d5c3444 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,29 @@ +stages: + - sync + + +.git_push_mirror: + image: alpine:latest + stage: sync + before_script: + - apk update + - apk add git + script: + - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/$CI_PROJECT_PATH --mirror $CI_PROJECT_NAME + - cd $CI_PROJECT_NAME + - git remote add destination $GIT_SYNC_URL + - echo "https://gitlab.com/ $CI_PROJECT_PATH / $CI_PROJECT_NAME .git" + - git push destination --mirror + artifacts: + expire_in: 1 day + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == "development"' + when: always + + +Github (Push --mirror): + variables: + GIT_SYNC_URL: "https://$GITHUB_USERNAME_ROBOT:$GITHUB_TOKEN_ROBOT@github.com/NoFussComputing/execution_environment.git" + extends: + - .git_push_mirror From 2cbc4e11d9e4e935db54ae1e3e319b1e4d813299 Mon Sep 17 00:00:00 2001 From: nfc_bot Date: Sun, 7 May 2023 12:19:13 +0930 Subject: [PATCH 02/16] docs(readme): Updated Repository README Updated by ansible MR !$MR_NUMBER --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0124703..136c269 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,56 @@ -# execution environment Repository. +
+ + + +# No Fuss Computing - Execution Environment + +
+ +![Project Status - Active](https://img.shields.io/badge/Project%20Status-Active-green?logo=gitlab&style=plastic) + +
+ +![Gitlab forks count](https://img.shields.io/badge/dynamic/json?label=Forks&query=%24.forks_count&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F45741845%2F&color=ff782e&logo=gitlab&style=plastic) ![Gitlab stars](https://img.shields.io/badge/dynamic/json?label=Stars&query=%24.star_count&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F45741845%2F&color=ff782e&logo=gitlab&style=plastic) [![Open Issues](https://img.shields.io/badge/dynamic/json?color=ff782e&logo=gitlab&style=plastic&label=Open%20Issues&query=%24.statistics.counts.opened&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F45741845%2Fissues_statistics)](https://gitlab.com/nofusscomputing/projects/ansible/execution_environment/-/issues) + + + +![GitHub forks](https://img.shields.io/github/forks/NofussComputing/git_configuration?logo=github&style=plastic&color=000000&labell=Forks) ![GitHub stars](https://img.shields.io/github/stars/NofussComputing/git_configuration?color=000000&logo=github&style=plastic) ![Github Watchers](https://img.shields.io/github/watchers/NofussComputing/git_configuration?color=000000&label=Watchers&logo=github&style=plastic) +
+ +This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/ansible/execution_environment) and has a read-only copy hosted on [Github](https://github.com/NofussComputing/git_configuration). + +---- + +**Stable Branch** + +![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F45741845%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/execution_environment%2F-%2Fraw%2Fmaster%2F.cz.yaml) + +---- + +**Development Branch** + +![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F45741845%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/execution_environment-%2Fraw%2Fdevelopment%2F.cz.yaml) + +---- +
+ +
+ +links: + +- [Issues](https://gitlab.com/nofusscomputing/projects/ansible/execution_environment/-/issues) + +- [Merge Requests (Pull Requests)](https://gitlab.com/nofusscomputing/projects/ansible/execution_environment/-/merge_requests) + + + +## Contributing +All contributions for this project must conducted from [Gitlab](https://gitlab.com/nofusscomputing/projects/ansible/execution_environment). + +For further details on contributing please refer to the [contribution guide](CONTRIBUTING.md). + + +## Other + +This repo is release under this [license](LICENSE) -This repository was created by ansible!! \ No newline at end of file From e6ce3f9581a90f5ce4d11eaf16e89b6956ec39c9 Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 7 May 2023 12:40:24 +0930 Subject: [PATCH 03/16] ci: add dummy job !1 --- .gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d5c3444..3afa28c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,17 @@ stages: + - validation - sync +dummy: + image: alpine:latest + stage: validation + script: + - echo dummy job + rules: + - if: "$CI_COMMIT_BRANCH" + when: always + - when: never + .git_push_mirror: image: alpine:latest From 669689e8df76bc3d3225081db99918cbd4b51688 Mon Sep 17 00:00:00 2001 From: nfc_bot Date: Sun, 7 May 2023 12:52:26 +0930 Subject: [PATCH 04/16] docs(readme): Updated Repository README Updated by ansible MR !$MR_NUMBER --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 136c269..9daf078 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,10 @@ -![GitHub forks](https://img.shields.io/github/forks/NofussComputing/git_configuration?logo=github&style=plastic&color=000000&labell=Forks) ![GitHub stars](https://img.shields.io/github/stars/NofussComputing/git_configuration?color=000000&logo=github&style=plastic) ![Github Watchers](https://img.shields.io/github/watchers/NofussComputing/git_configuration?color=000000&label=Watchers&logo=github&style=plastic) +![GitHub forks](https://img.shields.io/github/forks/NofussComputing/execution_environment?logo=github&style=plastic&color=000000&labell=Forks) ![GitHub stars](https://img.shields.io/github/stars/NofussComputing/execution_environment?color=000000&logo=github&style=plastic) ![Github Watchers](https://img.shields.io/github/watchers/NofussComputing/execution_environment?color=000000&label=Watchers&logo=github&style=plastic)
-This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/ansible/execution_environment) and has a read-only copy hosted on [Github](https://github.com/NofussComputing/git_configuration). +This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/ansible/execution_environment) and has a read-only copy hosted on [Github](https://github.com/NofussComputing/execution_environment). ---- From f43f265c7f1691c58bcdf8bb4ec0a0bf8fb0be6f Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 8 May 2023 10:19:40 +0930 Subject: [PATCH 05/16] ci: added initial ci jobs !3 --- .cz.yaml | 7 +++ .gitlab-ci.yml | 157 ++++++++++++++++++++++++++++++++++++++++--------- .gitmodules | 3 + gitlab-ci | 1 + 4 files changed, 141 insertions(+), 27 deletions(-) create mode 100644 .cz.yaml create mode 100644 .gitmodules create mode 160000 gitlab-ci diff --git a/.cz.yaml b/.cz.yaml new file mode 100644 index 0000000..62b2ada --- /dev/null +++ b/.cz.yaml @@ -0,0 +1,7 @@ +commitizen: + bump_message: "build(version): bump version $current_version \u2192 $new_version" + changelog_incremental: false + name: cz_conventional_commits + tag_format: $major.$minor.$patch$prerelease + update_changelog_on_bump: true + version: 0.0.1 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3afa28c..873f467 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,36 +1,139 @@ stages: - validation + - build + - prepare + - test + - release - sync + - publish + +include: + - project: nofusscomputing/projects/gitlab-ci + ref: 36ce0b0b76e6769c7a2e0d4ea0f3fcd2cc2d6bb1 + file: + - conventional_commits/.gitlab-ci.yml + - gitlab_release/.gitlab-ci.yml + - git_push_mirror/.gitlab-ci.yml + + +Docker Container: + stage: build + image: docker:latest + services: + #- docker:19.03.12-dind + - docker:latest-dind + variables: + IMAGE_NAME: ansible-ee + before_script: + - docker info +# - docker login $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - apk update + - apk add --update --no-cache python3 git && ln -sf python3 /usr/bin/python + - python3 -m ensurepip + - pip3 install -r gitlab-ci/gitlab_release/requirements.txt + - pip3 install gitlab-ci/gitlab_release/python-module/cz_nfc/. + script: | + docker build . \ + --label org.opencontainers.image.created="$(date '+%Y-%m-%d %H:%M:%S%:z')" \ + --label org.opencontainers.image.documentation="$CI_PROJECT_URL" \ + --label org.opencontainers.image.source="$CI_PROJECT_URL" \ + --label org.opencontainers.image.url="$CI_PROJECT_URL/-/releases/$(cz -n cz_nfc version --project)" \ + --label org.opencontainers.image.version="$(cz -n cz_nfc version --project)" \ + --label org.opencontainers.image.revision="$CI_COMMIT_SHA" \ + --no-cache \ + --tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA; + + docker image inspect $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA; + + docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA; + + #allow_failure: true + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - if: $CI_COMMIT_TAG + when: on_success + # - if: '$CI_COMMIT_BRANCH == "development" && $CI_COMMIT_TITLE =~ /Merge branch.*/' + # when: on_success + - if: '$CI_COMMIT_BRANCH != "master"' + allow_failure: true + when: manual + - when: never + + + +Docker Hub: + stage: publish + image: docker:latest + services: + #- docker:19.03.12-dind + - docker:latest-dind + variables: + IMAGE_NAME: ansible-ee + before_script: + - export + - docker login $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD + - docker pull $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA + - docker logout $CI_REGISTRY + script: +# - Release_TAG=$(cat $CI_PROJECT_DIR/dist/version) + - docker login docker.io -u $NFC_DOCKERHUB_USERNAME -p $NFC_DOCKERHUB_TOKEN +# - if [ "m$(echo $CI_BUILD_REF_NAME | grep rc)" == "m$CI_BUILD_REF_NAME" ]; then Branch_TAG=dev; else Branch_TAG=stable; fi + - echo Branch tag is $Branch_TAG + - docker image ls + - | + if [ "0$CI_COMMIT_BRANCH" == "0development" ]; then + + DH_LATEST_TAG=dev + + elif [ "0$CI_COMMIT_TAG" != "0" ] ; then + + if [ "0$CI_COMMIT_TAG" == *"rc"* ]; then + + DH_LATEST_TAG=dev + + else + + DH_LATEST_TAG=latest + + fi + + docker image tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA nofusscomputing/$IMAGE_NAME:$CI_COMMIT_TAG; + + docker image ls; + + docker push nofusscomputing/$IMAGE_NAME:$CI_COMMIT_TAG; + + fi + + if [ "0$DH_LATEST_TAG" != "0" ]; then + + docker image tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA nofusscomputing/$IMAGE_NAME:$DH_LATEST_TAG; + + docker image ls; + + docker push nofusscomputing/$IMAGE_NAME:$DH_LATEST_TAG; + + fi + needs: [ "Docker Container" ] + environment: + name: DockerHub + url: https://hub.docker.com/r/nofusscomputing/docker-mail + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - if: $CI_COMMIT_TAG + when: on_success + # - if: '$CI_COMMIT_BRANCH == "development" && $CI_COMMIT_TITLE =~ /Merge branch.*/' + # when: on_success + - if: '$CI_COMMIT_BRANCH != "master"' + allow_failure: true + when: manual + - when: never -dummy: - image: alpine:latest - stage: validation - script: - - echo dummy job - rules: - - if: "$CI_COMMIT_BRANCH" - when: always - - when: never -.git_push_mirror: - image: alpine:latest - stage: sync - before_script: - - apk update - - apk add git - script: - - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/$CI_PROJECT_PATH --mirror $CI_PROJECT_NAME - - cd $CI_PROJECT_NAME - - git remote add destination $GIT_SYNC_URL - - echo "https://gitlab.com/ $CI_PROJECT_PATH / $CI_PROJECT_NAME .git" - - git push destination --mirror - artifacts: - expire_in: 1 day - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - - if: '$CI_COMMIT_BRANCH == "development"' - when: always Github (Push --mirror): diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..abc28e7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "gitlab-ci"] + path = gitlab-ci + url = https://gitlab.com/nofusscomputing/projects/gitlab-ci.git diff --git a/gitlab-ci b/gitlab-ci new file mode 160000 index 0000000..36ce0b0 --- /dev/null +++ b/gitlab-ci @@ -0,0 +1 @@ +Subproject commit 36ce0b0b76e6769c7a2e0d4ea0f3fcd2cc2d6bb1 From f944a3c6ee2fb997a05f891c1c40f5c2d0aba7c0 Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 8 May 2023 10:20:02 +0930 Subject: [PATCH 06/16] chore: initial project files !3 --- .dockerignore | 2 ++ ansible.cfg | 4 ++++ dockerfile | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 .dockerignore create mode 100644 ansible.cfg create mode 100644 dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a03425c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.gitlab-ci.yml +README.md diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..047d6f8 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,4 @@ +[defaults] +collections_path=/etc/ansible/collections +roles_path=/etc/ansible/roles +log_path=/ansible.log diff --git a/dockerfile b/dockerfile new file mode 100644 index 0000000..61f2dcd --- /dev/null +++ b/dockerfile @@ -0,0 +1,56 @@ +# docker pull python:3.9.16-slim-bullseye +# docker pull python:3.10.11-slim-bullseye + +FROM python:3.9.16-slim-bullseye as scratchpad + + +RUN apt update && \ + apt install --no-install-recommends -y \ + git + + +RUN git clone -b development --depth 1 https://gitlab.com/nofusscomputing/projects/ansible-roles.git /tmp/ansible-roles + + + +FROM python:3.9.16-slim-bullseye + +COPY --from=scratchpad /tmp/ansible-roles/roles /etc/ansible/roles + +# Ref: https://github.com/opencontainers/image-spec/blob/d86384efdb8c30770a92415c100f57a9bffbb64e/annotations.md + +LABEL \ + # org.opencontainers.image.authors="{contributor url}" \ + org.opencontainers.image.vendor="No Fuss Computing" \ + # org.opencontainers.image.url="{dockerhub url}" \ + # org.opencontainers.image.documentation="{docs url}" \ + # org.opencontainers.image.source="{repo url}" \ + # org.opencontainers.image.revision="{git commit sha at time of build}" \ + org.opencontainers.image.title="No Fuss Computings Ansible Execution Environment" \ + org.opencontainers.image.description="An ansible execution environment for awx/tower and CI/CD pipelines" \ + org.opencontainers.image.vendor="No Fuss Computing" \ + # org.opencontainers.image.version="{git tag}" + + +RUN apt update && \ + apt install --no-install-recommends -y \ + git \ + ssh && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir -p /etc/ansible/roles && \ + mkdir -p /etc/ansible/collections && \ + mkdir -p /workdir + +WORKDIR /workdir + +COPY ansible.cfg /etc/ansible/ansible.cfg + + +RUN pip install --upgrade pip \ + && pip install \ + ansible \ + ansible-lint + +RUN ansible-galaxy collection install \ + awx.awx \ + kubernetes.core \ No newline at end of file From cd82c7c3d7a81a27f10c15ec3a6aaedbe2712894 Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 8 May 2023 10:24:46 +0930 Subject: [PATCH 07/16] fix(ci): ensure submodules expanded !3 --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 873f467..376f160 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,9 @@ +--- +variables: + GIT_SUBMODULE_STRATEGY: recursive + MY_PROJECT_ID: "45741845" + + stages: - validation - build From e1ffb03ce4473b4ef8d56a56847fc27390df8bfd Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 8 May 2023 10:29:20 +0930 Subject: [PATCH 08/16] fix(ci): use specified dind imge !3 --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 376f160..4dd1ffb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,8 +26,8 @@ Docker Container: stage: build image: docker:latest services: - #- docker:19.03.12-dind - - docker:latest-dind + - docker:19.03.12-dind + #- docker:latest-dind variables: IMAGE_NAME: ansible-ee before_script: @@ -73,8 +73,8 @@ Docker Hub: stage: publish image: docker:latest services: - #- docker:19.03.12-dind - - docker:latest-dind + - docker:19.03.12-dind + #- docker:latest-dind variables: IMAGE_NAME: ansible-ee before_script: From 53e445d15511bb7691acc273d2e0c90c02da30d8 Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 8 May 2023 10:37:08 +0930 Subject: [PATCH 09/16] fix(ci): specify commitezen version this is a temp fix so that the existing task scripts work !3 --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4dd1ffb..e6be181 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,7 +37,8 @@ Docker Container: - apk update - apk add --update --no-cache python3 git && ln -sf python3 /usr/bin/python - python3 -m ensurepip - - pip3 install -r gitlab-ci/gitlab_release/requirements.txt + #- pip3 install -r gitlab-ci/gitlab_release/requirements.txt + - pip3 install commitizen==2.21.0 # version specified so current build works. - pip3 install gitlab-ci/gitlab_release/python-module/cz_nfc/. script: | docker build . \ From 42d2cb79dfc9d1ecbdab9945b327d819f00631fe Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 8 May 2023 10:50:46 +0930 Subject: [PATCH 10/16] fix(ci): syntax error !3 --- dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile b/dockerfile index 61f2dcd..9d5c061 100644 --- a/dockerfile +++ b/dockerfile @@ -28,7 +28,7 @@ LABEL \ # org.opencontainers.image.revision="{git commit sha at time of build}" \ org.opencontainers.image.title="No Fuss Computings Ansible Execution Environment" \ org.opencontainers.image.description="An ansible execution environment for awx/tower and CI/CD pipelines" \ - org.opencontainers.image.vendor="No Fuss Computing" \ + org.opencontainers.image.vendor="No Fuss Computing" # org.opencontainers.image.version="{git tag}" From 0a367146bfa5b409273c0412fca8f4d33287068d Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 8 May 2023 11:17:42 +0930 Subject: [PATCH 11/16] ci(publish): default to dev tag for image !3 --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e6be181..5776a5f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,6 +90,8 @@ Docker Hub: - echo Branch tag is $Branch_TAG - docker image ls - | + DH_LATEST_TAG=dev + if [ "0$CI_COMMIT_BRANCH" == "0development" ]; then DH_LATEST_TAG=dev @@ -126,7 +128,7 @@ Docker Hub: needs: [ "Docker Container" ] environment: name: DockerHub - url: https://hub.docker.com/r/nofusscomputing/docker-mail + url: https://hub.docker.com/r/nofusscomputing/$IMAGE_NAME rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: never From 3f6e77b0ed651a1f0418ada7d8849b5f048880f1 Mon Sep 17 00:00:00 2001 From: Jon Date: Sat, 13 May 2023 14:32:15 +0930 Subject: [PATCH 12/16] feat(python_build): compile and cache pip packages !3 --- dockerfile | 68 ++++++++++++++++++++++++++------------ dockerfile-build_cache | 74 ++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 ++ 3 files changed, 123 insertions(+), 21 deletions(-) create mode 100644 dockerfile-build_cache create mode 100644 requirements.txt diff --git a/dockerfile b/dockerfile index 9d5c061..62b190e 100644 --- a/dockerfile +++ b/dockerfile @@ -1,24 +1,31 @@ -# docker pull python:3.9.16-slim-bullseye -# docker pull python:3.10.11-slim-bullseye - -FROM python:3.9.16-slim-bullseye as scratchpad +FROM --platform=$BUILDPLATFORM python:3.11-bullseye as fetch-ansible-roles -RUN apt update && \ - apt install --no-install-recommends -y \ - git +ENV LC_ALL en_US.UTF-8 + + +RUN export DEBIAN_FRONTEND=noninteractive \ + && dpkg-reconfigure debconf -f noninteractive + + +RUN apt update \ + && apt install --reinstall -yq \ + git || true RUN git clone -b development --depth 1 https://gitlab.com/nofusscomputing/projects/ansible-roles.git /tmp/ansible-roles -FROM python:3.9.16-slim-bullseye +FROM --platform=$TARGETPLATFORM python:3.11-bullseye -COPY --from=scratchpad /tmp/ansible-roles/roles /etc/ansible/roles +# Ansible chucks a wobbler without. see: https://github.com/ansible/ansible/issues/78283 +ENV LC_ALL en_US.UTF-8 + + +COPY --from=fetch-ansible-roles /tmp/ansible-roles/roles /etc/ansible/roles # Ref: https://github.com/opencontainers/image-spec/blob/d86384efdb8c30770a92415c100f57a9bffbb64e/annotations.md - LABEL \ # org.opencontainers.image.authors="{contributor url}" \ org.opencontainers.image.vendor="No Fuss Computing" \ @@ -32,24 +39,43 @@ LABEL \ # org.opencontainers.image.version="{git tag}" -RUN apt update && \ - apt install --no-install-recommends -y \ + +# This Black Magic exists as libc-bin was being a turd and returning errors when trying to install git, ssh. +# see: https://askubuntu.com/questions/1339558/cant-build-dockerfile-for-arm64-due-to-libc-bin-segmentation-fault +# see: https://github.com/dcycle/prepare-docker-buildx/blob/09057fe4879e31ee780b9e69b87f41327ca8cd8e/example/Dockerfile#L8-L10 +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt update \ + && apt --fix-broken install \ + && apt install -y libc-bin locales-all \ + && apt update \ + && apt install --reinstall --no-install-recommends -yq \ + openssh-client \ + git || true \ + && dpkg --purge --force-all libc-bin \ + && apt-get install --no-install-recommends -y \ + openssh-client \ git \ - ssh && \ - rm -rf /var/lib/apt/lists/* && \ - mkdir -p /etc/ansible/roles && \ - mkdir -p /etc/ansible/collections && \ - mkdir -p /workdir + # End of Black Magic + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /etc/ansible/roles \ + && mkdir -p /etc/ansible/collections \ + && mkdir -p /workdir \ + && apt list --installed + WORKDIR /workdir + COPY ansible.cfg /etc/ansible/ansible.cfg -RUN pip install --upgrade pip \ - && pip install \ - ansible \ - ansible-lint +COPY requirements.txt /tmp/requirements.txt + + +RUN pip install --upgrade pip + +RUN pip install --index-url https://gitlab.com/api/v4/projects/45741845/packages/pypi/simple -r /tmp/requirements.txt + RUN ansible-galaxy collection install \ awx.awx \ diff --git a/dockerfile-build_cache b/dockerfile-build_cache new file mode 100644 index 0000000..53c99ca --- /dev/null +++ b/dockerfile-build_cache @@ -0,0 +1,74 @@ +FROM --platform=$TARGETPLATFORM python:3.11-bullseye as wheelbuild + +ARG CI_JOB_TOKEN +ARG CI_API_V4_URL +ARG CI_PROJECT_ID + + +ENV LC_ALL en_US.UTF-8 +ENV PATH /usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.cargo/bin::~/.cargo/bin + +RUN export DEBIAN_FRONTEND=noninteractive \ + && dpkg-reconfigure debconf -f noninteractive + +RUN apt update \ + && apt install --reinstall -yq \ + git \ + curl \ + rustc cargo || true + + +RUN mkdir -p /tmp/python_modules /tmp/python_builds + + +RUN rustc --version \ + && cargo --version + +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt update \ + && apt --fix-broken install \ + && apt install -y libc-bin locales-all \ + && apt update \ + && apt install --reinstall -yq \ + openssh-client \ + git || true \ + && dpkg --purge --force-all libc-bin \ + && apt-get install -y \ + openssh-client \ + git + + +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt update \ + && apt install --no-install-recommends -yq libc-bin locales-all \ + && apt-get install --reinstall -yq \ + build-essential \ + libssl-dev \ + libffi-dev \ + python3-dev \ + cargo \ + pkg-config || true + + +RUN pip install --upgrade pip + +RUN pip install --upgrade \ + setuptools \ + wheel \ + setuptools-rust \ + twine + +COPY requirements.txt /tmp/requirements.txt + +RUN cd /tmp/python_modules \ + && pip download --dest . --check-build-dependencies \ + -r /tmp/requirements.txt + + +RUN cd /tmp/python_modules \ + # && export PATH=$PATH:~/.cargo/bin \ + && echo "[DEBUG] PATH=$PATH" \ + && pip wheel --wheel-dir /tmp/python_builds --find-links . *.whl + + +RUN TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --skip-existing --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi /tmp/python_builds/* diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..aee9e30 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +ansible-core==2.14.5 +ansible-lint==6.15.0 From 3e7094c243e8a22bf3443bf8db176a4f0b69776c Mon Sep 17 00:00:00 2001 From: Jon Date: Sat, 13 May 2023 14:33:24 +0930 Subject: [PATCH 13/16] feat(ci): use nfc gitlab-ci project for ci !3 --- .gitlab-ci.yml | 153 ++++++++++--------------------------------------- 1 file changed, 30 insertions(+), 123 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5776a5f..62a4f90 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,151 +2,58 @@ variables: GIT_SUBMODULE_STRATEGY: recursive MY_PROJECT_ID: "45741845" + GIT_SYNC_URL: "https://$GITHUB_USERNAME_ROBOT:$GITHUB_TOKEN_ROBOT@github.com/NoFussComputing/execution_environment.git" + + DOCKER_IMAGE_BUILD_TARGET_PLATFORMS: "linux/amd64,linux/arm64,linux/arm/v7" + DOCKER_IMAGE_PUBLISH_NAME: 'ansible-ee' + DOCKER_IMAGE_PUBLISH_REGISTRY: docker.io/nofusscomputing + DOCKER_IMAGE_PUBLISH_URL: https://hub.docker.com/r/nofusscomputing/$DOCKER_IMAGE_PUBLISH_NAME -stages: - - validation - - build - - prepare - - test - - release - - sync - - publish include: - project: nofusscomputing/projects/gitlab-ci - ref: 36ce0b0b76e6769c7a2e0d4ea0f3fcd2cc2d6bb1 + ref: restructure file: - - conventional_commits/.gitlab-ci.yml - - gitlab_release/.gitlab-ci.yml - - git_push_mirror/.gitlab-ci.yml + - .gitlab-ci_common.yaml + - type/docker-image.gitlab-ci.yaml + #- template: Jobs/Container-Scanning.gitlab-ci.yml # see https://gitlab.com/gitlab-org/gitlab/-/issues/381665 -Docker Container: + +Build/Cache Python Packages: stage: build - image: docker:latest + image: nofusscomputing/docker-buildx-qemu:dev services: - - docker:19.03.12-dind - #- docker:latest-dind + - name: docker:23-dind + entrypoint: ["env", "-u", "DOCKER_HOST"] + command: ["dockerd-entrypoint.sh"] variables: - IMAGE_NAME: ansible-ee + DOCKER_HOST: tcp://docker:2375/ + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" before_script: - docker info -# - docker login $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - apk update - - apk add --update --no-cache python3 git && ln -sf python3 /usr/bin/python - - python3 -m ensurepip - #- pip3 install -r gitlab-ci/gitlab_release/requirements.txt + #- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - pip3 install setuptools wheel - pip3 install commitizen==2.21.0 # version specified so current build works. - pip3 install gitlab-ci/gitlab_release/python-module/cz_nfc/. + - update-binfmts --enable # Important: Ensures execution of other binary formats is enabled in the kernel + - docker buildx create --driver=docker-container --driver-opt image=moby/buildkit:v0.11.6 --use + - docker buildx inspect --bootstrap script: | - docker build . \ - --label org.opencontainers.image.created="$(date '+%Y-%m-%d %H:%M:%S%:z')" \ - --label org.opencontainers.image.documentation="$CI_PROJECT_URL" \ - --label org.opencontainers.image.source="$CI_PROJECT_URL" \ - --label org.opencontainers.image.url="$CI_PROJECT_URL/-/releases/$(cz -n cz_nfc version --project)" \ - --label org.opencontainers.image.version="$(cz -n cz_nfc version --project)" \ - --label org.opencontainers.image.revision="$CI_COMMIT_SHA" \ - --no-cache \ - --tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA; - - docker image inspect $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA; - - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA; - - #allow_failure: true + docker buildx build \ + --file dockerfile-build_cache \ + --platform=$DOCKER_IMAGE_BUILD_TARGET_PLATFORMS . \ + --build-arg CI_JOB_TOKEN=$CI_JOB_TOKEN \ + --build-arg CI_API_V4_URL=$CI_API_V4_URL \ + --build-arg CI_PROJECT_ID=$CI_PROJECT_ID rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: never - if: $CI_COMMIT_TAG - when: on_success - # - if: '$CI_COMMIT_BRANCH == "development" && $CI_COMMIT_TITLE =~ /Merge branch.*/' - # when: on_success - - if: '$CI_COMMIT_BRANCH != "master"' - allow_failure: true - when: manual - - when: never - - - -Docker Hub: - stage: publish - image: docker:latest - services: - - docker:19.03.12-dind - #- docker:latest-dind - variables: - IMAGE_NAME: ansible-ee - before_script: - - export - - docker login $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD - - docker pull $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA - - docker logout $CI_REGISTRY - script: -# - Release_TAG=$(cat $CI_PROJECT_DIR/dist/version) - - docker login docker.io -u $NFC_DOCKERHUB_USERNAME -p $NFC_DOCKERHUB_TOKEN -# - if [ "m$(echo $CI_BUILD_REF_NAME | grep rc)" == "m$CI_BUILD_REF_NAME" ]; then Branch_TAG=dev; else Branch_TAG=stable; fi - - echo Branch tag is $Branch_TAG - - docker image ls - - | - DH_LATEST_TAG=dev - - if [ "0$CI_COMMIT_BRANCH" == "0development" ]; then - - DH_LATEST_TAG=dev - - elif [ "0$CI_COMMIT_TAG" != "0" ] ; then - - if [ "0$CI_COMMIT_TAG" == *"rc"* ]; then - - DH_LATEST_TAG=dev - - else - - DH_LATEST_TAG=latest - - fi - - docker image tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA nofusscomputing/$IMAGE_NAME:$CI_COMMIT_TAG; - - docker image ls; - - docker push nofusscomputing/$IMAGE_NAME:$CI_COMMIT_TAG; - - fi - - if [ "0$DH_LATEST_TAG" != "0" ]; then - - docker image tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHA nofusscomputing/$IMAGE_NAME:$DH_LATEST_TAG; - - docker image ls; - - docker push nofusscomputing/$IMAGE_NAME:$DH_LATEST_TAG; - - fi - needs: [ "Docker Container" ] - environment: - name: DockerHub - url: https://hub.docker.com/r/nofusscomputing/$IMAGE_NAME - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: never - - if: $CI_COMMIT_TAG - when: on_success - # - if: '$CI_COMMIT_BRANCH == "development" && $CI_COMMIT_TITLE =~ /Merge branch.*/' - # when: on_success - if: '$CI_COMMIT_BRANCH != "master"' allow_failure: true when: manual - when: never - - - - - -Github (Push --mirror): - variables: - GIT_SYNC_URL: "https://$GITHUB_USERNAME_ROBOT:$GITHUB_TOKEN_ROBOT@github.com/NoFussComputing/execution_environment.git" - extends: - - .git_push_mirror From fb7db64235133aac3ff02331b14ae8131f9da9bf Mon Sep 17 00:00:00 2001 From: Jon Date: Sat, 13 May 2023 14:37:15 +0930 Subject: [PATCH 14/16] docs: todo !3 --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9daf078..5623770 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,12 @@ links: - [Merge Requests (Pull Requests)](https://gitlab.com/nofusscomputing/projects/ansible/execution_environment/-/merge_requests) +## Documentation + + +**TO DO:** + +point docs to gitlab-ci for docker build ## Contributing @@ -53,4 +59,3 @@ For further details on contributing please refer to the [contribution guide](CON ## Other This repo is release under this [license](LICENSE) - From 5484d6315a7179a579dca5cb05cee5dec9bff74b Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 14 May 2023 16:15:11 +0930 Subject: [PATCH 15/16] chore: update submodule to current dev !3 --- .gitlab-ci.yml | 2 +- .gitmodules | 1 + gitlab-ci | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 62a4f90..fcec1c7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,7 +13,7 @@ variables: include: - project: nofusscomputing/projects/gitlab-ci - ref: restructure + ref: development file: - .gitlab-ci_common.yaml - type/docker-image.gitlab-ci.yaml diff --git a/.gitmodules b/.gitmodules index abc28e7..cc4d8a5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "gitlab-ci"] path = gitlab-ci url = https://gitlab.com/nofusscomputing/projects/gitlab-ci.git + branch = development diff --git a/gitlab-ci b/gitlab-ci index 36ce0b0..a073dd0 160000 --- a/gitlab-ci +++ b/gitlab-ci @@ -1 +1 @@ -Subproject commit 36ce0b0b76e6769c7a2e0d4ea0f3fcd2cc2d6bb1 +Subproject commit a073dd0bad691992d0b8618ff0616175031a669a From b0f236d93553f1301a8487092e0cb5105b7abccc Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 15 May 2023 10:50:47 +0930 Subject: [PATCH 16/16] ci: update to current gitlab-ci project dev head !3 --- gitlab-ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab-ci b/gitlab-ci index a073dd0..18a2808 160000 --- a/gitlab-ci +++ b/gitlab-ci @@ -1 +1 @@ -Subproject commit a073dd0bad691992d0b8618ff0616175031a669a +Subproject commit 18a280878799fe077efa0ed5b11bceedd53eb5c8