From 5980123e7aada72419ad5aab5d951e23fedccbd7 Mon Sep 17 00:00:00 2001 From: Jon Date: Sat, 30 Mar 2024 03:11:17 +0930 Subject: [PATCH] feat(test): add integration test. playbook install !53 --- .gitignore | 2 + .gitlab-ci.yml | 1 + .gitlab/integration_test.gitlab-ci.yml | 207 ++++++++++++++++++ .gitlab/integration_test_trace.sh | 42 ++++ .gitlab/test_results.yaml | 19 ++ .vscode/settings.json | 3 +- README.md | 5 + .../ansible/collection/kubernetes/index.md | 2 + roles/nfc_kubernetes/tasks/install.yaml | 9 + roles/nfc_kubernetes/tasks/k3s/install.yaml | 33 ++- 10 files changed, 314 insertions(+), 9 deletions(-) create mode 100644 .gitlab/integration_test.gitlab-ci.yml create mode 100644 .gitlab/integration_test_trace.sh create mode 100644 .gitlab/test_results.yaml diff --git a/.gitignore b/.gitignore index 5fa197d..953f083 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ artifacts/ build/ +test_results/ +test_results.json *.tar.gz \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab3349b..5f9ffb3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ variables: include: + - local: .gitlab/integration_test.gitlab-ci.yml - project: nofusscomputing/projects/gitlab-ci ref: development file: diff --git a/.gitlab/integration_test.gitlab-ci.yml b/.gitlab/integration_test.gitlab-ci.yml new file mode 100644 index 0000000..2eb1c44 --- /dev/null +++ b/.gitlab/integration_test.gitlab-ci.yml @@ -0,0 +1,207 @@ + +.integration_test: + + stage: test + + needs: + - "Build Collection" + + image: + name: nofusscomputing/docker-buildx-qemu:dev + pull_policy: always + + variables: + DOCKER_HOST: tcp://docker:2375/ + DOCKER_DRIVER: overlay2 + # GIT_STRATEGY: none + + services: + - name: docker:23-dind + entrypoint: ["env", "-u", "DOCKER_HOST"] + command: ["dockerd-entrypoint.sh"] + before_script: + - | # start test container + docker run -d \ + --privileged \ + -v ${PWD}:/workdir \ + -v ${PWD}/artifacts/galaxy:/collection \ + --workdir /workdir \ + --rm \ + --env "ANSIBLE_FORCE_COLOR=true" \ + --env "CI_COMMIT_SHA=${CI_COMMIT_SHA}" \ + --env "ANSIBLE_LOG_PATH=/workdir/ansible.log" \ + --name test_image_${CI_JOB_ID} \ + nofusscomputing/ansible-docker-os:dev-${test_image} + + - | # enter test container + docker exec -i test_image_${CI_JOB_ID} ps aux + - docker ps + - docker exec -i test_image_${CI_JOB_ID} apt update + - docker exec -i test_image_${CI_JOB_ID} apt install -y --no-install-recommends python3-pip net-tools dnsutils iptables + - | + if [ "${test_image}" == 'debian-12' ]; then + + echo "Debian 12": + + docker exec -i test_image_${CI_JOB_ID} pip install ansible-core --break-system-packages; + + docker exec -i test_image_${CI_JOB_ID} update-alternatives --set iptables /usr/sbin/iptables-legacy; + + else + + echo " Not Debian 12": + + docker exec -i test_image_${CI_JOB_ID} pip install ansible-core; + + fi + + - docker exec -i test_image_${CI_JOB_ID} cat /etc/hosts + - docker exec -i test_image_${CI_JOB_ID} cat /etc/resolv.conf + - | # check if DNS working + docker exec -i test_image_${CI_JOB_ID} nslookup google.com + script: + - | # inside container? + docker exec -i test_image_${CI_JOB_ID} ls -l /collection; + docker exec -i test_image_${CI_JOB_ID} echo $PWD; + + - | # Show Network Interfaces + docker exec -i test_image_${CI_JOB_ID} ifconfig; + + - | # Install the collection + docker exec -i test_image_${CI_JOB_ID} bash -c 'ansible-galaxy collection install $(ls /collection/*.tar.gz)' + + - | # output ansible vars + docker exec -i test_image_${CI_JOB_ID} ansible -m setup localhost + + - | # run the collection + docker exec -i test_image_${CI_JOB_ID} \ + ${test_command} \ + --extra-vars "nfc_role_firewall_policy_input=ACCEPT" \ + --extra-vars "nfc_role_firewall_policy_forward=ACCEPT" \ + -vv + + - | # Create test.yaml + mkdir -p test_results; + cat < test_results/${test_image}.json + { + "$( echo ${test_image} | sed -e 's/\./_/')": "passed" + } + + EOF + + after_script: + - | # Create test.yaml if not exists + if [ ! -f test_results/${test_image}.json ]; then + + echo "[TRACE] Test has failed" + + mkdir -p test_results; + + cat < test_results/${test_image}.json + { + "$( echo ${test_image} | sed -e 's/\./_/')": "fail" + } + + EOF + + fi + + - | # Run trace script for debugging + chmod +x ./.gitlab/integration_test_trace.sh; + + ./.gitlab/integration_test_trace.sh; + + artifacts: + untracked: false + paths: + - ansible.log + - test_results/* + when: always + + rules: + + - if: $CI_COMMIT_TAG + allow_failure: true + when: on_success + + - if: "$CI_COMMIT_AUTHOR =='nfc_bot '" + when: never + + - if: # Occur on merge + $CI_COMMIT_BRANCH == "development" + && + $CI_PIPELINE_SOURCE == "push" + allow_failure: true + when: always + + - if: + $CI_COMMIT_BRANCH != "development" + && + $CI_COMMIT_BRANCH != "master" + && + $CI_PIPELINE_SOURCE == "push" + allow_failure: true + when: always + + - when: never + + + +Playbook - Install: + extends: .integration_test + parallel: + matrix: + - test_image: debian-11 + test_command: ansible-playbook nofusscomputing.kubernetes.install + - test_image: debian-12 + test_command: ansible-playbook nofusscomputing.kubernetes.install + - test_image: ubuntu-20.04 + test_command: ansible-playbook nofusscomputing.kubernetes.install + - test_image: ubuntu-22.04 + test_command: ansible-playbook nofusscomputing.kubernetes.install + + + +test_results: + stage: test + + extends: .ansible_playbook + + variables: + ansible_playbook: .gitlab/test_results.yaml + ANSIBLE_PLAYBOOK_DIR: $CI_PROJECT_DIR + + needs: + - Playbook - Install + + artifacts: + untracked: false + when: always + access: all + expire_in: "3 days" + paths: + - test_results.json + + rules: + + - if: $CI_COMMIT_TAG + when: on_success + + - if: "$CI_COMMIT_AUTHOR =='nfc_bot '" + when: never + + - if: # Occur on merge + $CI_COMMIT_BRANCH == "development" + && + $CI_PIPELINE_SOURCE == "push" + when: always + + - if: + $CI_COMMIT_BRANCH != "development" + && + $CI_COMMIT_BRANCH != "master" + && + $CI_PIPELINE_SOURCE == "push" + when: always + + - when: never \ No newline at end of file diff --git a/.gitlab/integration_test_trace.sh b/.gitlab/integration_test_trace.sh new file mode 100644 index 0000000..8fa485f --- /dev/null +++ b/.gitlab/integration_test_trace.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# colour ref: https://stackoverflow.com/a/28938235 + +NC='\033[0m' # Text Reset + +# Regular Colors +Black='\033[0;30m' # Black +Red='\033[0;31m' # Red +Green='\033[0;32m' # Green +Yellow='\033[0;33m' # Yellow +Blue='\033[0;34m' # Blue +Purple='\033[0;35m' # Purple +Cyan='\033[0;36m' # Cyan + + +cmd() { + + echo -e "${Yellow}[TRACE] ${Green}executing ${Cyan}'$1'${NC}" + + docker exec -i test_image_${CI_JOB_ID} $1 || true + +} + + +cmd "journalctl -xeu netfilter-persistent.service"; + +cmd "journalctl -xeu iptables.service" + +cmd "journalctl -xeu k3s.service" + +cmd "systemctl status netfilter-persistent.service" + +cmd "systemctl status iptables.servic" + +cmd "systemctl status k3s.service" + +cmd "kubectl get po -A -o wide" + +cmd "kubectl get no -o wide" + +cmd "iptables -nvL --line-numbers" diff --git a/.gitlab/test_results.yaml b/.gitlab/test_results.yaml new file mode 100644 index 0000000..fe9441a --- /dev/null +++ b/.gitlab/test_results.yaml @@ -0,0 +1,19 @@ +--- + +- name: Create Test Results File + hosts: localhost + gather_facts: false + + + tasks: + + + - name: Load Test Results + ansible.builtin.include_vars: + dir: ../test_results + name: test_results + + - name: Create Results file + ansible.builtin.copy: + content: "{{ (test_results) | to_nice_json }}" + dest: ../test_results.json diff --git a/.vscode/settings.json b/.vscode/settings.json index 458df7b..c383ad8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,7 +7,8 @@ ], "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/vars.json": [ "roles/nfc_kubernetes/variables/**.yaml" - ] + ], + "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/playbook": ".gitlab/test_results.yaml" }, "gitlab.aiAssistedCodeSuggestions.enabled": false, "gitlab.duoChat.enabled": false, diff --git a/README.md b/README.md index 0c741fa..4cb3f8c 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,17 @@ This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/a ![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fmaster%2F.cz.yaml) +![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814) + + ---- **Development Branch** ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fdevelopment%2F.cz.yaml) +![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814) + ----
diff --git a/docs/projects/ansible/collection/kubernetes/index.md b/docs/projects/ansible/collection/kubernetes/index.md index 83f8100..534a6c4 100644 --- a/docs/projects/ansible/collection/kubernetes/index.md +++ b/docs/projects/ansible/collection/kubernetes/index.md @@ -13,6 +13,8 @@ about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernete ![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) +![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814) + [![Downloads](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fkubernetes%2F&query=%24.download_count&style=plastic&logo=ansible&logoColor=white&label=Galaxy%20Downloads&labelColor=black&color=cyan)](https://galaxy.ansible.com/ui/repo/published/nofusscomputing/kubernetes/) diff --git a/roles/nfc_kubernetes/tasks/install.yaml b/roles/nfc_kubernetes/tasks/install.yaml index c7808a0..98fb182 100644 --- a/roles/nfc_kubernetes/tasks/install.yaml +++ b/roles/nfc_kubernetes/tasks/install.yaml @@ -18,6 +18,15 @@ not ansible_check_mode +- name: Testing Env Variables + ansible.builtin.set_fact: + ansible_default_ipv4: { + "address": "127.0.0.1" + } + when: > + lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != '' + + - name: Check Machine Architecture ansible.builtin.set_fact: nfc_kubernetes_install_architectures: "{{ nfc_kubernetes_install_architectures | default({}) | combine({ansible_architecture: ''}) }}" diff --git a/roles/nfc_kubernetes/tasks/k3s/install.yaml b/roles/nfc_kubernetes/tasks/k3s/install.yaml index 12fc4e4..ba1c41b 100644 --- a/roles/nfc_kubernetes/tasks/k3s/install.yaml +++ b/roles/nfc_kubernetes/tasks/k3s/install.yaml @@ -32,6 +32,7 @@ loop_var: package vars: packages: + - wget - curl - iptables - jq @@ -52,14 +53,29 @@ - install -- name: Disable swap - ansible.builtin.command: - cmd: swapoff -a - changed_when: false - when: - - ansible_os_family == 'Debian' - tags: - - install +- name: Testing Environment try/catch + block: + + + - name: Disable swap + ansible.builtin.command: + cmd: swapoff -a + changed_when: false + when: + - ansible_os_family == 'Debian' + tags: + - install + + + rescue: + + - name: Check if inside Gitlab CI + ansible.builtin.assert: + that: + - lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != '' + success_msg: "Inside testing enviroment, 'Disable swap' error OK" + fail_msg: "You should figure out what went wrong" + - name: Check an armbian os system ansible.builtin.stat: @@ -226,6 +242,7 @@ {%- else -%} false {%- endif -%}"; + export running_version="{{ kubernetes_node.resources[0].status.nodeInfo.kubeletVersion | default('0') }}"; export correct_hash=$(wget -q https://github.com/k3s-io/k3s/releases/download/v