feat(test): add integration test. playbook install

!53

.gitignore

@@ -1,3 +1,5 @@
 artifacts/
 build/
+test_results/
+test_results.json
 *.tar.gz

.gitlab-ci.yml

@@ -9,6 +9,7 @@ variables:
 
 
 include:
+  - local: .gitlab/integration_test.gitlab-ci.yml
   - project: nofusscomputing/projects/gitlab-ci
     ref: development
     file:

.gitlab/integration_test.gitlab-ci.yml

@@ -0,0 +1,207 @@
+
+.integration_test:
+
+  stage: test
+
+  needs:
+    - "Build Collection"
+
+  image: 
+    name: nofusscomputing/docker-buildx-qemu:dev
+    pull_policy: always
+
+  variables:
+    DOCKER_HOST: tcp://docker:2375/
+    DOCKER_DRIVER: overlay2
+    # GIT_STRATEGY: none
+
+  services:
+    - name: docker:23-dind
+      entrypoint: ["env", "-u", "DOCKER_HOST"]
+      command: ["dockerd-entrypoint.sh"]
+  before_script:
+    - | # start test container
+      docker run -d \
+        --privileged \
+        -v ${PWD}:/workdir \
+        -v ${PWD}/artifacts/galaxy:/collection \
+        --workdir /workdir \
+        --rm \
+        --env "ANSIBLE_FORCE_COLOR=true" \
+        --env "CI_COMMIT_SHA=${CI_COMMIT_SHA}" \
+        --env "ANSIBLE_LOG_PATH=/workdir/ansible.log" \
+        --name test_image_${CI_JOB_ID} \
+        nofusscomputing/ansible-docker-os:dev-${test_image}
+
+    - | # enter test container
+      docker exec -i test_image_${CI_JOB_ID} ps aux
+    - docker ps
+    - docker exec -i test_image_${CI_JOB_ID} apt update
+    - docker exec -i test_image_${CI_JOB_ID} apt install -y --no-install-recommends python3-pip net-tools dnsutils iptables
+    - |
+      if [ "${test_image}" == 'debian-12' ]; then
+
+        echo "Debian 12":
+
+        docker exec -i test_image_${CI_JOB_ID} pip install ansible-core --break-system-packages;
+
+        docker exec -i test_image_${CI_JOB_ID} update-alternatives --set iptables /usr/sbin/iptables-legacy;
+
+      else
+
+        echo " Not Debian 12":
+
+        docker exec -i test_image_${CI_JOB_ID} pip install ansible-core;
+
+      fi
+
+    - docker exec -i test_image_${CI_JOB_ID} cat /etc/hosts
+    - docker exec -i test_image_${CI_JOB_ID} cat /etc/resolv.conf
+    - | # check if DNS working
+      docker exec -i test_image_${CI_JOB_ID} nslookup google.com
+  script:
+    - | # inside container?
+      docker exec -i test_image_${CI_JOB_ID} ls -l /collection;
+      docker exec -i test_image_${CI_JOB_ID} echo $PWD;
+
+    - | # Show Network Interfaces
+      docker exec -i test_image_${CI_JOB_ID} ifconfig;
+
+    - | # Install the collection
+      docker exec -i test_image_${CI_JOB_ID} bash -c 'ansible-galaxy collection install $(ls /collection/*.tar.gz)'
+
+    - | # output ansible vars
+      docker exec -i test_image_${CI_JOB_ID} ansible -m setup localhost
+
+    - | # run the collection
+      docker exec -i test_image_${CI_JOB_ID} \
+        ${test_command} \
+        --extra-vars "nfc_role_firewall_policy_input=ACCEPT" \
+        --extra-vars "nfc_role_firewall_policy_forward=ACCEPT" \
+        -vv
+
+    - | # Create test.yaml
+      mkdir -p test_results;
+      cat <<EOF > test_results/${test_image}.json
+      {
+        "$( echo ${test_image}  | sed -e 's/\./_/')": "passed"
+      }
+
+      EOF
+
+  after_script:
+    - | # Create test.yaml if not exists
+      if [ ! -f test_results/${test_image}.json ]; then
+
+      echo "[TRACE] Test has failed"
+
+      mkdir -p test_results;
+
+      cat <<EOF > test_results/${test_image}.json
+      {
+        "$( echo ${test_image}  | sed -e 's/\./_/')": "fail"
+      }
+
+      EOF
+
+      fi
+
+    - | # Run trace script for debugging
+      chmod +x ./.gitlab/integration_test_trace.sh;
+
+      ./.gitlab/integration_test_trace.sh;
+
+  artifacts:
+    untracked: false
+    paths:
+      - ansible.log
+      - test_results/*
+    when: always
+
+  rules:
+
+    - if: $CI_COMMIT_TAG
+      allow_failure: true
+      when: on_success
+
+    - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+      when: never
+
+    - if: # Occur on merge
+        $CI_COMMIT_BRANCH == "development"
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      allow_failure: true
+      when: always
+
+    - if:
+        $CI_COMMIT_BRANCH != "development"
+          &&
+        $CI_COMMIT_BRANCH != "master"
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      allow_failure: true
+      when: always
+
+    - when: never
+
+
+
+Playbook - Install:
+  extends: .integration_test
+  parallel:
+    matrix:
+      - test_image: debian-11
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+      - test_image: debian-12
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+      - test_image: ubuntu-20.04
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+      - test_image: ubuntu-22.04
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+
+
+
+test_results:
+  stage: test
+
+  extends: .ansible_playbook
+
+  variables:
+    ansible_playbook: .gitlab/test_results.yaml
+    ANSIBLE_PLAYBOOK_DIR: $CI_PROJECT_DIR
+
+  needs:
+    - Playbook - Install
+
+  artifacts:
+    untracked: false
+    when: always
+    access: all
+    expire_in: "3 days"
+    paths:
+      - test_results.json
+
+  rules:
+
+    - if: $CI_COMMIT_TAG
+      when: on_success
+
+    - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+      when: never
+
+    - if: # Occur on merge
+        $CI_COMMIT_BRANCH == "development"
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      when: always
+
+    - if:
+        $CI_COMMIT_BRANCH != "development"
+          &&
+        $CI_COMMIT_BRANCH != "master"
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      when: always
+
+    - when: never

.gitlab/integration_test_trace.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# colour ref: https://stackoverflow.com/a/28938235
+
+NC='\033[0m'       # Text Reset
+
+# Regular Colors
+Black='\033[0;30m'        # Black
+Red='\033[0;31m'          # Red
+Green='\033[0;32m'        # Green
+Yellow='\033[0;33m'       # Yellow
+Blue='\033[0;34m'         # Blue
+Purple='\033[0;35m'       # Purple
+Cyan='\033[0;36m'         # Cyan
+
+
+cmd() {
+
+  echo -e "${Yellow}[TRACE] ${Green}executing ${Cyan}'$1'${NC}"
+
+  docker exec -i test_image_${CI_JOB_ID} $1 || true
+
+}
+
+
+cmd "journalctl -xeu netfilter-persistent.service";
+
+cmd "journalctl -xeu iptables.service"
+
+cmd "journalctl -xeu k3s.service"
+
+cmd "systemctl status netfilter-persistent.service"
+
+cmd "systemctl status iptables.servic"
+
+cmd "systemctl status k3s.service"
+
+cmd "kubectl get po -A -o wide"
+
+cmd "kubectl get no -o wide"
+
+cmd "iptables -nvL --line-numbers"

.gitlab/test_results.yaml

@@ -0,0 +1,19 @@
+---
+
+- name: Create Test Results File
+  hosts: localhost
+  gather_facts: false
+
+
+  tasks:
+    
+
+    - name: Load Test Results
+      ansible.builtin.include_vars:
+        dir: ../test_results
+        name: test_results
+
+    - name: Create Results file
+      ansible.builtin.copy:
+        content: "{{ (test_results) | to_nice_json }}"
+        dest: ../test_results.json

.vscode/settings.json

@@ -7,7 +7,8 @@
         ],
         "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/vars.json": [
             "roles/nfc_kubernetes/variables/**.yaml"
-        ]
+        ],
+        "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/playbook": ".gitlab/test_results.yaml"
     },
     "gitlab.aiAssistedCodeSuggestions.enabled": false,
     "gitlab.duoChat.enabled": false,

README.md

@@ -29,12 +29,17 @@ This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/a
 
 ![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fmaster%2F.cz.yaml) 
 
+![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814)
+
+
 ----
 
 **Development Branch** 
 
 ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fdevelopment%2F.cz.yaml)
 
+![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814)
+
 ----
 <br>
 

docs/projects/ansible/collection/kubernetes/index.md

@@ -13,6 +13,8 @@ about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernete
 
 ![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic)
 
+![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814)
+
 
 [![Downloads](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fkubernetes%2F&query=%24.download_count&style=plastic&logo=ansible&logoColor=white&label=Galaxy%20Downloads&labelColor=black&color=cyan)](https://galaxy.ansible.com/ui/repo/published/nofusscomputing/kubernetes/)
 

roles/nfc_kubernetes/tasks/install.yaml

@@ -18,6 +18,15 @@
     not ansible_check_mode
 
 
+- name: Testing Env Variables
+  ansible.builtin.set_fact:
+    ansible_default_ipv4: {
+      "address": "127.0.0.1"
+    }
+  when: >
+    lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != ''
+
+
 - name: Check Machine Architecture
   ansible.builtin.set_fact:
     nfc_kubernetes_install_architectures: "{{ nfc_kubernetes_install_architectures | default({}) | combine({ansible_architecture: ''}) }}"

roles/nfc_kubernetes/tasks/k3s/install.yaml

@@ -32,6 +32,7 @@
     loop_var: package
   vars:
     packages:
+      - wget
       - curl
       - iptables
       - jq
@@ -52,6 +53,10 @@
     - install
 
 
+- name: Testing Environment try/catch
+  block:
+
+
     - name: Disable swap
       ansible.builtin.command:
         cmd: swapoff -a
@@ -61,6 +66,17 @@
       tags:
         - install
 
+
+  rescue:
+
+    - name: Check if inside Gitlab CI
+      ansible.builtin.assert:
+        that:
+          - lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != ''
+        success_msg: "Inside testing enviroment, 'Disable swap' error OK"
+        fail_msg: "You should figure out what went wrong"
+
+
 - name: Check an armbian os system
   ansible.builtin.stat:
     path: /etc/default/armbian-zram-config
@@ -226,6 +242,7 @@
       {%- else -%}
         false
       {%- endif -%}";
+
       export running_version="{{ kubernetes_node.resources[0].status.nodeInfo.kubeletVersion | default('0') }}";
 
       export correct_hash=$(wget -q https://github.com/k3s-io/k3s/releases/download/v