diff --git a/.gitignore b/.gitignore index 5fa197d..953f083 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ artifacts/ build/ +test_results/ +test_results.json *.tar.gz \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab3349b..5f9ffb3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ variables: include: + - local: .gitlab/integration_test.gitlab-ci.yml - project: nofusscomputing/projects/gitlab-ci ref: development file: diff --git a/.gitlab/integration_test.gitlab-ci.yml b/.gitlab/integration_test.gitlab-ci.yml new file mode 100644 index 0000000..2eb1c44 --- /dev/null +++ b/.gitlab/integration_test.gitlab-ci.yml @@ -0,0 +1,207 @@ + +.integration_test: + + stage: test + + needs: + - "Build Collection" + + image: + name: nofusscomputing/docker-buildx-qemu:dev + pull_policy: always + + variables: + DOCKER_HOST: tcp://docker:2375/ + DOCKER_DRIVER: overlay2 + # GIT_STRATEGY: none + + services: + - name: docker:23-dind + entrypoint: ["env", "-u", "DOCKER_HOST"] + command: ["dockerd-entrypoint.sh"] + before_script: + - | # start test container + docker run -d \ + --privileged \ + -v ${PWD}:/workdir \ + -v ${PWD}/artifacts/galaxy:/collection \ + --workdir /workdir \ + --rm \ + --env "ANSIBLE_FORCE_COLOR=true" \ + --env "CI_COMMIT_SHA=${CI_COMMIT_SHA}" \ + --env "ANSIBLE_LOG_PATH=/workdir/ansible.log" \ + --name test_image_${CI_JOB_ID} \ + nofusscomputing/ansible-docker-os:dev-${test_image} + + - | # enter test container + docker exec -i test_image_${CI_JOB_ID} ps aux + - docker ps + - docker exec -i test_image_${CI_JOB_ID} apt update + - docker exec -i test_image_${CI_JOB_ID} apt install -y --no-install-recommends python3-pip net-tools dnsutils iptables + - | + if [ "${test_image}" == 'debian-12' ]; then + + echo "Debian 12": + + docker exec -i test_image_${CI_JOB_ID} pip install ansible-core --break-system-packages; + + docker exec -i test_image_${CI_JOB_ID} update-alternatives --set iptables /usr/sbin/iptables-legacy; + + else + + echo " Not Debian 12": + + docker exec -i test_image_${CI_JOB_ID} pip install ansible-core; + + fi + + - docker exec -i test_image_${CI_JOB_ID} cat /etc/hosts + - docker exec -i test_image_${CI_JOB_ID} cat /etc/resolv.conf + - | # check if DNS working + docker exec -i test_image_${CI_JOB_ID} nslookup google.com + script: + - | # inside container? + docker exec -i test_image_${CI_JOB_ID} ls -l /collection; + docker exec -i test_image_${CI_JOB_ID} echo $PWD; + + - | # Show Network Interfaces + docker exec -i test_image_${CI_JOB_ID} ifconfig; + + - | # Install the collection + docker exec -i test_image_${CI_JOB_ID} bash -c 'ansible-galaxy collection install $(ls /collection/*.tar.gz)' + + - | # output ansible vars + docker exec -i test_image_${CI_JOB_ID} ansible -m setup localhost + + - | # run the collection + docker exec -i test_image_${CI_JOB_ID} \ + ${test_command} \ + --extra-vars "nfc_role_firewall_policy_input=ACCEPT" \ + --extra-vars "nfc_role_firewall_policy_forward=ACCEPT" \ + -vv + + - | # Create test.yaml + mkdir -p test_results; + cat < test_results/${test_image}.json + { + "$( echo ${test_image} | sed -e 's/\./_/')": "passed" + } + + EOF + + after_script: + - | # Create test.yaml if not exists + if [ ! -f test_results/${test_image}.json ]; then + + echo "[TRACE] Test has failed" + + mkdir -p test_results; + + cat < test_results/${test_image}.json + { + "$( echo ${test_image} | sed -e 's/\./_/')": "fail" + } + + EOF + + fi + + - | # Run trace script for debugging + chmod +x ./.gitlab/integration_test_trace.sh; + + ./.gitlab/integration_test_trace.sh; + + artifacts: + untracked: false + paths: + - ansible.log + - test_results/* + when: always + + rules: + + - if: $CI_COMMIT_TAG + allow_failure: true + when: on_success + + - if: "$CI_COMMIT_AUTHOR =='nfc_bot '" + when: never + + - if: # Occur on merge + $CI_COMMIT_BRANCH == "development" + && + $CI_PIPELINE_SOURCE == "push" + allow_failure: true + when: always + + - if: + $CI_COMMIT_BRANCH != "development" + && + $CI_COMMIT_BRANCH != "master" + && + $CI_PIPELINE_SOURCE == "push" + allow_failure: true + when: always + + - when: never + + + +Playbook - Install: + extends: .integration_test + parallel: + matrix: + - test_image: debian-11 + test_command: ansible-playbook nofusscomputing.kubernetes.install + - test_image: debian-12 + test_command: ansible-playbook nofusscomputing.kubernetes.install + - test_image: ubuntu-20.04 + test_command: ansible-playbook nofusscomputing.kubernetes.install + - test_image: ubuntu-22.04 + test_command: ansible-playbook nofusscomputing.kubernetes.install + + + +test_results: + stage: test + + extends: .ansible_playbook + + variables: + ansible_playbook: .gitlab/test_results.yaml + ANSIBLE_PLAYBOOK_DIR: $CI_PROJECT_DIR + + needs: + - Playbook - Install + + artifacts: + untracked: false + when: always + access: all + expire_in: "3 days" + paths: + - test_results.json + + rules: + + - if: $CI_COMMIT_TAG + when: on_success + + - if: "$CI_COMMIT_AUTHOR =='nfc_bot '" + when: never + + - if: # Occur on merge + $CI_COMMIT_BRANCH == "development" + && + $CI_PIPELINE_SOURCE == "push" + when: always + + - if: + $CI_COMMIT_BRANCH != "development" + && + $CI_COMMIT_BRANCH != "master" + && + $CI_PIPELINE_SOURCE == "push" + when: always + + - when: never \ No newline at end of file diff --git a/.gitlab/integration_test_trace.sh b/.gitlab/integration_test_trace.sh new file mode 100644 index 0000000..8fa485f --- /dev/null +++ b/.gitlab/integration_test_trace.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# colour ref: https://stackoverflow.com/a/28938235 + +NC='\033[0m' # Text Reset + +# Regular Colors +Black='\033[0;30m' # Black +Red='\033[0;31m' # Red +Green='\033[0;32m' # Green +Yellow='\033[0;33m' # Yellow +Blue='\033[0;34m' # Blue +Purple='\033[0;35m' # Purple +Cyan='\033[0;36m' # Cyan + + +cmd() { + + echo -e "${Yellow}[TRACE] ${Green}executing ${Cyan}'$1'${NC}" + + docker exec -i test_image_${CI_JOB_ID} $1 || true + +} + + +cmd "journalctl -xeu netfilter-persistent.service"; + +cmd "journalctl -xeu iptables.service" + +cmd "journalctl -xeu k3s.service" + +cmd "systemctl status netfilter-persistent.service" + +cmd "systemctl status iptables.servic" + +cmd "systemctl status k3s.service" + +cmd "kubectl get po -A -o wide" + +cmd "kubectl get no -o wide" + +cmd "iptables -nvL --line-numbers" diff --git a/.gitlab/test_results.yaml b/.gitlab/test_results.yaml new file mode 100644 index 0000000..fe9441a --- /dev/null +++ b/.gitlab/test_results.yaml @@ -0,0 +1,19 @@ +--- + +- name: Create Test Results File + hosts: localhost + gather_facts: false + + + tasks: + + + - name: Load Test Results + ansible.builtin.include_vars: + dir: ../test_results + name: test_results + + - name: Create Results file + ansible.builtin.copy: + content: "{{ (test_results) | to_nice_json }}" + dest: ../test_results.json diff --git a/.vscode/settings.json b/.vscode/settings.json index 56d55e7..c383ad8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,13 +1,14 @@ { "yaml.schemas": { "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/tasks": [ - "roles/nfc_firewall/tasks/*.yaml", - "roles/nfc_firewall/tasks/*/*.yaml", - "roles/nfc_firewall/tasks/*/*/*.yaml" + "roles/nfc_kubernetes/tasks/*.yaml", + "roles/nfc_kubernetes/tasks/*/*.yaml", + "roles/nfc_kubernetes/tasks/*/*/*.yaml" ], "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/vars.json": [ "roles/nfc_kubernetes/variables/**.yaml" - ] + ], + "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/playbook": ".gitlab/test_results.yaml" }, "gitlab.aiAssistedCodeSuggestions.enabled": false, "gitlab.duoChat.enabled": false, diff --git a/README.md b/README.md index 0c741fa..4cb3f8c 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,17 @@ This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/a ![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fmaster%2F.cz.yaml) +![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814) + + ---- **Development Branch** ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fdevelopment%2F.cz.yaml) +![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814) + ----
diff --git a/docs/projects/ansible/collection/kubernetes/index.md b/docs/projects/ansible/collection/kubernetes/index.md index 5126ba7..534a6c4 100644 --- a/docs/projects/ansible/collection/kubernetes/index.md +++ b/docs/projects/ansible/collection/kubernetes/index.md @@ -13,6 +13,8 @@ about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernete ![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) +![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814) + [![Downloads](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fkubernetes%2F&query=%24.download_count&style=plastic&logo=ansible&logoColor=white&label=Galaxy%20Downloads&labelColor=black&color=cyan)](https://galaxy.ansible.com/ui/repo/published/nofusscomputing/kubernetes/) @@ -29,14 +31,20 @@ To install this collection use `ansible-galaxy collection install nofusscomputin ## Features -Most of the features of this collection are from the included role `nfc_kubernetes`, please [view its page for feature details](roles/nfc_kubernetes/index.md). +- Install k3s cluster. Both Single and multi-node clusters + +- Configure the cluster + +- Upgrade a cluster + +For a more detailed list of featured checkout the roles [documentation](roles/nfc_kubernetes/index.md). ## Using this collection This collection has been designed to be a complete and self-contained management tool for a K3s kubernetes cluster. -## K3s Kubernetes Installation +## Cluster Installation By default the install playbook will install to localhost. @@ -64,4 +72,12 @@ The install playbook has a dynamic `hosts` key. This has been done to specifical For the available variables please view the [nfc_kubernetes role docs](roles/nfc_kubernetes/index.md#default-variables) +## Cluster Upgrade +[In place cluster upgrades](https://docs.k3s.io/upgrades/manual#upgrade-k3s-using-the-binary) is the method used to conduct the cluster upgrades. The logic for the upgrades first confirms that K3s is installed and that the local binary and running k3s version are the desired versions. If they are not, they will be updated to the desired version. On completion of this the node has its `k3s` service restarted which completes the upgrade process. + +!!! info + If an upgrade occurs, no other task within the play will run. This is by design. if you have further tasks to be run in addition to the upgrade, run the play again. + +!!! danger + not following the [Kubernetes version skew policy](https://kubernetes.io/releases/version-skew-policy/) when upgrading your cluster may break your cluster. diff --git a/docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md b/docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md index 3fe6509..2b766d3 100644 --- a/docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md +++ b/docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md @@ -70,6 +70,8 @@ This Ansible role is designed to deploy a K3s Kubernetes cluster. Without adding - Install the Helm Binary +- Upgrade cluster + ## Role Workflow diff --git a/galaxy.yml b/galaxy.yml index ed9263e..baf204a 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -44,9 +44,9 @@ tags: # L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version # range specifiers can be set and are separated by ',' dependencies: - ansible.posix: '1.5.4' - kubernetes.core: '3.0.0' - nofusscomputing.firewall: '1.1.0' + ansible.posix: '>=1.5.4' + kubernetes.core: '>=3.0.0' + nofusscomputing.firewall: '>=1.1.0' # The URL of the originating SCM repository diff --git a/roles/nfc_kubernetes/tasks/helm/main.yaml b/roles/nfc_kubernetes/tasks/helm/main.yaml index c1c4887..cb55052 100644 --- a/roles/nfc_kubernetes/tasks/helm/main.yaml +++ b/roles/nfc_kubernetes/tasks/helm/main.yaml @@ -5,6 +5,8 @@ url: https://baltocdn.com/helm/signing.asc dest: /usr/share/keyrings/helm.asc mode: 740 + delay: 10 + retries: 3 - name: Add Helm Repository diff --git a/roles/nfc_kubernetes/tasks/install.yaml b/roles/nfc_kubernetes/tasks/install.yaml index 18857ff..98fb182 100644 --- a/roles/nfc_kubernetes/tasks/install.yaml +++ b/roles/nfc_kubernetes/tasks/install.yaml @@ -18,6 +18,15 @@ not ansible_check_mode +- name: Testing Env Variables + ansible.builtin.set_fact: + ansible_default_ipv4: { + "address": "127.0.0.1" + } + when: > + lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != '' + + - name: Check Machine Architecture ansible.builtin.set_fact: nfc_kubernetes_install_architectures: "{{ nfc_kubernetes_install_architectures | default({}) | combine({ansible_architecture: ''}) }}" @@ -35,6 +44,19 @@ nfc_role_kubernetes_configure_firewall +- name: Install required software + ansible.builtin.apt: + name: python3-pip + install_recommends: false + state: present + when: > + install_kubernetes | default(true) | bool + and + not kubernetes_installed | default(false) | bool + tags: + - always + + - name: K3s Install ansible.builtin.include_tasks: file: k3s/install.yaml @@ -59,6 +81,8 @@ install_kubernetes | default(true) | bool and kubernetes_installed | default(false) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool tags: - always @@ -75,6 +99,8 @@ kubernetes_config.kube_virt.enabled | default(nfc_role_kubernetes_install_kubevirt) and inventory_hostname in kubernetes_config.kube_virt.nodes | default([ inventory_hostname ]) | list + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool tags: - always @@ -91,5 +117,7 @@ kubernetes_config.helm.enabled | default(nfc_role_kubernetes_install_helm) and nfc_role_kubernetes_master + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool tags: - always diff --git a/roles/nfc_kubernetes/tasks/k3s/install.yaml b/roles/nfc_kubernetes/tasks/k3s/install.yaml index 88a74ae..ba1c41b 100644 --- a/roles/nfc_kubernetes/tasks/k3s/install.yaml +++ b/roles/nfc_kubernetes/tasks/k3s/install.yaml @@ -1,5 +1,16 @@ --- +- name: Install required python modules + ansible.builtin.pip: + name: "{{ item }}" + state: present + loop: "{{ pip_packages }}" + vars: + pip_packages: + - kubernetes>=12.0.0 + - PyYAML>=3.11 + + - name: Check for calico deployment manifest ansible.builtin.stat: name: /var/lib/rancher/k3s/server/manifests/calico.yaml @@ -21,6 +32,7 @@ loop_var: package vars: packages: + - wget - curl - iptables - jq @@ -41,14 +53,29 @@ - install -- name: Disable swap - ansible.builtin.command: - cmd: swapoff -a - changed_when: false - when: - - ansible_os_family == 'Debian' - tags: - - install +- name: Testing Environment try/catch + block: + + + - name: Disable swap + ansible.builtin.command: + cmd: swapoff -a + changed_when: false + when: + - ansible_os_family == 'Debian' + tags: + - install + + + rescue: + + - name: Check if inside Gitlab CI + ansible.builtin.assert: + that: + - lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != '' + success_msg: "Inside testing enviroment, 'Disable swap' error OK" + fail_msg: "You should figure out what went wrong" + - name: Check an armbian os system ansible.builtin.stat: @@ -138,28 +165,218 @@ when: directory_network_manager_metadata.stat.exists -- name: Check if K3s Installed +- name: File Metadata - k3s binary + ansible.builtin.stat: + checksum_algorithm: sha256 + name: /usr/local/bin/k3s + register: metadata_file_k3s_existing_binary + + +- name: File Metadata - k3s[-agent].service + ansible.builtin.stat: + checksum_algorithm: sha256 + name: |- + /etc/systemd/system/k3s + {%- if not nfc_role_kubernetes_master | default(false) | bool -%} + -agent + {%- endif -%} + .service + register: metadata_file_k3s_service + + +- name: Directory Metadata - /etc/rancher/k3s/k3s.yaml + ansible.builtin.stat: + name: /etc/rancher/k3s/k3s.yaml + register: metadata_dir_etc_k3s + + +- name: File Metadata - /var/lib/rancher/k3s/server/token + ansible.builtin.stat: + checksum_algorithm: sha256 + name: /var/lib/rancher/k3s/server/token + register: metadata_file_var_k3s_token + + +- name: Config Link ansible.builtin.shell: - cmd: | - if [[ $(service k3s status) ]]; then exit 0; else exit 1; fi - executable: /bin/bash - changed_when: false - failed_when: false - register: k3s_installed + cmd: > + ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config + executable: bash + creates: ~/.kube/config when: > nfc_role_kubernetes_master | default(false) | bool + and + metadata_dir_etc_k3s.stat.exists | default(false) | bool -- name: Check if K3s Installed +- name: Fetch Kubernetes Node Object + kubernetes.core.k8s_info: + kind: Node + name: "{{ inventory_hostname }}" + register: kubernetes_node + when: > + metadata_file_k3s_existing_binary.stat.exists | default(false) | bool + and + metadata_file_k3s_service.stat.exists | default(false) | bool + and + metadata_dir_etc_k3s.stat.exists | default(false) | bool + and + metadata_file_var_k3s_token.stat.exists | default(false) | bool + + +- name: Fetch Installed K3s Metadata ansible.builtin.shell: cmd: | - if [[ $(service k3s-agent status) ]]; then exit 0; else exit 1; fi + export installed_version=$(k3s --version | grep k3s | awk '{print $3}'); + export installed=" + {%- if + metadata_file_k3s_existing_binary.stat.exists | default(false) | bool + and + metadata_file_k3s_service.stat.exists | default(false) | bool + and + metadata_dir_etc_k3s.stat.exists | default(false) | bool + and + metadata_file_var_k3s_token.stat.exists | default(false) | bool + -%} + true + {%- else -%} + false + {%- endif -%}"; + + export running_version="{{ kubernetes_node.resources[0].status.nodeInfo.kubeletVersion | default('0') }}"; + + export correct_hash=$(wget -q https://github.com/k3s-io/k3s/releases/download/v + {{-KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}} + /sha256sum- + {%- if ansible_architecture | lower == 'x86_64' -%} + amd64 + {%- elif ansible_architecture | lower == 'aarch64' -%} + arm64 + {%- endif %}.txt -O - | grep -v 'images' | awk '{print $1}'); + + cat < - not nfc_role_kubernetes_worker | default(false) | bool + file_cached_k3s_binary.stat.checksum | default('0') != node_k3s.desired_hash + + +- name: Download K3s Binary + ansible.builtin.uri: + url: |- + https://github.com/k3s-io/k3s/releases/download/ + {{- node_k3s.desired_version | urlencode -}} + /k3s + {%- if cpu_arch.key == 'aarch64' -%} + -arm64 + {%- endif %} + method: GET + return_content: false + status_code: + - 200 + - 304 + dest: "/tmp/k3s.{{ cpu_arch.key }}" + mode: "744" + delay: 10 + retries: 3 + register: k3s_download_files + delegate_to: localhost + failed_when: > + (lookup('ansible.builtin.file', '/tmp/k3s.' + cpu_arch.key) | hash('sha256') | string) != node_k3s.desired_hash + and + ( + k3s_download_files.status | int != 200 + or + k3s_download_files.status | int != 304 + ) + run_once: true + when: ansible_os_family == 'Debian' + loop: "{{ nfc_kubernetes_install_architectures | dict2items }}" + loop_control: + loop_var: cpu_arch + vars: + ansible_connection: local + + +- name: Copy K3s binary to Host + ansible.builtin.copy: + src: "/tmp/k3s.{{ ansible_architecture }}" + dest: "/usr/local/bin/k3s" + mode: '741' + owner: root + group: root + register: k3s_binary_copy + when: > + node_k3s.current_hash != node_k3s.desired_hash + + +- name: K3s Binary Upgrade + ansible.builtin.service: + name: |- + {%- if nfc_role_kubernetes_master | default(false) | bool -%} + k3s + {%- else -%} + k3s-agent + {%- endif %} + state: restarted + register: k3s_upgrade_service_restart + when: > + ( + k3s_binary_copy.changed | default(false) | bool + and + node_k3s.installed | default(false) | bool + ) + or + ( + node_k3s.running_version != node_k3s.desired_version + and + node_k3s.installed | default(false) | bool + ) + + +- name: Create Fact - cluster_upgraded + ansible.builtin.set_fact: + nfc_role_kubernetes_cluster_upgraded: true + cacheable: true + when: > + k3s_upgrade_service_restart.changed | default(false) | bool - name: Download Install Scripts @@ -173,6 +390,8 @@ dest: "{{ item.dest }}" mode: "744" changed_when: false + delay: 10 + retries: 3 register: k3s_download_script delegate_to: localhost run_once: true @@ -181,6 +400,8 @@ ansible_os_family == 'Debian' and item.when | default(true) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool loop: "{{ download_files }}" vars: ansible_connection: local @@ -192,61 +413,6 @@ when: "{{ nfc_role_kubernetes_install_olm }}" -- name: Download K3s Binary - ansible.builtin.uri: - url: |- - https://github.com/k3s-io/k3s/releases/download/v - {{- KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}} - /k3s - {%- if cpu_arch.key == 'aarch64' -%} - -arm64 - {%- endif %} - method: GET - return_content: false - status_code: - - 200 - - 304 - dest: "/tmp/k3s.{{ cpu_arch.key }}" - mode: "744" - changed_when: false - register: k3s_download_files - delegate_to: localhost - run_once: true - # no_log: true - when: ansible_os_family == 'Debian' - loop: "{{ nfc_kubernetes_install_architectures | dict2items }}" - loop_control: - loop_var: cpu_arch - vars: - ansible_connection: local - - -- name: "[TRACE] Downloaded File SHA256" - ansible.builtin.set_fact: - hash_sha256_k3s_downloaded_binary: "{{ lookup('ansible.builtin.file', '/tmp/k3s.' + cpu_arch.key) | hash('sha256') | string }}" - delegate_to: localhost - loop: "{{ nfc_kubernetes_install_architectures | dict2items }}" - loop_control: - loop_var: cpu_arch - - -- name: Existing k3s File hash - ansible.builtin.stat: - checksum_algorithm: sha256 - name: /usr/local/bin/k3s - register: hash_sha256_k3s_existing_binary - - -- name: Copy K3s binary to Host - ansible.builtin.copy: - src: "/tmp/k3s.{{ ansible_architecture }}" - dest: "/usr/local/bin/k3s" - mode: '741' - owner: root - group: root - when: hash_sha256_k3s_existing_binary.stat.checksum | default('0') != hash_sha256_k3s_downloaded_binary - - - name: Copy install scripts to Host ansible.builtin.copy: src: "{{ item.path }}" @@ -263,6 +429,8 @@ when: "{{ nfc_role_kubernetes_install_olm }}" when: > item.when | default(true) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Required Initial config files @@ -276,7 +444,8 @@ loop: "{{ k3s.files }}" when: > item.when | default(true) | bool - # kubernetes_config.cluster.prime.name == inventory_hostname + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Copy Intial required templates @@ -291,6 +460,8 @@ diff: true when: > item.when | default(true) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool vars: templates_to_apply: - src: k3s-config.yaml.j2 @@ -308,7 +479,7 @@ and file_calico_yaml_metadata.stat.exists and - k3s_installed.rc == 0 + not node_k3s.installed | bool ) or 'calico_manifest' in ansible_run_tags @@ -335,19 +506,23 @@ ansible.builtin.command: cmd: update-alternatives --set iptables /usr/sbin/iptables-legacy changed_when: false + when: > + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install K3s (prime master) ansible.builtin.shell: cmd: | INSTALL_K3S_SKIP_DOWNLOAD=true \ - INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \ + INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \ /tmp/install.sh {% if nfc_role_kubernetes_etcd_enabled %}--cluster-init{% endif %} changed_when: false when: > kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and - k3s_installed.rc == 1 + not node_k3s.installed | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install Calico Operator @@ -370,6 +545,8 @@ 'calico_manifest' not in ansible_run_tags and kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install MetalLB Operator @@ -389,6 +566,8 @@ nfc_kubernetes_enable_metallb | default(false) | bool and kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Wait for kubernetes prime to be ready @@ -413,6 +592,20 @@ kubernetes_ready_check.rc != 0 changed_when: false failed_when: kubernetes_ready_check.rc != 0 + when: > + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool + + +- name: Config Link + ansible.builtin.shell: + cmd: > + ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config + executable: bash + creates: ~/.kube/config + when: > + nfc_role_kubernetes_master | default(false) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install olm @@ -429,6 +622,8 @@ kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and nfc_role_kubernetes_install_olm | default(false) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Uninstall OLM @@ -457,6 +652,8 @@ kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and 'olm_uninstall' in ansible_run_tags + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Enable Cluster Encryption @@ -469,6 +666,8 @@ and kubernetes_config.cluster.networking.encrypt | default(false) | bool and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool + and ( 'calico_manifest' in ansible_run_tags or @@ -487,6 +686,8 @@ run_once: true register: k3s_join_token no_log: true # Value is sensitive + when: > + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Create Token fact @@ -495,6 +696,8 @@ delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}" run_once: true no_log: true # Value is sensitive + when: > + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install K3s (master nodes) @@ -502,7 +705,7 @@ cmd: | INSTALL_K3S_EXEC="server" \ INSTALL_K3S_SKIP_DOWNLOAD=true \ - INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \ + INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \ K3S_TOKEN="{{ k3s_join_token }}" \ /tmp/install.sh executable: /bin/bash @@ -512,7 +715,9 @@ and not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and - k3s_installed.rc == 1 + not node_k3s.installed | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install K3s (worker nodes) @@ -521,7 +726,7 @@ set -o pipefail INSTALL_K3S_EXEC="agent" \ INSTALL_K3S_SKIP_DOWNLOAD=true \ - INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \ + INSTALL_K3S_VERSION="v{{ node_k3s.desired_version }}" \ K3S_TOKEN="{{ k3s_join_token }}" \ K3S_URL="https://{{ hostvars[kubernetes_config.cluster.prime.name | default(inventory_hostname)].ansible_host }}:6443" \ /tmp/install.sh - @@ -532,7 +737,9 @@ and not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and - k3s_installed.rc == 1 + not node_k3s.installed | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Set Kubernetes Final Install Fact