From 6c4616873ec636846a678c4858639fb34f7b5ac0 Mon Sep 17 00:00:00 2001 From: Jon Date: Fri, 29 Mar 2024 17:17:46 +0930 Subject: [PATCH 1/6] feat: support upgrading cluster In place binary upgrades was chosen as its just a matter of changing binary and restarting the service !53 closes #14 --- .../ansible/collection/kubernetes/index.md | 15 +- .../kubernetes/roles/nfc_kubernetes/index.md | 2 + roles/nfc_kubernetes/tasks/install.yaml | 13 + roles/nfc_kubernetes/tasks/k3s/install.yaml | 299 +++++++++++++----- 4 files changed, 254 insertions(+), 75 deletions(-) diff --git a/docs/projects/ansible/collection/kubernetes/index.md b/docs/projects/ansible/collection/kubernetes/index.md index 5126ba7..7fa5489 100644 --- a/docs/projects/ansible/collection/kubernetes/index.md +++ b/docs/projects/ansible/collection/kubernetes/index.md @@ -29,14 +29,20 @@ To install this collection use `ansible-galaxy collection install nofusscomputin ## Features -Most of the features of this collection are from the included role `nfc_kubernetes`, please [view its page for feature details](roles/nfc_kubernetes/index.md). +- Install k3s cluster. Both Single and multi-node clusters + +- Configure the cluster + +- Upgrade a cluster + +For a more detailed list of featured checkout the roles [documentation](roles/nfc_kubernetes/index.md). ## Using this collection This collection has been designed to be a complete and self-contained management tool for a K3s kubernetes cluster. -## K3s Kubernetes Installation +## Cluster Installation By default the install playbook will install to localhost. @@ -64,4 +70,9 @@ The install playbook has a dynamic `hosts` key. This has been done to specifical For the available variables please view the [nfc_kubernetes role docs](roles/nfc_kubernetes/index.md#default-variables) +## Cluster Upgrade +[In place cluster upgrades](https://docs.k3s.io/upgrades/manual#upgrade-k3s-using-the-binary) is the method used to conduct the cluster upgrades. The logic for the upgrades first confirms that K3s is installed and that the local binary and running k3s version are the desired versions. If they are not, they will be updated to the desired version. On completion of this the node has its `k3s` service restarted which completes the upgrade process. + +!!! danger + not following the [Kubernetes version skew policy](https://kubernetes.io/releases/version-skew-policy/) when upgrading your cluster may break your cluster. diff --git a/docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md b/docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md index 3fe6509..2b766d3 100644 --- a/docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md +++ b/docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md @@ -70,6 +70,8 @@ This Ansible role is designed to deploy a K3s Kubernetes cluster. Without adding - Install the Helm Binary +- Upgrade cluster + ## Role Workflow diff --git a/roles/nfc_kubernetes/tasks/install.yaml b/roles/nfc_kubernetes/tasks/install.yaml index 18857ff..61742a6 100644 --- a/roles/nfc_kubernetes/tasks/install.yaml +++ b/roles/nfc_kubernetes/tasks/install.yaml @@ -35,6 +35,19 @@ nfc_role_kubernetes_configure_firewall +- name: Install required software + ansible.builtin.apt: + name: python3-pip + install_recommends: false + state: present + when: > + install_kubernetes | default(true) | bool + and + not kubernetes_installed | default(false) | bool + tags: + - always + + - name: K3s Install ansible.builtin.include_tasks: file: k3s/install.yaml diff --git a/roles/nfc_kubernetes/tasks/k3s/install.yaml b/roles/nfc_kubernetes/tasks/k3s/install.yaml index 88a74ae..a233779 100644 --- a/roles/nfc_kubernetes/tasks/k3s/install.yaml +++ b/roles/nfc_kubernetes/tasks/k3s/install.yaml @@ -1,5 +1,16 @@ --- +- name: Install required python modules + ansible.builtin.pip: + name: "{{ item }}" + state: present + loop: "{{ pip_packages }}" + vars: + pip_packages: + - kubernetes>=12.0.0 + - PyYAML>=3.11 + + - name: Check for calico deployment manifest ansible.builtin.stat: name: /var/lib/rancher/k3s/server/manifests/calico.yaml @@ -138,28 +149,215 @@ when: directory_network_manager_metadata.stat.exists -- name: Check if K3s Installed +- name: File Metadata - k3s binary + ansible.builtin.stat: + checksum_algorithm: sha256 + name: /usr/local/bin/k3s + register: metadata_file_k3s_existing_binary + + +- name: File Metadata - k3s[-agent].service + ansible.builtin.stat: + checksum_algorithm: sha256 + name: |- + /etc/systemd/system/k3s + {%- if not nfc_role_kubernetes_master | default(false) | bool -%} + -agent + {%- endif -%} + .service + register: metadata_file_k3s_service + + +- name: Directory Metadata - /etc/rancher/k3s/k3s.yaml + ansible.builtin.stat: + name: /etc/rancher/k3s/k3s.yaml + register: metadata_dir_etc_k3s + + +- name: File Metadata - /var/lib/rancher/k3s/server/token + ansible.builtin.stat: + checksum_algorithm: sha256 + name: /var/lib/rancher/k3s/server/token + register: metadata_file_var_k3s_token + + +- name: Config Link ansible.builtin.shell: - cmd: | - if [[ $(service k3s status) ]]; then exit 0; else exit 1; fi - executable: /bin/bash - changed_when: false - failed_when: false - register: k3s_installed + cmd: > + ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config + executable: bash + creates: ~/.kube/config when: > nfc_role_kubernetes_master | default(false) | bool + and + metadata_dir_etc_k3s.stat.exists | default(false) | bool -- name: Check if K3s Installed +- name: Fetch Kubernetes Node Object + kubernetes.core.k8s_info: + kind: Node + name: "{{ inventory_hostname }}" + register: kubernetes_node + when: > + metadata_file_k3s_existing_binary.stat.exists | default(false) | bool + and + metadata_file_k3s_service.stat.exists | default(false) | bool + and + metadata_dir_etc_k3s.stat.exists | default(false) | bool + and + metadata_file_var_k3s_token.stat.exists | default(false) | bool + + +- name: Fetch Installed K3s Metadata ansible.builtin.shell: cmd: | - if [[ $(service k3s-agent status) ]]; then exit 0; else exit 1; fi + export installed_version=$(k3s --version | grep k3s | awk '{print $3}'); + export installed=" + {%- if + metadata_file_k3s_existing_binary.stat.exists | default(false) | bool + and + metadata_file_k3s_service.stat.exists | default(false) | bool + and + metadata_dir_etc_k3s.stat.exists | default(false) | bool + and + metadata_file_var_k3s_token.stat.exists | default(false) | bool + -%} + true + {%- else -%} + false + {%- endif -%}"; + export running_version="{{ kubernetes_node.resources[0].status.nodeInfo.kubeletVersion | default('0') }}"; + + export correct_hash=$(wget -q https://github.com/k3s-io/k3s/releases/download/v + {{-KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}} + /sha256sum- + {%- if ansible_architecture | lower == 'x86_64' -%} + amd64 + {%- elif ansible_architecture | lower == 'aarch64' -%} + arm64 + {%- endif %}.txt -O - | grep -v 'images' | awk '{print $1}'); + + cat < - not nfc_role_kubernetes_worker | default(false) | bool + file_cached_k3s_binary.stat.checksum | default('0') != node_k3s.desired_hash + + +- name: Download K3s Binary + ansible.builtin.uri: + url: |- + https://github.com/k3s-io/k3s/releases/download/ + {{- node_k3s.desired_version | urlencode -}} + /k3s + {%- if cpu_arch.key == 'aarch64' -%} + -arm64 + {%- endif %} + method: GET + return_content: false + status_code: + - 200 + - 304 + dest: "/tmp/k3s.{{ cpu_arch.key }}" + mode: "744" + register: k3s_download_files + delegate_to: localhost + failed_when: > + (lookup('ansible.builtin.file', '/tmp/k3s.' + cpu_arch.key) | hash('sha256') | string) != node_k3s.desired_hash + and + ( + k3s_download_files.status | int != 200 + or + k3s_download_files.status | int != 304 + ) + run_once: true + when: ansible_os_family == 'Debian' + loop: "{{ nfc_kubernetes_install_architectures | dict2items }}" + loop_control: + loop_var: cpu_arch + vars: + ansible_connection: local + + +- name: Copy K3s binary to Host + ansible.builtin.copy: + src: "/tmp/k3s.{{ ansible_architecture }}" + dest: "/usr/local/bin/k3s" + mode: '741' + owner: root + group: root + register: k3s_binary_copy + when: > + node_k3s.current_hash != node_k3s.desired_hash + + +- name: K3s Binary Upgrade + ansible.builtin.service: + name: |- + {%- if nfc_role_kubernetes_master | default(false) | bool -%} + k3s + {%- else -%} + k3s-agent + {%- endif %} + state: restarted + register: k3s_upgrade_service_restart + when: > + ( + k3s_binary_copy.changed | default(false) | bool + and + node_k3s.installed | default(false) | bool + ) + or + ( + node_k3s.running_version != node_k3s.desired_version + and + node_k3s.installed | default(false) | bool + ) + + +- name: Create Fact - cluster_upgraded + ansible.builtin.set_fact: + nfc_role_kubernetes_cluster_upgraded: true + cacheable: true + when: > + k3s_upgrade_service_restart.changed | default(false) | bool - name: Download Install Scripts @@ -192,61 +390,6 @@ when: "{{ nfc_role_kubernetes_install_olm }}" -- name: Download K3s Binary - ansible.builtin.uri: - url: |- - https://github.com/k3s-io/k3s/releases/download/v - {{- KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}} - /k3s - {%- if cpu_arch.key == 'aarch64' -%} - -arm64 - {%- endif %} - method: GET - return_content: false - status_code: - - 200 - - 304 - dest: "/tmp/k3s.{{ cpu_arch.key }}" - mode: "744" - changed_when: false - register: k3s_download_files - delegate_to: localhost - run_once: true - # no_log: true - when: ansible_os_family == 'Debian' - loop: "{{ nfc_kubernetes_install_architectures | dict2items }}" - loop_control: - loop_var: cpu_arch - vars: - ansible_connection: local - - -- name: "[TRACE] Downloaded File SHA256" - ansible.builtin.set_fact: - hash_sha256_k3s_downloaded_binary: "{{ lookup('ansible.builtin.file', '/tmp/k3s.' + cpu_arch.key) | hash('sha256') | string }}" - delegate_to: localhost - loop: "{{ nfc_kubernetes_install_architectures | dict2items }}" - loop_control: - loop_var: cpu_arch - - -- name: Existing k3s File hash - ansible.builtin.stat: - checksum_algorithm: sha256 - name: /usr/local/bin/k3s - register: hash_sha256_k3s_existing_binary - - -- name: Copy K3s binary to Host - ansible.builtin.copy: - src: "/tmp/k3s.{{ ansible_architecture }}" - dest: "/usr/local/bin/k3s" - mode: '741' - owner: root - group: root - when: hash_sha256_k3s_existing_binary.stat.checksum | default('0') != hash_sha256_k3s_downloaded_binary - - - name: Copy install scripts to Host ansible.builtin.copy: src: "{{ item.path }}" @@ -308,7 +451,7 @@ and file_calico_yaml_metadata.stat.exists and - k3s_installed.rc == 0 + not node_k3s.installed | bool ) or 'calico_manifest' in ansible_run_tags @@ -341,13 +484,23 @@ ansible.builtin.shell: cmd: | INSTALL_K3S_SKIP_DOWNLOAD=true \ - INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \ + INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \ /tmp/install.sh {% if nfc_role_kubernetes_etcd_enabled %}--cluster-init{% endif %} changed_when: false when: > kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and - k3s_installed.rc == 1 + not node_k3s.installed | bool + + +- name: Config Link + ansible.builtin.shell: + cmd: > + ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config + executable: bash + creates: ~/.kube/config + when: > + nfc_role_kubernetes_master | default(false) | bool - name: Install Calico Operator @@ -502,7 +655,7 @@ cmd: | INSTALL_K3S_EXEC="server" \ INSTALL_K3S_SKIP_DOWNLOAD=true \ - INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \ + INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \ K3S_TOKEN="{{ k3s_join_token }}" \ /tmp/install.sh executable: /bin/bash @@ -512,7 +665,7 @@ and not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and - k3s_installed.rc == 1 + not node_k3s.installed | bool - name: Install K3s (worker nodes) @@ -521,7 +674,7 @@ set -o pipefail INSTALL_K3S_EXEC="agent" \ INSTALL_K3S_SKIP_DOWNLOAD=true \ - INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \ + INSTALL_K3S_VERSION="v{{ node_k3s.desired_version }}" \ K3S_TOKEN="{{ k3s_join_token }}" \ K3S_URL="https://{{ hostvars[kubernetes_config.cluster.prime.name | default(inventory_hostname)].ansible_host }}:6443" \ /tmp/install.sh - @@ -532,7 +685,7 @@ and not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and - k3s_installed.rc == 1 + not node_k3s.installed | bool - name: Set Kubernetes Final Install Fact -- 2.49.0 From 7c20146660e29eb0ce6616885cba9f8734e5657d Mon Sep 17 00:00:00 2001 From: Jon Date: Fri, 29 Mar 2024 19:18:42 +0930 Subject: [PATCH 2/6] chore: fix yaml schema paths for vscode !53 --- .vscode/settings.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 56d55e7..458df7b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,9 +1,9 @@ { "yaml.schemas": { "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/tasks": [ - "roles/nfc_firewall/tasks/*.yaml", - "roles/nfc_firewall/tasks/*/*.yaml", - "roles/nfc_firewall/tasks/*/*/*.yaml" + "roles/nfc_kubernetes/tasks/*.yaml", + "roles/nfc_kubernetes/tasks/*/*.yaml", + "roles/nfc_kubernetes/tasks/*/*/*.yaml" ], "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/vars.json": [ "roles/nfc_kubernetes/variables/**.yaml" -- 2.49.0 From c5371b8ff4cf60de507290ba3b1e068f537cf8ef Mon Sep 17 00:00:00 2001 From: Jon Date: Fri, 29 Mar 2024 19:44:38 +0930 Subject: [PATCH 3/6] feat(upgrade): If upgrade occurs, dont run remaining tasks !53 --- .../ansible/collection/kubernetes/index.md | 3 + roles/nfc_kubernetes/tasks/install.yaml | 6 ++ roles/nfc_kubernetes/tasks/k3s/install.yaml | 55 +++++++++++++++---- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/docs/projects/ansible/collection/kubernetes/index.md b/docs/projects/ansible/collection/kubernetes/index.md index 7fa5489..83f8100 100644 --- a/docs/projects/ansible/collection/kubernetes/index.md +++ b/docs/projects/ansible/collection/kubernetes/index.md @@ -74,5 +74,8 @@ For the available variables please view the [nfc_kubernetes role docs](roles/nfc [In place cluster upgrades](https://docs.k3s.io/upgrades/manual#upgrade-k3s-using-the-binary) is the method used to conduct the cluster upgrades. The logic for the upgrades first confirms that K3s is installed and that the local binary and running k3s version are the desired versions. If they are not, they will be updated to the desired version. On completion of this the node has its `k3s` service restarted which completes the upgrade process. +!!! info + If an upgrade occurs, no other task within the play will run. This is by design. if you have further tasks to be run in addition to the upgrade, run the play again. + !!! danger not following the [Kubernetes version skew policy](https://kubernetes.io/releases/version-skew-policy/) when upgrading your cluster may break your cluster. diff --git a/roles/nfc_kubernetes/tasks/install.yaml b/roles/nfc_kubernetes/tasks/install.yaml index 61742a6..c7808a0 100644 --- a/roles/nfc_kubernetes/tasks/install.yaml +++ b/roles/nfc_kubernetes/tasks/install.yaml @@ -72,6 +72,8 @@ install_kubernetes | default(true) | bool and kubernetes_installed | default(false) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool tags: - always @@ -88,6 +90,8 @@ kubernetes_config.kube_virt.enabled | default(nfc_role_kubernetes_install_kubevirt) and inventory_hostname in kubernetes_config.kube_virt.nodes | default([ inventory_hostname ]) | list + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool tags: - always @@ -104,5 +108,7 @@ kubernetes_config.helm.enabled | default(nfc_role_kubernetes_install_helm) and nfc_role_kubernetes_master + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool tags: - always diff --git a/roles/nfc_kubernetes/tasks/k3s/install.yaml b/roles/nfc_kubernetes/tasks/k3s/install.yaml index a233779..661538a 100644 --- a/roles/nfc_kubernetes/tasks/k3s/install.yaml +++ b/roles/nfc_kubernetes/tasks/k3s/install.yaml @@ -379,6 +379,8 @@ ansible_os_family == 'Debian' and item.when | default(true) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool loop: "{{ download_files }}" vars: ansible_connection: local @@ -406,6 +408,8 @@ when: "{{ nfc_role_kubernetes_install_olm }}" when: > item.when | default(true) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Required Initial config files @@ -419,7 +423,8 @@ loop: "{{ k3s.files }}" when: > item.when | default(true) | bool - # kubernetes_config.cluster.prime.name == inventory_hostname + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Copy Intial required templates @@ -434,6 +439,8 @@ diff: true when: > item.when | default(true) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool vars: templates_to_apply: - src: k3s-config.yaml.j2 @@ -478,6 +485,8 @@ ansible.builtin.command: cmd: update-alternatives --set iptables /usr/sbin/iptables-legacy changed_when: false + when: > + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install K3s (prime master) @@ -491,16 +500,8 @@ kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and not node_k3s.installed | bool - - -- name: Config Link - ansible.builtin.shell: - cmd: > - ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config - executable: bash - creates: ~/.kube/config - when: > - nfc_role_kubernetes_master | default(false) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install Calico Operator @@ -523,6 +524,8 @@ 'calico_manifest' not in ansible_run_tags and kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install MetalLB Operator @@ -542,6 +545,8 @@ nfc_kubernetes_enable_metallb | default(false) | bool and kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Wait for kubernetes prime to be ready @@ -566,6 +571,20 @@ kubernetes_ready_check.rc != 0 changed_when: false failed_when: kubernetes_ready_check.rc != 0 + when: > + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool + + +- name: Config Link + ansible.builtin.shell: + cmd: > + ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config + executable: bash + creates: ~/.kube/config + when: > + nfc_role_kubernetes_master | default(false) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install olm @@ -582,6 +601,8 @@ kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and nfc_role_kubernetes_install_olm | default(false) | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Uninstall OLM @@ -610,6 +631,8 @@ kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and 'olm_uninstall' in ansible_run_tags + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Enable Cluster Encryption @@ -622,6 +645,8 @@ and kubernetes_config.cluster.networking.encrypt | default(false) | bool and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool + and ( 'calico_manifest' in ansible_run_tags or @@ -640,6 +665,8 @@ run_once: true register: k3s_join_token no_log: true # Value is sensitive + when: > + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Create Token fact @@ -648,6 +675,8 @@ delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}" run_once: true no_log: true # Value is sensitive + when: > + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install K3s (master nodes) @@ -666,6 +695,8 @@ not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and not node_k3s.installed | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Install K3s (worker nodes) @@ -686,6 +717,8 @@ not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname and not node_k3s.installed | bool + and + not nfc_role_kubernetes_cluster_upgraded | default(false) | bool - name: Set Kubernetes Final Install Fact -- 2.49.0 From 4d44c01b3263c72342fa2aa85788f5c83872aaa3 Mon Sep 17 00:00:00 2001 From: Jon Date: Fri, 29 Mar 2024 20:03:43 +0930 Subject: [PATCH 4/6] refactor(galaxy): for dependent collections prefix with `>=` so as to not cause version lock !53 --- galaxy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/galaxy.yml b/galaxy.yml index ed9263e..baf204a 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -44,9 +44,9 @@ tags: # L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version # range specifiers can be set and are separated by ',' dependencies: - ansible.posix: '1.5.4' - kubernetes.core: '3.0.0' - nofusscomputing.firewall: '1.1.0' + ansible.posix: '>=1.5.4' + kubernetes.core: '>=3.0.0' + nofusscomputing.firewall: '>=1.1.0' # The URL of the originating SCM repository -- 2.49.0 From 7ef739d0638b96327a85ee5257e7a7020875feca Mon Sep 17 00:00:00 2001 From: Jon Date: Sat, 30 Mar 2024 03:08:17 +0930 Subject: [PATCH 5/6] feat: add retry=3 delay=10 secs to all ansible url modules !53 --- roles/nfc_kubernetes/tasks/helm/main.yaml | 2 ++ roles/nfc_kubernetes/tasks/k3s/install.yaml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/roles/nfc_kubernetes/tasks/helm/main.yaml b/roles/nfc_kubernetes/tasks/helm/main.yaml index c1c4887..cb55052 100644 --- a/roles/nfc_kubernetes/tasks/helm/main.yaml +++ b/roles/nfc_kubernetes/tasks/helm/main.yaml @@ -5,6 +5,8 @@ url: https://baltocdn.com/helm/signing.asc dest: /usr/share/keyrings/helm.asc mode: 740 + delay: 10 + retries: 3 - name: Add Helm Repository diff --git a/roles/nfc_kubernetes/tasks/k3s/install.yaml b/roles/nfc_kubernetes/tasks/k3s/install.yaml index 661538a..12fc4e4 100644 --- a/roles/nfc_kubernetes/tasks/k3s/install.yaml +++ b/roles/nfc_kubernetes/tasks/k3s/install.yaml @@ -297,6 +297,8 @@ - 304 dest: "/tmp/k3s.{{ cpu_arch.key }}" mode: "744" + delay: 10 + retries: 3 register: k3s_download_files delegate_to: localhost failed_when: > @@ -371,6 +373,8 @@ dest: "{{ item.dest }}" mode: "744" changed_when: false + delay: 10 + retries: 3 register: k3s_download_script delegate_to: localhost run_once: true -- 2.49.0 From 5980123e7aada72419ad5aab5d951e23fedccbd7 Mon Sep 17 00:00:00 2001 From: Jon Date: Sat, 30 Mar 2024 03:11:17 +0930 Subject: [PATCH 6/6] feat(test): add integration test. playbook install !53 --- .gitignore | 2 + .gitlab-ci.yml | 1 + .gitlab/integration_test.gitlab-ci.yml | 207 ++++++++++++++++++ .gitlab/integration_test_trace.sh | 42 ++++ .gitlab/test_results.yaml | 19 ++ .vscode/settings.json | 3 +- README.md | 5 + .../ansible/collection/kubernetes/index.md | 2 + roles/nfc_kubernetes/tasks/install.yaml | 9 + roles/nfc_kubernetes/tasks/k3s/install.yaml | 33 ++- 10 files changed, 314 insertions(+), 9 deletions(-) create mode 100644 .gitlab/integration_test.gitlab-ci.yml create mode 100644 .gitlab/integration_test_trace.sh create mode 100644 .gitlab/test_results.yaml diff --git a/.gitignore b/.gitignore index 5fa197d..953f083 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ artifacts/ build/ +test_results/ +test_results.json *.tar.gz \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab3349b..5f9ffb3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ variables: include: + - local: .gitlab/integration_test.gitlab-ci.yml - project: nofusscomputing/projects/gitlab-ci ref: development file: diff --git a/.gitlab/integration_test.gitlab-ci.yml b/.gitlab/integration_test.gitlab-ci.yml new file mode 100644 index 0000000..2eb1c44 --- /dev/null +++ b/.gitlab/integration_test.gitlab-ci.yml @@ -0,0 +1,207 @@ + +.integration_test: + + stage: test + + needs: + - "Build Collection" + + image: + name: nofusscomputing/docker-buildx-qemu:dev + pull_policy: always + + variables: + DOCKER_HOST: tcp://docker:2375/ + DOCKER_DRIVER: overlay2 + # GIT_STRATEGY: none + + services: + - name: docker:23-dind + entrypoint: ["env", "-u", "DOCKER_HOST"] + command: ["dockerd-entrypoint.sh"] + before_script: + - | # start test container + docker run -d \ + --privileged \ + -v ${PWD}:/workdir \ + -v ${PWD}/artifacts/galaxy:/collection \ + --workdir /workdir \ + --rm \ + --env "ANSIBLE_FORCE_COLOR=true" \ + --env "CI_COMMIT_SHA=${CI_COMMIT_SHA}" \ + --env "ANSIBLE_LOG_PATH=/workdir/ansible.log" \ + --name test_image_${CI_JOB_ID} \ + nofusscomputing/ansible-docker-os:dev-${test_image} + + - | # enter test container + docker exec -i test_image_${CI_JOB_ID} ps aux + - docker ps + - docker exec -i test_image_${CI_JOB_ID} apt update + - docker exec -i test_image_${CI_JOB_ID} apt install -y --no-install-recommends python3-pip net-tools dnsutils iptables + - | + if [ "${test_image}" == 'debian-12' ]; then + + echo "Debian 12": + + docker exec -i test_image_${CI_JOB_ID} pip install ansible-core --break-system-packages; + + docker exec -i test_image_${CI_JOB_ID} update-alternatives --set iptables /usr/sbin/iptables-legacy; + + else + + echo " Not Debian 12": + + docker exec -i test_image_${CI_JOB_ID} pip install ansible-core; + + fi + + - docker exec -i test_image_${CI_JOB_ID} cat /etc/hosts + - docker exec -i test_image_${CI_JOB_ID} cat /etc/resolv.conf + - | # check if DNS working + docker exec -i test_image_${CI_JOB_ID} nslookup google.com + script: + - | # inside container? + docker exec -i test_image_${CI_JOB_ID} ls -l /collection; + docker exec -i test_image_${CI_JOB_ID} echo $PWD; + + - | # Show Network Interfaces + docker exec -i test_image_${CI_JOB_ID} ifconfig; + + - | # Install the collection + docker exec -i test_image_${CI_JOB_ID} bash -c 'ansible-galaxy collection install $(ls /collection/*.tar.gz)' + + - | # output ansible vars + docker exec -i test_image_${CI_JOB_ID} ansible -m setup localhost + + - | # run the collection + docker exec -i test_image_${CI_JOB_ID} \ + ${test_command} \ + --extra-vars "nfc_role_firewall_policy_input=ACCEPT" \ + --extra-vars "nfc_role_firewall_policy_forward=ACCEPT" \ + -vv + + - | # Create test.yaml + mkdir -p test_results; + cat < test_results/${test_image}.json + { + "$( echo ${test_image} | sed -e 's/\./_/')": "passed" + } + + EOF + + after_script: + - | # Create test.yaml if not exists + if [ ! -f test_results/${test_image}.json ]; then + + echo "[TRACE] Test has failed" + + mkdir -p test_results; + + cat < test_results/${test_image}.json + { + "$( echo ${test_image} | sed -e 's/\./_/')": "fail" + } + + EOF + + fi + + - | # Run trace script for debugging + chmod +x ./.gitlab/integration_test_trace.sh; + + ./.gitlab/integration_test_trace.sh; + + artifacts: + untracked: false + paths: + - ansible.log + - test_results/* + when: always + + rules: + + - if: $CI_COMMIT_TAG + allow_failure: true + when: on_success + + - if: "$CI_COMMIT_AUTHOR =='nfc_bot '" + when: never + + - if: # Occur on merge + $CI_COMMIT_BRANCH == "development" + && + $CI_PIPELINE_SOURCE == "push" + allow_failure: true + when: always + + - if: + $CI_COMMIT_BRANCH != "development" + && + $CI_COMMIT_BRANCH != "master" + && + $CI_PIPELINE_SOURCE == "push" + allow_failure: true + when: always + + - when: never + + + +Playbook - Install: + extends: .integration_test + parallel: + matrix: + - test_image: debian-11 + test_command: ansible-playbook nofusscomputing.kubernetes.install + - test_image: debian-12 + test_command: ansible-playbook nofusscomputing.kubernetes.install + - test_image: ubuntu-20.04 + test_command: ansible-playbook nofusscomputing.kubernetes.install + - test_image: ubuntu-22.04 + test_command: ansible-playbook nofusscomputing.kubernetes.install + + + +test_results: + stage: test + + extends: .ansible_playbook + + variables: + ansible_playbook: .gitlab/test_results.yaml + ANSIBLE_PLAYBOOK_DIR: $CI_PROJECT_DIR + + needs: + - Playbook - Install + + artifacts: + untracked: false + when: always + access: all + expire_in: "3 days" + paths: + - test_results.json + + rules: + + - if: $CI_COMMIT_TAG + when: on_success + + - if: "$CI_COMMIT_AUTHOR =='nfc_bot '" + when: never + + - if: # Occur on merge + $CI_COMMIT_BRANCH == "development" + && + $CI_PIPELINE_SOURCE == "push" + when: always + + - if: + $CI_COMMIT_BRANCH != "development" + && + $CI_COMMIT_BRANCH != "master" + && + $CI_PIPELINE_SOURCE == "push" + when: always + + - when: never \ No newline at end of file diff --git a/.gitlab/integration_test_trace.sh b/.gitlab/integration_test_trace.sh new file mode 100644 index 0000000..8fa485f --- /dev/null +++ b/.gitlab/integration_test_trace.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# colour ref: https://stackoverflow.com/a/28938235 + +NC='\033[0m' # Text Reset + +# Regular Colors +Black='\033[0;30m' # Black +Red='\033[0;31m' # Red +Green='\033[0;32m' # Green +Yellow='\033[0;33m' # Yellow +Blue='\033[0;34m' # Blue +Purple='\033[0;35m' # Purple +Cyan='\033[0;36m' # Cyan + + +cmd() { + + echo -e "${Yellow}[TRACE] ${Green}executing ${Cyan}'$1'${NC}" + + docker exec -i test_image_${CI_JOB_ID} $1 || true + +} + + +cmd "journalctl -xeu netfilter-persistent.service"; + +cmd "journalctl -xeu iptables.service" + +cmd "journalctl -xeu k3s.service" + +cmd "systemctl status netfilter-persistent.service" + +cmd "systemctl status iptables.servic" + +cmd "systemctl status k3s.service" + +cmd "kubectl get po -A -o wide" + +cmd "kubectl get no -o wide" + +cmd "iptables -nvL --line-numbers" diff --git a/.gitlab/test_results.yaml b/.gitlab/test_results.yaml new file mode 100644 index 0000000..fe9441a --- /dev/null +++ b/.gitlab/test_results.yaml @@ -0,0 +1,19 @@ +--- + +- name: Create Test Results File + hosts: localhost + gather_facts: false + + + tasks: + + + - name: Load Test Results + ansible.builtin.include_vars: + dir: ../test_results + name: test_results + + - name: Create Results file + ansible.builtin.copy: + content: "{{ (test_results) | to_nice_json }}" + dest: ../test_results.json diff --git a/.vscode/settings.json b/.vscode/settings.json index 458df7b..c383ad8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,7 +7,8 @@ ], "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/vars.json": [ "roles/nfc_kubernetes/variables/**.yaml" - ] + ], + "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/playbook": ".gitlab/test_results.yaml" }, "gitlab.aiAssistedCodeSuggestions.enabled": false, "gitlab.duoChat.enabled": false, diff --git a/README.md b/README.md index 0c741fa..4cb3f8c 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,17 @@ This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/a ![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fmaster%2F.cz.yaml) +![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814) + + ---- **Development Branch** ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fdevelopment%2F.cz.yaml) +![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fdevelopment%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814) + ----
diff --git a/docs/projects/ansible/collection/kubernetes/index.md b/docs/projects/ansible/collection/kubernetes/index.md index 83f8100..534a6c4 100644 --- a/docs/projects/ansible/collection/kubernetes/index.md +++ b/docs/projects/ansible/collection/kubernetes/index.md @@ -13,6 +13,8 @@ about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernete ![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) +![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fnofusscomputing%2Fprojects%2Fansible%2Fcollections%2Fkubernetes%2F-%2Fjobs%2Fartifacts%2Fmaster%2Fbrowse%2Ftest_results.json?job%2Btest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814) + [![Downloads](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fkubernetes%2F&query=%24.download_count&style=plastic&logo=ansible&logoColor=white&label=Galaxy%20Downloads&labelColor=black&color=cyan)](https://galaxy.ansible.com/ui/repo/published/nofusscomputing/kubernetes/) diff --git a/roles/nfc_kubernetes/tasks/install.yaml b/roles/nfc_kubernetes/tasks/install.yaml index c7808a0..98fb182 100644 --- a/roles/nfc_kubernetes/tasks/install.yaml +++ b/roles/nfc_kubernetes/tasks/install.yaml @@ -18,6 +18,15 @@ not ansible_check_mode +- name: Testing Env Variables + ansible.builtin.set_fact: + ansible_default_ipv4: { + "address": "127.0.0.1" + } + when: > + lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != '' + + - name: Check Machine Architecture ansible.builtin.set_fact: nfc_kubernetes_install_architectures: "{{ nfc_kubernetes_install_architectures | default({}) | combine({ansible_architecture: ''}) }}" diff --git a/roles/nfc_kubernetes/tasks/k3s/install.yaml b/roles/nfc_kubernetes/tasks/k3s/install.yaml index 12fc4e4..ba1c41b 100644 --- a/roles/nfc_kubernetes/tasks/k3s/install.yaml +++ b/roles/nfc_kubernetes/tasks/k3s/install.yaml @@ -32,6 +32,7 @@ loop_var: package vars: packages: + - wget - curl - iptables - jq @@ -52,14 +53,29 @@ - install -- name: Disable swap - ansible.builtin.command: - cmd: swapoff -a - changed_when: false - when: - - ansible_os_family == 'Debian' - tags: - - install +- name: Testing Environment try/catch + block: + + + - name: Disable swap + ansible.builtin.command: + cmd: swapoff -a + changed_when: false + when: + - ansible_os_family == 'Debian' + tags: + - install + + + rescue: + + - name: Check if inside Gitlab CI + ansible.builtin.assert: + that: + - lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != '' + success_msg: "Inside testing enviroment, 'Disable swap' error OK" + fail_msg: "You should figure out what went wrong" + - name: Check an armbian os system ansible.builtin.stat: @@ -226,6 +242,7 @@ {%- else -%} false {%- endif -%}"; + export running_version="{{ kubernetes_node.resources[0].status.nodeInfo.kubeletVersion | default('0') }}"; export correct_hash=$(wget -q https://github.com/k3s-io/k3s/releases/download/v -- 2.49.0