feat: support upgrading cluster

In place binary upgrades was chosen as its just a matter of changing binary and restarting the service

!53 closes #14

docs/projects/ansible/collection/kubernetes/index.md

@@ -29,14 +29,20 @@ To install this collection use `ansible-galaxy collection install nofusscomputin
 
 ## Features
 
-Most of the features of this collection are from the included role `nfc_kubernetes`, please [view its page for feature details](roles/nfc_kubernetes/index.md).
+- Install k3s cluster. Both Single and multi-node clusters
+
+- Configure the cluster
+
+- Upgrade a cluster
+
+For a more detailed list of featured checkout the roles [documentation](roles/nfc_kubernetes/index.md).
 
 
 ## Using this collection
 
 This collection has been designed to be a complete and self-contained management tool for a K3s kubernetes cluster.
 
-## K3s Kubernetes Installation
+## Cluster Installation
 
 By default the install playbook will install to localhost.
 
@@ -64,4 +70,9 @@ The install playbook has a dynamic `hosts` key. This has been done to specifical
 For the available variables please view the [nfc_kubernetes role docs](roles/nfc_kubernetes/index.md#default-variables)
 
 
+## Cluster Upgrade
 
+[In place cluster upgrades](https://docs.k3s.io/upgrades/manual#upgrade-k3s-using-the-binary) is the method used to conduct the cluster upgrades. The logic for the upgrades first confirms that K3s is installed and that the local binary and running k3s version are the desired versions. If they are not, they will be updated to the desired version. On completion of this the node has its `k3s` service restarted which completes the upgrade process.
+
+!!! danger
+    not following the [Kubernetes version skew policy](https://kubernetes.io/releases/version-skew-policy/) when upgrading your cluster may break your cluster.

docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md

@@ -70,6 +70,8 @@ This Ansible role is designed to deploy a K3s Kubernetes cluster. Without adding
 
 - Install the Helm Binary
 
+- Upgrade cluster
+
 
 ## Role Workflow
 

roles/nfc_kubernetes/tasks/install.yaml

@@ -35,6 +35,19 @@
     nfc_role_kubernetes_configure_firewall
 
 
+- name: Install required software
+  ansible.builtin.apt:
+    name: python3-pip
+    install_recommends: false
+    state: present
+  when: >
+    install_kubernetes | default(true) | bool
+      and
+    not kubernetes_installed | default(false) | bool
+  tags:
+    - always
+
+
 - name: K3s Install
   ansible.builtin.include_tasks:
     file: k3s/install.yaml

roles/nfc_kubernetes/tasks/k3s/install.yaml

@@ -1,5 +1,16 @@
 ---
 
+- name: Install required python modules
+  ansible.builtin.pip:
+    name: "{{ item }}"
+    state: present
+  loop: "{{ pip_packages }}"
+  vars:
+    pip_packages:
+      - kubernetes>=12.0.0
+      - PyYAML>=3.11
+
+
 - name: Check for calico deployment manifest
   ansible.builtin.stat:
     name: /var/lib/rancher/k3s/server/manifests/calico.yaml
@@ -138,28 +149,215 @@
   when: directory_network_manager_metadata.stat.exists
 
 
-- name: Check if K3s Installed
+- name: File Metadata - k3s binary
+  ansible.builtin.stat:
+    checksum_algorithm: sha256
+    name: /usr/local/bin/k3s
+  register: metadata_file_k3s_existing_binary
+
+
+- name: File Metadata - k3s[-agent].service
+  ansible.builtin.stat:
+    checksum_algorithm: sha256
+    name: |-
+      /etc/systemd/system/k3s
+      {%- if not nfc_role_kubernetes_master | default(false) | bool -%}
+        -agent
+      {%- endif -%}
+      .service
+  register: metadata_file_k3s_service
+
+
+- name: Directory Metadata - /etc/rancher/k3s/k3s.yaml
+  ansible.builtin.stat:
+    name: /etc/rancher/k3s/k3s.yaml
+  register: metadata_dir_etc_k3s
+
+
+- name: File Metadata - /var/lib/rancher/k3s/server/token
+  ansible.builtin.stat:
+    checksum_algorithm: sha256
+    name: /var/lib/rancher/k3s/server/token
+  register: metadata_file_var_k3s_token
+
+
+- name: Config Link
   ansible.builtin.shell:
-    cmd: |
+    cmd: >
-      if [[ $(service k3s status) ]]; then exit 0; else exit 1; fi
+      ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config
-    executable: /bin/bash
+    executable: bash
-  changed_when: false
+    creates: ~/.kube/config
-  failed_when: false
-  register: k3s_installed
   when: >
     nfc_role_kubernetes_master | default(false) | bool
+      and
+    metadata_dir_etc_k3s.stat.exists | default(false) | bool
 
 
-- name: Check if K3s Installed
+- name: Fetch Kubernetes Node Object
+  kubernetes.core.k8s_info:
+    kind: Node
+    name: "{{ inventory_hostname }}"
+  register: kubernetes_node
+  when: >
+    metadata_file_k3s_existing_binary.stat.exists | default(false) | bool
+      and
+    metadata_file_k3s_service.stat.exists | default(false) | bool
+      and
+    metadata_dir_etc_k3s.stat.exists | default(false) | bool
+      and
+    metadata_file_var_k3s_token.stat.exists | default(false) | bool
+
+
+- name: Fetch Installed K3s Metadata
   ansible.builtin.shell:
     cmd: |
-      if [[ $(service k3s-agent status) ]]; then exit 0; else exit 1; fi
+      export installed_version=$(k3s --version | grep k3s | awk '{print $3}');
+      export installed="
+      {%- if
+        metadata_file_k3s_existing_binary.stat.exists | default(false) | bool
+          and
+        metadata_file_k3s_service.stat.exists | default(false) | bool
+          and
+        metadata_dir_etc_k3s.stat.exists | default(false) | bool
+          and
+        metadata_file_var_k3s_token.stat.exists | default(false) | bool
+      -%}
+        true
+      {%- else -%}
+        false
+      {%- endif -%}";
+      export running_version="{{ kubernetes_node.resources[0].status.nodeInfo.kubeletVersion | default('0') }}";
+
+      export correct_hash=$(wget -q https://github.com/k3s-io/k3s/releases/download/v
+        {{-KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}}
+          /sha256sum-
+          {%- if ansible_architecture | lower == 'x86_64' -%}
+            amd64
+          {%- elif ansible_architecture | lower == 'aarch64' -%}
+            arm64
+          {%- endif %}.txt -O - | grep -v 'images' | awk '{print $1}');
+
+      cat <<EOF
+      {
+        "current_hash": "{{ metadata_file_k3s_existing_binary.stat.checksum | default('') }}",
+        "current_version": "${installed_version}",
+        "desired_hash": "${correct_hash}",
+        "desired_version": "v{{ KubernetesVersion + KubernetesVersion_k3s_prefix | default('') }}",
+        "installed": ${installed},
+        "running_version": "${running_version}"
+      }
+
+      EOF
+
     executable: /bin/bash
   changed_when: false
   failed_when: false
-  register: k3s_installed
+  register: k3s_metadata
+
+
+- name: K3s Metadata Fact
+  ansible.builtin.set_fact:
+    node_k3s: "{{ k3s_metadata.stdout | from_yaml }}"
+
+
+- name: Cached K3s Binary Details
+  ansible.builtin.stat:
+    path: "/tmp/k3s.{{ ansible_architecture }}"
+    checksum_algorithm: sha256
+  delegate_to: localhost
+  register: file_cached_k3s_binary
+  vars:
+    ansible_connection: local
+
+
+- name: Remove Cached K3s Binaries
+  ansible.builtin.file:
+    path: "/tmp/k3s.{{ ansible_architecture }}"
+    state: absent
+  delegate_to: localhost
+  vars:
+    ansible_connection: local
   when: >
-    not nfc_role_kubernetes_worker | default(false) | bool
+    file_cached_k3s_binary.stat.checksum | default('0') != node_k3s.desired_hash
+
+
+- name: Download K3s Binary
+  ansible.builtin.uri:
+    url: |-
+      https://github.com/k3s-io/k3s/releases/download/
+        {{- node_k3s.desired_version | urlencode -}}
+      /k3s
+      {%- if cpu_arch.key == 'aarch64' -%}
+      -arm64
+      {%- endif %}
+    method: GET
+    return_content: false
+    status_code:
+      - 200
+      - 304
+    dest: "/tmp/k3s.{{ cpu_arch.key }}"
+    mode: "744"
+  register: k3s_download_files
+  delegate_to: localhost
+  failed_when: >
+    (lookup('ansible.builtin.file', '/tmp/k3s.' + cpu_arch.key) | hash('sha256') | string) != node_k3s.desired_hash
+      and
+    (
+      k3s_download_files.status | int != 200
+        or
+      k3s_download_files.status | int != 304
+    )
+  run_once: true
+  when: ansible_os_family == 'Debian'
+  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
+  loop_control:
+    loop_var: cpu_arch
+  vars:
+    ansible_connection: local
+
+
+- name: Copy K3s binary to Host
+  ansible.builtin.copy:
+    src: "/tmp/k3s.{{ ansible_architecture }}"
+    dest: "/usr/local/bin/k3s"
+    mode: '741'
+    owner: root
+    group: root
+  register: k3s_binary_copy
+  when: >
+    node_k3s.current_hash != node_k3s.desired_hash
+
+
+- name: K3s Binary Upgrade
+  ansible.builtin.service:
+    name: |-
+      {%- if nfc_role_kubernetes_master | default(false) | bool -%}
+        k3s
+      {%- else -%}
+        k3s-agent
+      {%- endif %}
+    state: restarted
+  register: k3s_upgrade_service_restart
+  when: >
+    (
+      k3s_binary_copy.changed | default(false) | bool
+        and
+      node_k3s.installed | default(false) | bool
+    )
+      or
+    (
+      node_k3s.running_version != node_k3s.desired_version
+        and
+      node_k3s.installed | default(false) | bool
+    )
+
+
+- name: Create Fact - cluster_upgraded
+  ansible.builtin.set_fact:
+    nfc_role_kubernetes_cluster_upgraded: true
+    cacheable: true
+  when: >
+    k3s_upgrade_service_restart.changed | default(false) | bool
 
 
 - name: Download Install Scripts
@@ -192,61 +390,6 @@
         when: "{{ nfc_role_kubernetes_install_olm }}"
 
 
-- name: Download K3s Binary
-  ansible.builtin.uri:
-    url: |-
-      https://github.com/k3s-io/k3s/releases/download/v
-        {{- KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}}
-      /k3s
-      {%- if cpu_arch.key == 'aarch64' -%}
-      -arm64
-      {%- endif %}
-    method: GET
-    return_content: false
-    status_code:
-      - 200
-      - 304
-    dest: "/tmp/k3s.{{ cpu_arch.key }}"
-    mode: "744"
-  changed_when: false
-  register: k3s_download_files
-  delegate_to: localhost
-  run_once: true
-  # no_log: true
-  when: ansible_os_family == 'Debian'
-  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
-  loop_control:
-    loop_var: cpu_arch
-  vars:
-    ansible_connection: local
-
-
-- name: "[TRACE] Downloaded File SHA256"
-  ansible.builtin.set_fact:
-    hash_sha256_k3s_downloaded_binary: "{{ lookup('ansible.builtin.file', '/tmp/k3s.' + cpu_arch.key) | hash('sha256') | string }}"
-  delegate_to: localhost
-  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
-  loop_control:
-    loop_var: cpu_arch
-
-
-- name: Existing k3s File hash
-  ansible.builtin.stat:
-    checksum_algorithm: sha256
-    name: /usr/local/bin/k3s
-  register: hash_sha256_k3s_existing_binary
-
-
-- name: Copy K3s binary to Host
-  ansible.builtin.copy:
-    src: "/tmp/k3s.{{ ansible_architecture }}"
-    dest: "/usr/local/bin/k3s"
-    mode: '741'
-    owner: root
-    group: root
-  when: hash_sha256_k3s_existing_binary.stat.checksum | default('0') != hash_sha256_k3s_downloaded_binary
-
-
 - name: Copy install scripts to Host
   ansible.builtin.copy:
     src: "{{ item.path }}"
@@ -308,7 +451,7 @@
                   and
                 file_calico_yaml_metadata.stat.exists
                   and
-                k3s_installed.rc == 0
+                not node_k3s.installed | bool
               )
                 or
               'calico_manifest' in ansible_run_tags
@@ -341,13 +484,23 @@
   ansible.builtin.shell:
     cmd: |
       INSTALL_K3S_SKIP_DOWNLOAD=true \
-      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
+      INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \
       /tmp/install.sh {% if nfc_role_kubernetes_etcd_enabled %}--cluster-init{% endif %}
   changed_when: false
   when: >
     kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
       and
-    k3s_installed.rc == 1
+    not node_k3s.installed | bool
+
+
+- name: Config Link
+  ansible.builtin.shell:
+    cmd: >
+      ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config
+    executable: bash
+    creates: ~/.kube/config
+  when: >
+    nfc_role_kubernetes_master | default(false) | bool
 
 
 - name: Install Calico Operator
@@ -502,7 +655,7 @@
     cmd: |
       INSTALL_K3S_EXEC="server" \
       INSTALL_K3S_SKIP_DOWNLOAD=true \
-      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
+      INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \
       K3S_TOKEN="{{ k3s_join_token }}" \
       /tmp/install.sh
     executable: /bin/bash
@@ -512,7 +665,7 @@
       and
     not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
       and
-    k3s_installed.rc == 1
+    not node_k3s.installed | bool
 
 
 - name: Install K3s (worker nodes)
@@ -521,7 +674,7 @@
       set -o pipefail
       INSTALL_K3S_EXEC="agent" \
       INSTALL_K3S_SKIP_DOWNLOAD=true \
-      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
+      INSTALL_K3S_VERSION="v{{ node_k3s.desired_version }}" \
       K3S_TOKEN="{{ k3s_join_token }}" \
       K3S_URL="https://{{ hostvars[kubernetes_config.cluster.prime.name | default(inventory_hostname)].ansible_host }}:6443" \
       /tmp/install.sh -
@@ -532,7 +685,7 @@
       and
     not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
       and
-    k3s_installed.rc == 1
+    not node_k3s.installed | bool
 
 
 - name: Set Kubernetes Final Install Fact