build: bump version 1.13.0 -> 1.13.1

!77

.cz.yaml

@@ -4,5 +4,5 @@ commitizen:
   prerelease_offset: 1
   tag_format: $version
   update_changelog_on_bump: false
-  version: 1.3.0
+  version: 1.13.1
   version_scheme: semver

.gitignore

@@ -1,3 +1,5 @@
 artifacts/
 build/
+test_results/
+test_results.json
 *.tar.gz

.gitlab-ci.yml

@@ -9,6 +9,7 @@ variables:
 
 
 include:
+  - local: .gitlab/integration_test.gitlab-ci.yml
   - project: nofusscomputing/projects/gitlab-ci
     ref: development
     file:
@@ -21,6 +22,40 @@ include:
       - automation/.gitlab-ci-ansible.yaml
 
 
+Build Collection:
+  extends: .ansible_collection_build
+  needs:
+    - job: Ansible Lint
+      optional: true
+    - job: Ansible Lint (galaxy.yml)
+      optional: true
+
+  rules:
+
+    - if: $CI_COMMIT_TAG
+      when: always
+
+    # Needs to run, even by bot as the test results need to be available
+    # - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+    #   when: never
+
+    - if: # Occur on merge
+        $CI_COMMIT_BRANCH
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      when: always
+
+    # - if:
+    #     $CI_COMMIT_BRANCH != "development"
+    #       &&
+    #     $CI_COMMIT_BRANCH != "master"
+    #       &&
+    #     $CI_PIPELINE_SOURCE == "push"
+    #   when: always
+
+    - when: never
+
+
 Update Git Submodules:
   extends: .ansible_playbook_git_submodule
 

.gitlab/integration_test.gitlab-ci.yml

@@ -0,0 +1,219 @@
+
+.integration_test:
+
+  stage: test
+
+  needs:
+    - "Build Collection"
+
+  image: 
+    name: nofusscomputing/docker-buildx-qemu:dev
+    pull_policy: always
+
+  variables:
+    DOCKER_HOST: tcp://docker:2375/
+    DOCKER_DRIVER: overlay2
+    # GIT_STRATEGY: none
+
+  services:
+    - name: docker:23-dind
+      entrypoint: ["env", "-u", "DOCKER_HOST"]
+      command: ["dockerd-entrypoint.sh"]
+  before_script:
+    - | # start test container
+      docker run -d \
+        --privileged \
+        -v ${PWD}:/workdir \
+        -v ${PWD}/artifacts/galaxy:/collection \
+        --workdir /workdir \
+        --rm \
+        --env "ANSIBLE_FORCE_COLOR=true" \
+        --env "CI_COMMIT_SHA=${CI_COMMIT_SHA}" \
+        --env "ANSIBLE_LOG_PATH=/workdir/ansible.log" \
+        --env "PIP_BREAK_SYSTEM_PACKAGES=1" \
+        --name test_image_${CI_JOB_ID} \
+        nofusscomputing/ansible-docker-os:dev-${test_image}
+
+    - | # enter test container
+      docker exec -i test_image_${CI_JOB_ID} ps aux
+    - docker ps
+    - docker exec -i test_image_${CI_JOB_ID} bash -c 'apt update || true'
+    - docker exec -i test_image_${CI_JOB_ID} bash -c 'apt update || true'
+    - docker exec -i test_image_${CI_JOB_ID} apt update
+    - docker exec -i test_image_${CI_JOB_ID} apt install -y --no-install-recommends python3-pip net-tools dnsutils iptables
+    - |
+      if [ "${test_image}" == 'debian-12' ]; then
+
+        echo "Debian 12":
+
+        docker exec -i test_image_${CI_JOB_ID} pip install ansible-core --break-system-packages;
+
+        docker exec -i test_image_${CI_JOB_ID} mkdir -p /etc/iptables;
+
+        docker exec -i test_image_${CI_JOB_ID} touch /etc/iptables/rules.v6;
+
+        docker exec -i test_image_${CI_JOB_ID} update-alternatives --set iptables /usr/sbin/iptables-legacy;
+
+      else
+
+        echo " Not Debian 12":
+
+        docker exec -i test_image_${CI_JOB_ID} pip install ansible-core;
+
+      fi
+
+    - docker exec -i test_image_${CI_JOB_ID} cat /etc/hosts
+    - docker exec -i test_image_${CI_JOB_ID} cat /etc/resolv.conf
+    - | # check if DNS working
+      docker exec -i test_image_${CI_JOB_ID} nslookup google.com
+  script:
+    - | # inside container?
+      docker exec -i test_image_${CI_JOB_ID} ls -l /collection;
+      docker exec -i test_image_${CI_JOB_ID} echo $PWD;
+
+    - | # Show Network Interfaces
+      docker exec -i test_image_${CI_JOB_ID} ifconfig;
+
+    - | # Install the collection
+      docker exec -i test_image_${CI_JOB_ID} bash -c 'ansible-galaxy collection install $(ls /collection/*.tar.gz)'
+
+    - | # output ansible vars
+      docker exec -i test_image_${CI_JOB_ID} ansible -m setup localhost
+
+    - | # run the collection
+      docker exec -i test_image_${CI_JOB_ID} \
+        ${test_command} \
+        --extra-vars "nfc_role_firewall_policy_input=ACCEPT" \
+        --extra-vars "nfc_role_firewall_policy_forward=ACCEPT" \
+        -vv
+
+    - | # Create test.yaml
+      mkdir -p test_results;
+      cat <<EOF > test_results/${test_image}.json
+      {
+        "$( echo ${test_image}  | sed -e 's/\./_/')": "Pass"
+      }
+
+      EOF
+
+  after_script:
+    - | # Create test.yaml if not exists
+      if [ ! -f test_results/${test_image}.json ]; then
+
+      echo "[TRACE] Test has failed"
+
+      mkdir -p test_results;
+
+      cat <<EOF > test_results/${test_image}.json
+      {
+        "$( echo ${test_image}  | sed -e 's/\./_/')": "Fail"
+      }
+
+      EOF
+
+      fi
+
+    - | # Run trace script for debugging
+      chmod +x ./.gitlab/integration_test_trace.sh;
+
+      ./.gitlab/integration_test_trace.sh;
+
+  artifacts:
+    untracked: false
+    paths:
+      - ansible.log
+      - test_results/*
+    when: always
+
+  rules:
+
+    - if: $CI_COMMIT_TAG
+      allow_failure: true
+      when: on_success
+
+    # Needs to run, even by bot as the test results need to be available
+    # - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+    #   when: never
+
+    - if: # Occur on merge
+        $CI_COMMIT_BRANCH
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      allow_failure: true
+      when: on_success
+
+    # - if:
+    #     $CI_COMMIT_BRANCH != "development"
+    #       &&
+    #     $CI_COMMIT_BRANCH != "master"
+    #       &&
+    #     $CI_PIPELINE_SOURCE == "push"
+    #   allow_failure: true
+    #   when: always
+
+    - when: never
+
+
+
+Playbook - Install:
+  extends: .integration_test
+  parallel:
+    matrix:
+      - test_image: debian-11
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+      - test_image: debian-12
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+      - test_image: ubuntu-20.04
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+      - test_image: ubuntu-22.04
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+
+
+
+test_results:
+  stage: test
+
+  extends: .ansible_playbook
+
+  variables:
+    ansible_playbook: .gitlab/test_results.yaml
+    ANSIBLE_PLAYBOOK_DIR: $CI_PROJECT_DIR
+
+  needs:
+    - Playbook - Install
+
+  artifacts:
+    untracked: false
+    when: always
+    access: all
+    expire_in: "3 days"
+    paths:
+      - test_results.json
+
+  rules:
+
+    - if: $CI_COMMIT_TAG
+      allow_failure: true
+      when: on_success
+
+    # Needs to run, even by bot as the test results need to be available
+    # - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+    #   when: never
+
+    - if: # Occur on merge
+        $CI_COMMIT_BRANCH
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      allow_failure: true
+      when: on_success
+
+    # - if:
+    #     $CI_COMMIT_BRANCH != "development"
+    #       &&
+    #     $CI_COMMIT_BRANCH != "master"
+    #       &&
+    #     $CI_PIPELINE_SOURCE == "push"
+    #   allow_failure: true
+    #   when: always
+
+    - when: never

.gitlab/integration_test_trace.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# colour ref: https://stackoverflow.com/a/28938235
+
+NC='\033[0m'       # Text Reset
+
+# Regular Colors
+Black='\033[0;30m'        # Black
+Red='\033[0;31m'          # Red
+Green='\033[0;32m'        # Green
+Yellow='\033[0;33m'       # Yellow
+Blue='\033[0;34m'         # Blue
+Purple='\033[0;35m'       # Purple
+Cyan='\033[0;36m'         # Cyan
+
+
+cmd() {
+
+  echo -e "${Yellow}[TRACE] ${Green}executing ${Cyan}'$1'${NC}"
+
+  docker exec -i test_image_${CI_JOB_ID} $1 || true
+
+}
+
+
+cmd "journalctl -xeu netfilter-persistent.service";
+
+cmd "journalctl -xeu iptables.service"
+
+cmd "journalctl -xeu k3s.service"
+
+cmd "systemctl status netfilter-persistent.service"
+
+cmd "systemctl status iptables.service"
+
+cmd "systemctl status k3s.service"
+
+cmd "kubectl get po -A -o wide"
+
+cmd "kubectl get no -o wide"
+
+cmd "iptables -nvL --line-numbers"

.gitlab/merge_request_templates/default.md

@@ -0,0 +1,22 @@
+### :books: Summary
+<!-- your summary here emojis ref: https://github.com/yodamad/gitlab-emoji -->
+
+
+
+### :link: Links / References
+<!-- using a list as any links to other references or links as required. if relevent, describe the link/reference -->
+
+
+### :construction_worker: Tasks
+
+ - [ ] Add your tasks here if required (delete)
+
+<!-- dont remove tasks below strike through including the checkbox by enclosing in double tidle '~~' -->
+
+- [ ] Playbook Update
+
+    This collection has a [corresponding playbook](https://gitlab.com/nofusscomputing/projects/ansible/ansible_playbooks/-/blob/development/role.yaml) that may need to be updated (Ansible Role), specifically [Role Validation](https://gitlab.com/nofusscomputing/projects/ansible/ansible_playbooks/-/blob/development/tasks/role/validation/nfc_kubernetes.yaml).
+
+- [ ] NetBox Rendered Config Update
+
+    This Collection has a [NetBox Rendered Config template](https://gitlab.com/nofusscomputing/infrastructure/configuration-management/netbox/-/blob/development/templates/cluster.json.j2) that may need to be updated. Specifically Section `cluster.type == 'kubernetes'`

.gitlab/test_results.yaml

@@ -0,0 +1,19 @@
+---
+
+- name: Create Test Results File
+  hosts: localhost
+  gather_facts: false
+
+
+  tasks:
+    
+
+    - name: Load Test Results
+      ansible.builtin.include_vars:
+        dir: ../test_results
+        name: test_results
+
+    - name: Create Results file
+      ansible.builtin.copy:
+        content: "{{ (test_results) | to_nice_json }}"
+        dest: ../test_results.json

.vscode/settings.json

@@ -1,13 +1,14 @@
 {
     "yaml.schemas": {
         "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/tasks": [
-            "roles/nfc_firewall/tasks/*.yaml",
-            "roles/nfc_firewall/tasks/*/*.yaml",
-            "roles/nfc_firewall/tasks/*/*/*.yaml"
+            "roles/nfc_kubernetes/tasks/*.yaml",
+            "roles/nfc_kubernetes/tasks/*/*.yaml",
+            "roles/nfc_kubernetes/tasks/*/*/*.yaml"
         ],
         "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/vars.json": [
             "roles/nfc_kubernetes/variables/**.yaml"
-        ]
+        ],
+        "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/playbook": ".gitlab/test_results.yaml"
     },
     "gitlab.aiAssistedCodeSuggestions.enabled": false,
     "gitlab.duoChat.enabled": false,

CHANGELOG.md

@@ -1,76 +1,74 @@
-## 1.3.0 (2024-03-18)
+## 1.13.1 (2024-07-28)
+
+### Fix
+
+- **nfc_kubernetes**: prime node to always be labeled prime
+
+## 1.13.0 (2024-07-28)
 
 ### Feat
 
-- dont attempt to install if already installed
+- **nfc_kubernetes**: prime node to always be labeled prime
+- **nfc_kubernetes**: ability to add node labels and taints
+
+## 1.12.0 (2024-07-08)
 
 ### Fix
 
-- **handler**: add missing 'reboot_host' handler
-- **firewall**: ensure slave nodes can access ALL masters API point
-- **firewall**: dont add rules for disabled features
+- **kubernetes_roles**: conditional checks for prime
+- **kubernetes_roles**: conditional checks for prime
+- **kubernetes_roles**: conditional checks for prime
+- **kubernetes_roles**: clean up white space
+- **kubernetes_roles**: clean up white space
+- **kubernetes_roles**: clean up white space
+- **kubernetes_roles**: clean up white space
+- **kubernetes_role**: delete leftover ]
+- **kubernetes_role**: Change "https://" + hostvars[ns.prime_name].ansible_host + ":6443" -> "https://" + ns.prime_name.ansible_host + ":6443"
+- **kubernetes_role**: get prime hostname
+- **kubernetes_role**: set server var -> "https://" + hostvars[nfc_role_kubernetes_node_prime].ansible_host + ":6443"
+- **kubernetes_role**: remove not nfc_role_kubernetes_cluster_upgraded | default(true) | bool section
+- **kubernetes_role**: undo previous change
 
-## 1.2.0 (2024-03-16)
+## 1.11.0 (2024-06-27)
 
 ### Feat
 
-- **firewall**: use collection nofusscomputing.firewall to configure kubernetes firewall
+- **firewall**: update collection nfc_firewall 1.1.0 -> 1.1.1
+
+## 1.10.3 (2024-06-27)
 
 ### Fix
 
-- **config**: use correct var name when setting node name
+- **install**: ensure ipv6 is installed before attempting to disable
 
-## 1.1.2 (2024-03-13)
+## 1.10.2 (2024-05-03)
 
 ### Fix
 
-- **readme**: update gitlab links to new loc
-- **configure**: dont attempt to configure firewall if install=false
-- **handler**: remove old k8s code causing handler to fail
-- **handler**: kubernetes restart handler now using updated node type vars
-- **config**: if hostname=localhost use hostname command to fetch hostname
-- limit the use of master group
-- add missing dependency ansible.posix
-- **install**: use correct var type for packages
+- **nfc_kubernetes**: set default for var so task 'Copy Template' when clause doesn't fail task with undefined var
 
-## 1.1.1 (2024-03-13)
+## 1.10.1 (2024-05-03)
 
 ### Fix
 
-- don't check hostname for localhost
+- **nfc_kubernetes**: set default for var so when clause doesn't fail task with undefined var
 
-## 1.1.0 (2024-03-13)
+## 1.10.0 (2024-05-03)
 
 ### Feat
 
-- add role readme and fix gitlab release job
-
-## 1.0.1 (2024-03-13)
+- **nfc_kubernetes**: new variable 'nfc_role_kubernetes_node_prime' to denote the hostname of the prime node
 
 ### Fix
 
-- **ci**: ensure correct package name is used
-
-## 1.0.0 (2024-03-13)
-
-### Feat
-
-- **playbook**: add the install playbook
+- **nfc_kubernetes**: correct 'Create Required directories' when logic
 
 ### Refactor
 
-- **nfc_kubernetes**: update meta file
-- remove dependency on role nfc_common
-- **nfc_kubernetes**: layout role ingress to install prime -> master -> worker nodes as separate groups
-- **docs**: restructure docs
+- **nfc_kubernetes**: remove usage of prime node name over is_prime var
 
-## 0.3.0 (2024-03-13)
+## 1.9.0 (2024-05-03)
 
 ### Feat
 
-- remove old var and update kube version
-- install helm binary
-
-### Refactor
-
-- image var update for calico
+- **nfc_kubernetes**: add debug out to k3s download on failure

README.md

@@ -29,12 +29,17 @@ This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/a
 
 ![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fmaster%2F.cz.yaml) 
 
+![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814)
+
+
 ----
 
 **Development Branch** 
 
 ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fdevelopment%2F.cz.yaml)
 
+![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fdevelopment%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fdevelopment%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fdevelopment%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fdevelopment%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814)
+
 ----
 <br>
 

docs/projects/ansible/collection/kubernetes/index.md

@@ -13,6 +13,8 @@ about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernete
 
 ![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic)
 
+![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814)
+
 
 [![Downloads](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fkubernetes%2F&query=%24.download_count&style=plastic&logo=ansible&logoColor=white&label=Galaxy%20Downloads&labelColor=black&color=cyan)](https://galaxy.ansible.com/ui/repo/published/nofusscomputing/kubernetes/)
 
@@ -29,14 +31,20 @@ To install this collection use `ansible-galaxy collection install nofusscomputin
 
 ## Features
 
-Most of the features of this collection are from the included role `nfc_kubernetes`, please [view its page for feature details](roles/nfc_kubernetes/index.md).
+- Install k3s cluster. Both Single and multi-node clusters
+
+- Configure the cluster
+
+- Upgrade a cluster
+
+For a more detailed list of featured checkout the roles [documentation](roles/nfc_kubernetes/index.md).
 
 
 ## Using this collection
 
 This collection has been designed to be a complete and self-contained management tool for a K3s kubernetes cluster.
 
-## K3s Kubernetes Installation
+## Cluster Installation
 
 By default the install playbook will install to localhost.
 
@@ -64,4 +72,12 @@ The install playbook has a dynamic `hosts` key. This has been done to specifical
 For the available variables please view the [nfc_kubernetes role docs](roles/nfc_kubernetes/index.md#default-variables)
 
 
+## Cluster Upgrade
 
+[In place cluster upgrades](https://docs.k3s.io/upgrades/manual#upgrade-k3s-using-the-binary) is the method used to conduct the cluster upgrades. The logic for the upgrades first confirms that K3s is installed and that the local binary and running k3s version are the desired versions. If they are not, they will be updated to the desired version. On completion of this the node has its `k3s` service restarted which completes the upgrade process.
+
+!!! info
+    If an upgrade occurs, no other task within the play will run. This is by design. if you have further tasks to be run in addition to the upgrade, run the play again.
+
+!!! danger
+    not following the [Kubernetes version skew policy](https://kubernetes.io/releases/version-skew-policy/) when upgrading your cluster may break your cluster.

docs/projects/ansible/collection/kubernetes/roles/kubernetes_netbox/index.md

@@ -0,0 +1,46 @@
+---
+title: NetBox Kubernetes
+description: No Fuss Computings Ansible role kubernetes_netbox
+date: 2023-10-24
+template: project.html
+about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
+---
+
+This Ansible role as part of our collection `nofusscomputing.kubernetes` is intended to be used to setup NetBox so that the settings for deploying a kubernetes cluster can be stored within NetBox.
+
+
+## Role Details
+
+| Item| Value | Description |
+|:---|:---:|:---|
+| Dependent Roles | _None_ | |
+| Optional Roles | _None_ | |
+| Idempotent | _Yes_ |  |
+| Stats Available | _Not Yet_ |  |
+| Tags | _Nil_ |  |
+| Requirements | _None_  |  |
+
+
+## Features
+
+- Adds custom fields to `cluster` object within NetBox that this collection can use to deploy a kubernetes cluster.
+
+!!! info
+    Due to a bug in ansible module `netbox.netbox.netbox_custom_field` The fields are not created as they should be. For example, the fields are supposed to be set to only display when not empty. for more information see [Github #1210](https://github.com/netbox-community/ansible_modules/issues/1210). We have [added a workaround](https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes/-/merge_requests/56#note_1876912267) so the fields are created.
+    
+    Other than that, the fields are created as they should.
+
+
+## Usage
+
+To configure NetBox, ensure that the NetBox Access variables are set and run playbook `nofusscomputing.netbox.kubernetes_netbox`. This will setup NetBox with the required fields that role [nfc_kubernetes](../nfc_kubernetes/index.md) uses.
+
+
+## Default Variables
+
+
+``` yaml title="defaults/main.yaml" linenums="1"
+
+--8<-- "roles/kubernetes_netbox/defaults/main.yaml"
+
+```

docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md

@@ -70,6 +70,8 @@ This Ansible role is designed to deploy a K3s Kubernetes cluster. Without adding
 
 - Install the Helm Binary
 
+- Upgrade cluster
+
 
 ## Role Workflow
 

galaxy.yml

@@ -8,7 +8,7 @@ namespace: nofusscomputing
 name: kubernetes
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 1.3.0
+version: 1.13.1
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md
@@ -44,9 +44,10 @@ tags:
 # L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version
 # range specifiers can be set and are separated by ','
 dependencies:
-  ansible.posix: '1.5.4'
-  kubernetes.core: '3.0.0'
-  nofusscomputing.firewall: '1.0.1'
+  ansible.posix: '>=1.5.4'
+  kubernetes.core: '>=3.0.0'
+  nofusscomputing.firewall: '>=1.1.1'
+  netbox.netbox: '>=3.16.0'
 
 
 # The URL of the originating SCM repository

mkdocs.yml

@@ -49,6 +49,9 @@ nav:
 
           - projects/ansible/collection/kubernetes/roles/nfc_kubernetes/release_notes.md
 
+        - Role kubernetes_netbox:
+
+          - projects/ansible/collection/kubernetes/roles/kubernetes_netbox/index.md
 
 - Operations: 
 

playbooks/netbox.yaml

@@ -0,0 +1,64 @@
+---
+- name: Install K3s Kubernetes
+  hosts: |-
+    {%- if nfc_pb_host is defined -%}
+
+      {{ nfc_pb_host }}
+
+    {%- elif nfc_pb_kubernetes_cluster_name is defined -%}
+
+      kubernetes_cluster_{{ nfc_pb_kubernetes_cluster_name | lower }}
+
+    {%- else -%}
+
+      {%- if ansible_limit is defined -%}
+
+        {{ ansible_limit }}
+
+      {%- else -%}
+
+        localhost
+
+      {%- endif -%}
+
+    {%- endif %}
+  become: false
+  gather_facts: false
+
+
+  tasks:
+
+
+    - name: Configure NetBox for Kubernetes Deployment(s)
+      ansible.builtin.include_role:
+        name: kubernetes_netbox
+      tags:
+        - always
+
+
+  # vars:
+
+  #
+  #   Future feature, add playbook to import to awx
+  #
+  #   nfc_pb_awx_tower_template:
+
+
+  #     - name: "Collection/NoFussComputing/Kubernetes/NetBox/Configure"
+  #       ask_credential_on_launch: true
+  #       ask_job_type_on_launch: true
+  #       ask_limit_on_launch: true
+  #       ask_tags_on_launch: true
+  #       ask_variables_on_launch: true
+  #       description: |
+  #         Playbook to Install/Configure Kubernetes using configuration
+  #         from code.
+  #       execution_environment: "No Fuss Computing EE"
+  #       job_type: "check"
+  #       labels:
+  #         - cluster
+  #         - k3s
+  #         - kubernetes
+  #       verbosity: 2
+  #       use_fact_cache: true
+  #       survey_enabled: false

requirements.txt

@@ -0,0 +1,2 @@
+pynetbox
+pytz

roles/defaults/main.yaml

@@ -0,0 +1,9 @@
+---
+
+#
+# NetBox Access Variables. Required
+#
+
+# nfc_pb_api_netbox_url:                   # ENV [NETBOX_API]
+# nfc_pb_api_netbox_token:                 # ENV [NETBOX_TOKEN]
+# nfc_pb_api_netbox_validate_cert: true    # ENV [NETBOX_VALIDATE_CERT]

roles/kubernetes_netbox/README.md

@@ -0,0 +1,3 @@
+## No Fuss Computing - Ansible Role kubernetes_netbox
+
+Nothing to see here

roles/kubernetes_netbox/meta/main.yaml

@@ -0,0 +1,30 @@
+galaxy_info:
+
+  role_name: kubernetes_netbox
+
+  author: No Fuss Computing
+
+  description: Configure the required items within Netbox to support deploying kubernetes from Netbox configuration.
+
+  issue_tracker_url: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
+
+  license: MIT
+
+  min_ansible_version: '2.15'
+
+  platforms:
+
+    - name: Debian
+      versions:
+        - bullseye
+        - bookworm
+
+    - name: Ubuntu
+      versions:
+        - 21
+
+  galaxy_tags:
+    - cluster
+    - k3s
+    - kubernetes
+    - netbox

roles/kubernetes_netbox/tasks/cluster.yaml

@@ -0,0 +1,255 @@
+---
+
+# add cluster type kubernetes
+
+- name: Create Custom Field - Configure Firewall
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Configure Firewall
+      name: nfc_role_kubernetes_configure_firewall
+      type: boolean
+      ui_visibility: 'hidden-ifunset'
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - ETCD Enabled
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: ETCD Enabled
+      name: nfc_role_kubernetes_etcd_enabled
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Install OLM
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Install OLM
+      name: nfc_role_kubernetes_install_olm
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Install Helm
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Install Helm
+      name: nfc_role_kubernetes_install_helm
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Install KubeVirt
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Install KubeVirt
+      name: nfc_role_kubernetes_install_kubevirt
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - KubeVirt Operator Replicas
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: KubeVirt Operator Replicas
+      name: nfc_role_kubernetes_kubevirt_operator_replicas
+      type: integer
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      validation_minimum: 1
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Enable MetalLB
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Enable MetalLB
+      name: nfc_kubernetes_enable_metallb
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Enable ServiceLB (klipper)
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Enable ServiceLB (klipper)
+      name: nfc_kubernetes_enable_servicelb
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Pod Subnet
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Pod Subnet
+      name: nfc_role_kubernetes_pod_subnet
+      object_type: ipam.prefix
+      type: object
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Service Subnet
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Service Subnet
+      name: nfc_role_kubernetes_service_subnet
+      object_type: ipam.prefix
+      type: object
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp

roles/kubernetes_netbox/tasks/main.yaml

@@ -0,0 +1,21 @@
+---
+
+- name: Setup NetBox for Kubernetes Cluster Deployments
+  ansible.builtin.include_tasks:
+    file: cluster.yaml
+    apply:
+      tags:
+        - always
+  tags:
+    - always
+
+
+- name: Setup NetBox for Kubernetes Service Deployments
+  ansible.builtin.include_tasks:
+    file: services.yaml
+    apply:
+      tags:
+        - always
+  tags:
+    - never
+    - services

roles/kubernetes_netbox/tasks/services.yaml

@@ -0,0 +1,50 @@
+---
+
+- name: Create Custom Field - Instance
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - ipam.service
+      group_name: Kubernetes
+      label: Instance Name
+      description: "Name of the Instance to be deployed"
+      name: service_kubernetes_instance
+      type: text
+      ui_visibility: hidden-ifunset
+      # is_cloneable: true
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Namespace
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - ipam.service
+      group_name: Kubernetes
+      label: Service Namespace
+      description: "Deployment Namespace"
+      name: service_kubernetes_namespace
+      type: text
+      ui_visibility: hidden-ifunset
+      # is_cloneable: true
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp

roles/nfc_kubernetes/defaults/main.yml

@@ -46,13 +46,19 @@ nfc_role_kubernetes_install_kubevirt: false
 
 nfc_role_kubernetes_kubevirt_operator_replicas: 1
 
+nfc_role_kubernetes_node_labels: {}     # Optional, Dict. Node labels.
+nfc_role_kubernetes_node_taints: {}     # Optional, Dict. Node taints.
+# nfc_role_kubernetes_node_prime: ''    # Mandatory*, string. the inventory_hostname of the prime node. ONLY required for multi-node deployments
+
 nfc_role_kubernetes_oidc_enabled: false
 
+nfc_role_kubernetes_resolv_conf_file: /etc/resolv.conf
+
 nfc_role_kubernetes_pod_subnet: 172.16.248.0/21
 nfc_role_kubernetes_service_subnet: 172.16.244.0/22
 
-nfc_role_kubernetes_prime: true                         # Mandatory for a node designated as the prime master node
-nfc_role_kubernetes_master: true                        # Mandatory for a node designated as a master node and the prime master node
+nfc_role_kubernetes_prime: false                         # Mandatory for a node designated as the prime master node
+nfc_role_kubernetes_master: false                        # Mandatory for a node designated as a master node and the prime master node
 nfc_role_kubernetes_worker: false                       # Mandatory for a node designated as a worker node
 
 ############################################################################################################
@@ -134,7 +140,7 @@ k3s:
         #       usernames: []
         #       runtimeClasses: []
         #       namespaces: [kube-system]
-      when: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname }}"
+      when: "{{ nfc_role_kubernetes_prime | bool }}"
 
 
 #############################################################################################

roles/nfc_kubernetes/tasks/helm/main.yaml

@@ -5,6 +5,9 @@
     url: https://baltocdn.com/helm/signing.asc
     dest: /usr/share/keyrings/helm.asc
     mode: 740
+  changed_when: not ansible_check_mode
+  delay: 10
+  retries: 3
 
 
 - name: Add Helm Repository

roles/nfc_kubernetes/tasks/install.yaml

@@ -4,7 +4,10 @@
   ansible.builtin.command:
     cmd: hostname
   changed_when: false
+  check_mode: false
   register: hostname_to_check
+  tags:
+    - always
 
 
 - name: Hostname Check
@@ -12,13 +15,45 @@
     that:
       - hostname_to_check.stdout == inventory_hostname
     msg: The hostname must match the inventory_hostname
+  tags:
+    - always
   when: >
     inventory_hostname != 'localhost'
 
 
+- name: Testing Env Variables
+  ansible.builtin.set_fact:
+    ansible_default_ipv4: {
+      "address": "127.0.0.1"
+    }
+  check_mode: false
+  tags:
+    - always
+  when: >
+    lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != ''
+
+
+- name: Gather Facts required by role
+  ansible.builtin.setup:
+    gather_subset:
+      - all_ipv4_addresses
+      - os_family
+      - processor
+  tags:
+    - always
+  when: >
+    ansible_architecture is not defined
+      or
+    ansible_default_ipv4 is not defined
+      or
+    ansible_os_family is not defined
+
+
 - name: Check Machine Architecture
   ansible.builtin.set_fact:
     nfc_kubernetes_install_architectures: "{{ nfc_kubernetes_install_architectures | default({}) | combine({ansible_architecture: ''}) }}"
+  tags:
+    - always
 
 
 - name: Configure Kubernetes Firewall Rules
@@ -33,6 +68,19 @@
     nfc_role_kubernetes_configure_firewall
 
 
+- name: Install required software
+  ansible.builtin.apt:
+    name: python3-pip
+    install_recommends: false
+    state: present
+  when: >
+    install_kubernetes | default(true) | bool
+      and
+    not kubernetes_installed | default(false) | bool
+  tags:
+    - always
+
+
 - name: K3s Install
   ansible.builtin.include_tasks:
     file: k3s/install.yaml
@@ -57,6 +105,8 @@
     install_kubernetes | default(true) | bool
       and
     kubernetes_installed | default(false) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
   tags:
     - always
 
@@ -73,6 +123,8 @@
     kubernetes_config.kube_virt.enabled | default(nfc_role_kubernetes_install_kubevirt)
       and
     inventory_hostname in kubernetes_config.kube_virt.nodes | default([ inventory_hostname ]) | list
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
   tags:
     - always
 
@@ -89,5 +141,7 @@
     kubernetes_config.helm.enabled | default(nfc_role_kubernetes_install_helm)
       and
     nfc_role_kubernetes_master
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
   tags:
     - always

roles/nfc_kubernetes/tasks/k3s/configure.yaml

@@ -34,13 +34,13 @@
 
       - src: kubernetes-manifest-rbac.yaml.j2
         dest: /var/lib/rancher/k3s/server/manifests/rbac-authorization-common.yaml
-        when: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname }}"
+        when: "{{ nfc_role_kubernetes_prime | bool }}"
 
       - src: iptables-kubernetes.rules.j2
         dest: "/etc/iptables-reloader/rules.d/iptables-kubernetes.rules"
         notify: firewall_reloader
         when: |-
-          {%- if firewall_installed -%}
+          {%- if firewall_installed | default(false) -%}
 
             {{ firewall_rules_dir_metadata.stat.exists }}
 
@@ -58,21 +58,41 @@
       kind: Node
       metadata:
         name: "{{ inventory_hostname }}"
-        {% if kubernetes_config.hosts[inventory_hostname].labels | default([]) | list | length > 0 -%}
+        {% if
+          nfc_role_kubernetes_node_labels
+            and
+          not nfc_role_kubernetes_prime | bool
+        -%}
         labels:
-          {{ kubernetes_config.hosts[inventory_hostname].labels | to_nice_yaml | indent(4) }}
+          {{ nfc_role_kubernetes_node_labels | to_nice_yaml(indent=0) | indent(4) }}
+
+        { elif
+          nfc_role_kubernetes_prime | bool
+        %}
+
+        labels:
+          node-role.kubernetes.io/prime: "true"
+
+          {% if nfc_role_kubernetes_node_labels %}
+
+          {{ nfc_role_kubernetes_node_labels | to_nice_yaml(indent=0) | indent(4) }}
+
+          {% endif %}
+
         {%- endif +%}
-      {% if kubernetes_config.hosts[inventory_hostname].taints | default([]) | list | length > 0 -%}
+      {% if nfc_role_kubernetes_node_taints -%}
       spec:
         taints:
-          {{ kubernetes_config.hosts[inventory_hostname].taints | to_nice_yaml(indent=0) | indent(4) }}
+          {{ nfc_role_kubernetes_node_taints | to_nice_yaml(indent=0) | indent(4) }}
       {% endif %}
     dest: /var/lib/rancher/k3s/server/manifests/node-manifest-{{ inventory_hostname }}.yaml
     owner: root
     group: root
     mode: '700'
-  delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
+  delegate_to: "{{ nfc_role_kubernetes_node_prime }}"
   when:
-    kubernetes_config.hosts[inventory_hostname].labels | default([]) | list | length > 0
+    nfc_role_kubernetes_node_labels
       or
-    kubernetes_config.hosts[inventory_hostname].taints | default([]) | list | length > 0
+    nfc_role_kubernetes_node_taints
+      or
+    nfc_role_kubernetes_prime | bool

roles/nfc_kubernetes/tasks/k3s/install.yaml

@@ -1,15 +1,30 @@
 ---
 
+- name: Install required python modules
+  ansible.builtin.pip:
+    name: "{{ item }}"
+    state: present
+  loop: "{{ pip_packages }}"
+  vars:
+    pip_packages:
+      - kubernetes>=12.0.0
+      - PyYAML>=3.11
+
+
 - name: Check for calico deployment manifest
   ansible.builtin.stat:
     name: /var/lib/rancher/k3s/server/manifests/calico.yaml
   register: file_calico_yaml_metadata
+  when: >
+    nfc_role_kubernetes_prime | bool
 
 
 - name: Check for calico Operator deployment manifest
   ansible.builtin.stat:
     name: /var/lib/rancher/k3s/ansible/deployment-manifest-calico_operator.yaml
   register: file_calico_operator_yaml_metadata
+  when: >
+    nfc_role_kubernetes_prime | bool
 
 
 - name: Install dependent packages
@@ -21,6 +36,7 @@
     loop_var: package
   vars:
     packages:
+      - wget
       - curl
       - iptables
       - jq
@@ -41,14 +57,29 @@
     - install
 
 
-- name: Disable swap
-  ansible.builtin.command:
-    cmd: swapoff -a
-  changed_when: false
-  when:
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
+- name: Testing Environment try/catch
+  block:
+
+
+    - name: Disable swap
+      ansible.builtin.command:
+        cmd: swapoff -a
+      changed_when: false
+      when:
+        - ansible_os_family == 'Debian'
+      tags:
+        - install
+
+
+  rescue:
+
+    - name: Check if inside Gitlab CI
+      ansible.builtin.assert:
+        that:
+          - lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != ''
+        success_msg: "Inside testing enviroment, 'Disable swap' error OK"
+        fail_msg: "You should figure out what went wrong"
+
 
 - name: Check an armbian os system
   ansible.builtin.stat:
@@ -81,15 +112,30 @@
       - name: /var/lib/rancher/k3s/server/logs
         state: directory
         mode: 700
+        when: >
+          {{ nfc_role_kubernetes_master | bool }}
       - name: /var/lib/rancher/k3s/server/manifests
         state: directory
         mode: 700
+        when: >
+          {{ nfc_role_kubernetes_master | bool }}
       - name: /var/lib/rancher/k3s/ansible
         state: directory
         mode: 700
+        when: >
+          {{ nfc_role_kubernetes_master | bool }}
+  when: >
+    item.when | default(true)
 
 
-- name: Add sysctl net.ipv4.ip_forward
+- name: Check if IPv6 Enabled
+  ansible.builtin.stat:
+    path: /proc/sys/net/ipv6/conf/all/disable_ipv6
+  register: ipv6_file
+  failed_when: false
+
+
+- name: Add sysctl settings
   ansible.posix.sysctl:
     name: "{{ item.name }}"
     value: "{{ item.value }}"
@@ -108,8 +154,11 @@
         value: '512'
       - name: net.ipv6.conf.all.disable_ipv6
         value: '1'
-  when:
-    - ansible_os_family == 'Debian'
+        when: "{{ ipv6_file.stat.exists }}"
+  when: >
+    ansible_os_family == 'Debian'
+      and
+    item.when | default(true) | bool
 
 
 - name: Check for Network Manager Directory
@@ -138,28 +187,245 @@
   when: directory_network_manager_metadata.stat.exists
 
 
-- name: Check if K3s Installed
+- name: File Metadata - k3s binary
+  ansible.builtin.stat:
+    checksum_algorithm: sha256
+    name: /usr/local/bin/k3s
+  register: metadata_file_k3s_existing_binary
+
+
+- name: File Metadata - k3s[-agent].service
+  ansible.builtin.stat:
+    checksum_algorithm: sha256
+    name: |-
+      /etc/systemd/system/k3s
+      {%- if not nfc_role_kubernetes_master | default(false) | bool -%}
+        -agent
+      {%- endif -%}
+      .service
+  register: metadata_file_k3s_service
+
+
+- name: Directory Metadata - /etc/rancher/k3s/k3s.yaml
+  ansible.builtin.stat:
+    name: /etc/rancher/k3s/k3s.yaml
+  register: metadata_dir_etc_k3s
+
+
+- name: File Metadata - /var/lib/rancher/k3s/server/token
+  ansible.builtin.stat:
+    checksum_algorithm: sha256
+    name: /var/lib/rancher/k3s/server/token
+  register: metadata_file_var_k3s_token
+
+
+- name: Config Link
   ansible.builtin.shell:
-    cmd: |
-      if [[ $(service k3s status) ]]; then exit 0; else exit 1; fi
-    executable: /bin/bash
-  changed_when: false
-  failed_when: false
-  register: k3s_installed
+    cmd: >
+      ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config
+    executable: bash
+    creates: ~/.kube/config
   when: >
     nfc_role_kubernetes_master | default(false) | bool
+      and
+    metadata_dir_etc_k3s.stat.exists | default(false) | bool
 
 
-- name: Check if K3s Installed
+- name: Fetch Kubernetes Node Object
+  kubernetes.core.k8s_info:
+    kind: Node
+    name: "{{ inventory_hostname }}"
+  register: kubernetes_node
+  when: >
+    metadata_file_k3s_existing_binary.stat.exists | default(false) | bool
+      and
+    metadata_file_k3s_service.stat.exists | default(false) | bool
+      and
+    metadata_dir_etc_k3s.stat.exists | default(false) | bool
+      and
+    metadata_file_var_k3s_token.stat.exists | default(false) | bool
+
+
+- name: Fetch Installed K3s Metadata
   ansible.builtin.shell:
     cmd: |
-      if [[ $(service k3s-agent status) ]]; then exit 0; else exit 1; fi
+      export installed_version=$(k3s --version | grep k3s | awk '{print $3}');
+      export installed="
+      {%- if
+        metadata_file_k3s_existing_binary.stat.exists | default(false) | bool
+          and
+        metadata_file_k3s_service.stat.exists | default(false) | bool
+          and
+        metadata_dir_etc_k3s.stat.exists | default(false) | bool
+          and
+        metadata_file_var_k3s_token.stat.exists | default(false) | bool
+      -%}
+        true
+      {%- else -%}
+        false
+      {%- endif -%}";
+
+      if ! service k3s status > /dev/null; then
+
+        export installed='false';
+
+      fi
+
+      export running_version="{{ kubernetes_node.resources[0].status.nodeInfo.kubeletVersion | default('0') }}";
+
+      export correct_hash=$(wget -q https://github.com/k3s-io/k3s/releases/download/v
+        {{-KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}}
+          /sha256sum-
+          {%- if ansible_architecture | lower == 'x86_64' -%}
+            amd64
+          {%- elif ansible_architecture | lower == 'aarch64' -%}
+            arm64
+          {%- endif %}.txt -O - | grep -v 'images' | awk '{print $1}');
+
+      cat <<EOF
+      {
+        "current_hash": "{{ metadata_file_k3s_existing_binary.stat.checksum | default('') }}",
+        "current_version": "${installed_version}",
+        "desired_hash": "${correct_hash}",
+        "desired_version": "v{{ KubernetesVersion + KubernetesVersion_k3s_prefix | default('') }}",
+        "installed": ${installed},
+        "running_version": "${running_version}"
+      }
+
+      EOF
+
     executable: /bin/bash
   changed_when: false
+  check_mode: false
   failed_when: false
-  register: k3s_installed
+  register: k3s_metadata
+
+
+- name: K3s Metadata Fact
+  ansible.builtin.set_fact:
+    node_k3s: "{{ k3s_metadata.stdout | from_yaml }}"
+
+
+- name: Cached K3s Binary Details
+  ansible.builtin.stat:
+    path: "/tmp/k3s.{{ ansible_architecture }}"
+    checksum_algorithm: sha256
+  delegate_to: localhost
+  register: file_cached_k3s_binary
+  vars:
+    ansible_connection: local
+
+
+- name: Remove Cached K3s Binaries
+  ansible.builtin.file:
+    path: "/tmp/k3s.{{ ansible_architecture }}"
+    state: absent
+  delegate_to: localhost
+  vars:
+    ansible_connection: local
   when: >
-    not nfc_role_kubernetes_worker | default(false) | bool
+    file_cached_k3s_binary.stat.checksum | default('0') != node_k3s.desired_hash
+
+- name: Try / Catch
+  block:
+
+
+    - name: Download K3s Binary
+      ansible.builtin.uri:
+        url: |-
+          https://github.com/k3s-io/k3s/releases/download/
+            {{- node_k3s.desired_version | urlencode -}}
+          /k3s
+          {%- if cpu_arch.key == 'aarch64' -%}
+          -arm64
+          {%- endif %}
+        method: GET
+        return_content: false
+        status_code:
+          - 200
+          - 304
+        dest: "/tmp/k3s.{{ ansible_architecture }}"
+        mode: "744"
+      changed_when: not ansible_check_mode
+      check_mode: false
+      delay: 10
+      retries: 3
+      register: k3s_download_files
+      delegate_to: localhost
+      failed_when: >
+        (lookup('ansible.builtin.file', '/tmp/k3s.' + ansible_architecture) | hash('sha256') | string) != node_k3s.desired_hash
+          and
+        (
+          k3s_download_files.status | int != 200
+            or
+          k3s_download_files.status | int != 304
+        )
+      run_once: true
+      when: ansible_os_family == 'Debian'
+      loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
+      loop_control:
+        loop_var: cpu_arch
+      vars:
+        ansible_connection: local
+
+  rescue:
+
+
+    - name: TRACE - Debug out
+      ansible.builtin.debug:
+        msg:
+          - "Download file hash: {{ (lookup('ansible.builtin.file', '/tmp/k3s.' + ansible_architecture) | hash('sha256') | string) }}"
+
+
+    - name: Fail task
+      ansible.builtin.assert:
+        that:
+          - false
+        msg: "Task failed, review previous task for error"
+
+
+- name: Copy K3s binary to Host
+  ansible.builtin.copy:
+    src: "/tmp/k3s.{{ ansible_architecture }}"
+    dest: "/usr/local/bin/k3s"
+    mode: '741'
+    owner: root
+    group: root
+  register: k3s_binary_copy
+  when: >
+    node_k3s.current_hash != node_k3s.desired_hash
+
+
+- name: K3s Binary Upgrade
+  ansible.builtin.service:
+    name: |-
+      {%- if nfc_role_kubernetes_master | default(false) | bool -%}
+        k3s
+      {%- else -%}
+        k3s-agent
+      {%- endif %}
+    state: restarted
+  register: k3s_upgrade_service_restart
+  when: >
+    (
+      k3s_binary_copy.changed | default(false) | bool
+        and
+      node_k3s.installed | default(false) | bool
+    )
+      or
+    (
+      node_k3s.running_version != node_k3s.desired_version
+        and
+      node_k3s.installed | default(false) | bool
+    )
+
+
+- name: Create Fact - cluster_upgraded
+  ansible.builtin.set_fact:
+    nfc_role_kubernetes_cluster_upgraded: true
+    cacheable: true
+  when: >
+    k3s_upgrade_service_restart.changed | default(false) | bool
 
 
 - name: Download Install Scripts
@@ -172,7 +438,10 @@
       - 304
     dest: "{{ item.dest }}"
     mode: "744"
+  check_mode: false
   changed_when: false
+  delay: 10
+  retries: 3
   register: k3s_download_script
   delegate_to: localhost
   run_once: true
@@ -181,6 +450,8 @@
     ansible_os_family == 'Debian'
       and
     item.when | default(true) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
   loop: "{{ download_files }}"
   vars:
     ansible_connection: local
@@ -192,61 +463,6 @@
         when: "{{ nfc_role_kubernetes_install_olm }}"
 
 
-- name: Download K3s Binary
-  ansible.builtin.uri:
-    url: |-
-      https://github.com/k3s-io/k3s/releases/download/v
-        {{- KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}}
-      /k3s
-      {%- if cpu_arch.key == 'aarch64' -%}
-      -arm64
-      {%- endif %}
-    method: GET
-    return_content: false
-    status_code:
-      - 200
-      - 304
-    dest: "/tmp/k3s.{{ cpu_arch.key }}"
-    mode: "744"
-  changed_when: false
-  register: k3s_download_files
-  delegate_to: localhost
-  run_once: true
-  # no_log: true
-  when: ansible_os_family == 'Debian'
-  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
-  loop_control:
-    loop_var: cpu_arch
-  vars:
-    ansible_connection: local
-
-
-- name: "[TRACE] Downloaded File SHA256"
-  ansible.builtin.set_fact:
-    hash_sha256_k3s_downloaded_binary: "{{ lookup('ansible.builtin.file', '/tmp/k3s.' + cpu_arch.key) | hash('sha256') | string }}"
-  delegate_to: localhost
-  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
-  loop_control:
-    loop_var: cpu_arch
-
-
-- name: Existing k3s File hash
-  ansible.builtin.stat:
-    checksum_algorithm: sha256
-    name: /usr/local/bin/k3s
-  register: hash_sha256_k3s_existing_binary
-
-
-- name: Copy K3s binary to Host
-  ansible.builtin.copy:
-    src: "/tmp/k3s.{{ ansible_architecture }}"
-    dest: "/usr/local/bin/k3s"
-    mode: '741'
-    owner: root
-    group: root
-  when: hash_sha256_k3s_existing_binary.stat.checksum | default('0') != hash_sha256_k3s_downloaded_binary
-
-
 - name: Copy install scripts to Host
   ansible.builtin.copy:
     src: "{{ item.path }}"
@@ -263,6 +479,8 @@
         when: "{{ nfc_role_kubernetes_install_olm }}"
   when: >
     item.when | default(true) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Required Initial config files
@@ -276,7 +494,8 @@
   loop: "{{ k3s.files }}"
   when: >
     item.when | default(true) | bool
-  #   kubernetes_config.cluster.prime.name == inventory_hostname
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Copy Intial required templates
@@ -291,6 +510,8 @@
   diff: true
   when: >
     item.when | default(true) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
   vars:
     templates_to_apply:
       - src: k3s-config.yaml.j2
@@ -300,7 +521,7 @@
         dest: /var/lib/rancher/k3s/server/manifests/calico.yaml
         when: >
           {{
-            kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+            nfc_role_kubernetes_prime | bool
               and
             (
               (
@@ -308,7 +529,7 @@
                   and
                 file_calico_yaml_metadata.stat.exists
                   and
-                k3s_installed.rc == 0
+                not node_k3s.installed | bool
               )
                 or
               'calico_manifest' in ansible_run_tags
@@ -331,23 +552,35 @@
 #     ipv6: true
 
 
+- name: Set IP6Tables to legacy mode
+  ansible.builtin.command:
+    cmd: update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+  changed_when: false
+  when: >
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
+
+
 - name: Set IPTables to legacy mode
   ansible.builtin.command:
     cmd: update-alternatives --set iptables /usr/sbin/iptables-legacy
   changed_when: false
+  when: >
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install K3s (prime master)
   ansible.builtin.shell:
     cmd: |
       INSTALL_K3S_SKIP_DOWNLOAD=true \
-      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
+      INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \
       /tmp/install.sh {% if nfc_role_kubernetes_etcd_enabled %}--cluster-init{% endif %}
   changed_when: false
   when: >
-    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+    nfc_role_kubernetes_prime | bool
       and
-    k3s_installed.rc == 1
+    not node_k3s.installed | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install Calico Operator
@@ -364,12 +597,14 @@
         'operator_calico' in ansible_run_tags
       )
         or
-      not file_calico_yaml_metadata.stat.exists
+      not file_calico_yaml_metadata.stat.exists | default(false)
     )
       and
     'calico_manifest' not in ansible_run_tags
       and
-    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+    nfc_role_kubernetes_prime | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install MetalLB Operator
@@ -388,7 +623,9 @@
   when: >-
     nfc_kubernetes_enable_metallb | default(false) | bool
       and
-    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+    nfc_role_kubernetes_prime | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Wait for kubernetes prime to be ready
@@ -402,7 +639,7 @@
         exit 127;
       fi
     executable: /bin/bash
-  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
+  delegate_to: "{{ nfc_role_kubernetes_node_prime }}"
   run_once: true
   register: kubernetes_ready_check
   retries: 30
@@ -413,6 +650,22 @@
     kubernetes_ready_check.rc != 0
   changed_when: false
   failed_when: kubernetes_ready_check.rc != 0
+  when: >
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
+      and
+    not ansible_check_mode
+
+
+- name: Config Link
+  ansible.builtin.shell:
+    cmd: >
+      ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config
+    executable: bash
+    creates: ~/.kube/config
+  when: >
+    nfc_role_kubernetes_master | default(false) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install olm
@@ -426,9 +679,11 @@
     install_olm.rc == 1
   register: install_olm
   when: >
-    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+    nfc_role_kubernetes_prime | bool
       and
     nfc_role_kubernetes_install_olm | default(false) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Uninstall OLM
@@ -454,9 +709,11 @@
   failed_when: false
   register: install_olm
   when: >
-    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+    nfc_role_kubernetes_prime | bool
       and
     'olm_uninstall' in ansible_run_tags
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Enable Cluster Encryption
@@ -465,10 +722,12 @@
   changed_when: false
   failed_when: false    # New cluster will fail
   when: >
-    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+    nfc_role_kubernetes_prime | bool
       and
     kubernetes_config.cluster.networking.encrypt | default(false) | bool
       and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
+      and
     (
       'calico_manifest' in ansible_run_tags
         or
@@ -483,18 +742,22 @@
 - name: Fetch Join Token
   ansible.builtin.slurp:
     src: /var/lib/rancher/k3s/server/token
-  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
+  delegate_to: "{{ nfc_role_kubernetes_node_prime }}"
   run_once: true
   register: k3s_join_token
   no_log: true # Value is sensitive
+  when: >
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Create Token fact
   ansible.builtin.set_fact:
     k3s_join_token: "{{ k3s_join_token.content | b64decode | replace('\n', '') }}"
-  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
+  delegate_to: "{{ nfc_role_kubernetes_node_prime }}"
   run_once: true
   no_log: true # Value is sensitive
+  when: >
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install K3s (master nodes)
@@ -502,7 +765,7 @@
     cmd: |
       INSTALL_K3S_EXEC="server" \
       INSTALL_K3S_SKIP_DOWNLOAD=true \
-      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
+      INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \
       K3S_TOKEN="{{ k3s_join_token }}" \
       /tmp/install.sh
     executable: /bin/bash
@@ -510,9 +773,11 @@
   when: >
     nfc_role_kubernetes_master | default(false) | bool
       and
-    not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+    not nfc_role_kubernetes_prime | bool
       and
-    k3s_installed.rc == 1
+    not node_k3s.installed | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install K3s (worker nodes)
@@ -521,18 +786,20 @@
       set -o pipefail
       INSTALL_K3S_EXEC="agent" \
       INSTALL_K3S_SKIP_DOWNLOAD=true \
-      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
+      INSTALL_K3S_VERSION="v{{ node_k3s.desired_version }}" \
       K3S_TOKEN="{{ k3s_join_token }}" \
-      K3S_URL="https://{{ hostvars[kubernetes_config.cluster.prime.name | default(inventory_hostname)].ansible_host }}:6443" \
+      K3S_URL="https://{{ hostvars[nfc_role_kubernetes_node_prime].ansible_host }}:6443" \
       /tmp/install.sh -
     executable: /bin/bash
   changed_when: false
   when: >
     not nfc_role_kubernetes_master | default(false) | bool
       and
-    not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+    not nfc_role_kubernetes_prime | bool
       and
-    k3s_installed.rc == 1
+    not node_k3s.installed | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Set Kubernetes Final Install Fact

roles/nfc_kubernetes/tasks/k3s/migrate_to_operator.yaml

@@ -110,6 +110,7 @@
         owner: root
         group: 'root'
       changed_when: false
+      check_mode: false
       become: true
       delegate_to: localhost
       loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"

roles/nfc_kubernetes/tasks/main.yaml

@@ -1,15 +1,29 @@
 ---
 
+- name: Default Variable adjustment [Probable Single Node Install]
+  ansible.builtin.set_fact:
+    cacheable: false
+    nfc_role_kubernetes_prime: true
+    nfc_role_kubernetes_master: true
+    nfc_role_kubernetes_node_prime: "{{ inventory_hostname }}"
+  when: >
+    not nfc_role_kubernetes_worker | bool
+      and
+    not nfc_role_kubernetes_prime | bool
+      and
+    not nfc_role_kubernetes_master | bool
+
+
 - name: Install/Configure Kubernetes Prime Master Node
   ansible.builtin.include_tasks:
     file: install.yaml
   tags:
     - always
   when:
-    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
-      and
     nfc_role_kubernetes_prime | bool
       and
+    not nfc_role_kubernetes_worker | bool
+      and
     not kubernetes_installed | default(false)
 
 
@@ -19,10 +33,12 @@
   tags:
     - always
   when:
-    kubernetes_config.cluster.prime.name | default(inventory_hostname) != inventory_hostname
-      and
     nfc_role_kubernetes_master | bool
       and
+    not nfc_role_kubernetes_prime | bool
+      and
+    not nfc_role_kubernetes_worker | bool
+      and
     not kubernetes_installed | default(false)
 
 

roles/nfc_kubernetes/templates/k3s-config.yaml.j2

@@ -9,9 +9,10 @@
 {%- if 
   nfc_role_kubernetes_master
     or
-  kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+  nfc_role_kubernetes_prime | default(true) | bool
+    and
+  not nfc_role_kubernetes_worker
 -%}
-
   {%
 
   set kube_apiserver_arg = [
@@ -146,6 +147,7 @@
       ",ephemeral-storage=" + kubelet_arg_system_reserved_storage
     ],
     "node-name": node_name,
+    "resolv-conf": nfc_role_kubernetes_resolv_conf_file,
   }
 
 -%}
@@ -153,7 +155,7 @@
 
 {%- if groups[kubernetes_config.cluster.group_name | default('make_me_optional')] | default([]) | list | length > 0 -%}
   
-  {%- if k3s_installed.rc == 0 -%}
+  {%- if node_k3s.installed -%}
 
     {%- set ns = namespace(server=[]) -%}
 
@@ -196,13 +198,28 @@
     }) -%}
 
   {%- elif 
-    kubernetes_config.cluster.prime.name != inventory_hostname
+    nfc_role_kubernetes_prime | default(true) | bool
       and
-    k3s_installed.rc == 1
+    not node_k3s.installed
+      and
+    not nfc_role_kubernetes_worker
+
   -%}
 
+    {%- set ns = namespace(prime_name) -%}
+
+    {%- for hostname, values in hostvars.iteritems() -%}
+      
+      {%- if values.nfc_role_kubernetes_node_prime ==true -%}
+
+        {%- set ns.prime_name = hostname -%}
+
+      {%- endif -%}
+
+    {%- endfor -%}
+    
     {%- set server = (server | default([])) + [
-      "https://" + hostvars[kubernetes_config.cluster.prime.name].ansible_host + ":6443"
+      "https://" + ns.prime_name.ansible_host + ":6443"
     ] -%}
 
     {%- set all_nodes_config = all_nodes_config | combine({
@@ -234,13 +251,18 @@
 
 {%- endif -%}
 
+
 {# EoF All Nodes #}
 
 
 {%- if 
-  nfc_role_kubernetes_master
-    or
-  kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+  (
+    nfc_role_kubernetes_master
+      or
+    nfc_role_kubernetes_prime | default(true) | bool
+  )
+    and
+  not nfc_role_kubernetes_worker
 -%}
 
   {%- set servers_config = servers_config | combine( all_nodes_config ) -%}