build: bump version 1.8.1-a1 -> 1.8.1-a2

!64 https://github.com/ansible/awx/issues/15161

.cz.yaml

@@ -1,7 +1,8 @@
+---
 commitizen:
-  bump_message: "build(version): bump version $current_version \u2192 $new_version"
-  changelog_incremental: false
   name: cz_conventional_commits
-  tag_format: $major.$minor.$patch$prerelease
-  update_changelog_on_bump: true
-  version: 0.2.0
+  prerelease_offset: 1
+  tag_format: $version
+  update_changelog_on_bump: false
+  version: 1.8.1-a2
+  version_scheme: semver

.gitignore

@@ -0,0 +1,5 @@
+artifacts/
+build/
+test_results/
+test_results.json
+*.tar.gz

.gitlab-ci.yml

@@ -1,24 +1,92 @@
 ---
+
+variables:
+  ANSIBLE_GALAXY_PACKAGE_NAME: kubernetes
+  MY_PROJECT_ID: "51640029"
+  GIT_SYNC_URL: "https://$GITHUB_USERNAME_ROBOT:$GITHUB_TOKEN_ROBOT@github.com/NoFussComputing/ansible_collection_kubernetes.git"
+  PAGES_ENVIRONMENT_PATH: projects/ansible/collection/kubernetes/
+  RELEASE_ADDITIONAL_ACTIONS_BUMP: ./.gitlab/additional_actions_bump.sh
+
+
 include:
+  - local: .gitlab/integration_test.gitlab-ci.yml
   - project: nofusscomputing/projects/gitlab-ci
     ref: development
     file:
       - .gitlab-ci_common.yaml
-      - template/automagic.gitlab-ci.yaml
-
-variables:
-  MY_PROJECT_ID: "51640029"
-  GIT_SYNC_URL: "https://$GITHUB_USERNAME_ROBOT:$GITHUB_TOKEN_ROBOT@github.com/NoFussComputing/ansible_role_nfc_kubernetes.git"
-  PAGES_ENVIRONMENT_PATH: projects/ansible/roles/kubernetes/
+      - conventional_commits/.gitlab-ci.yml
+      - template/ansible-collection.gitlab-ci.yaml
+      - template/mkdocs-documentation.gitlab-ci.yaml
+      # ToDo: update gitlabCI jobs for collections workflow
+      - git_push_mirror/.gitlab-ci.yml
+      - automation/.gitlab-ci-ansible.yaml
 
 
-Ansible-roles.Submodule.Deploy:
-  extends: .submodule_update_trigger
-  variables:
-    SUBMODULE_UPDATE_TRIGGER_PROJECT: nofusscomputing/projects/ansible/ansible-roles
-    GIT_COMMIT_TYPE: feat
-    GIT_COMMIT_TYPE_CATEGORY: $CI_PROJECT_NAME
-    GIT_CONFIG_SUBMODULE_NAME: nfc_kubernetes
+Build Collection:
+  extends: .ansible_collection_build
+  needs:
+    - job: Ansible Lint
+      optional: true
+    - job: Ansible Lint (galaxy.yml)
+      optional: true
+
+  rules:
+
+    - if: $CI_COMMIT_TAG
+      when: always
+
+    # Needs to run, even by bot as the test results need to be available
+    # - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+    #   when: never
+
+    - if: # Occur on merge
+        $CI_COMMIT_BRANCH
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      when: always
+
+    # - if:
+    #     $CI_COMMIT_BRANCH != "development"
+    #       &&
+    #     $CI_COMMIT_BRANCH != "master"
+    #       &&
+    #     $CI_PIPELINE_SOURCE == "push"
+    #   when: always
+
+    - when: never
+
+
+Update Git Submodules:
+  extends: .ansible_playbook_git_submodule
+
+
+Github (Push --mirror):
+  extends:
+    - .git_push_mirror
+  needs: []
+
+
+Gitlab Release:
+  extends: .ansible_collection_release
+  needs:
+    - Stage Collection
+  release:
+    tag_name: $CI_COMMIT_TAG
+    description: ./artifacts/release_notes.md
+    name: $CI_COMMIT_TAG
+    assets:
+      links:
+        - name: 'Ansible Galaxy'
+          url: https://galaxy.ansible.com/ui/repo/published/${ANSIBLE_GALAXY_NAMESPACE}/${ANSIBLE_GALAXY_PACKAGE_NAME}/?version=${CI_COMMIT_TAG}
+
+        - name: ${ANSIBLE_GALAXY_NAMESPACE}-${ANSIBLE_GALAXY_PACKAGE_NAME}-${CI_COMMIT_TAG}.tar.gz
+          url: https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/${ANSIBLE_GALAXY_NAMESPACE}-${ANSIBLE_GALAXY_PACKAGE_NAME}-${CI_COMMIT_TAG}.tar.gz
+          link_type: package
+
+        - name: Documentation
+          url: https://nofusscomputing.com/${PAGES_ENVIRONMENT_PATH}
+    milestones: 
+      - $CI_MERGE_REQUEST_MILESTONE
 
 
 Website.Submodule.Deploy:

.gitlab/additional_actions_bump.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Nothing to do here!!";

.gitlab/integration_test.gitlab-ci.yml

@@ -0,0 +1,217 @@
+
+.integration_test:
+
+  stage: test
+
+  needs:
+    - "Build Collection"
+
+  image: 
+    name: nofusscomputing/docker-buildx-qemu:dev
+    pull_policy: always
+
+  variables:
+    DOCKER_HOST: tcp://docker:2375/
+    DOCKER_DRIVER: overlay2
+    # GIT_STRATEGY: none
+
+  services:
+    - name: docker:23-dind
+      entrypoint: ["env", "-u", "DOCKER_HOST"]
+      command: ["dockerd-entrypoint.sh"]
+  before_script:
+    - | # start test container
+      docker run -d \
+        --privileged \
+        -v ${PWD}:/workdir \
+        -v ${PWD}/artifacts/galaxy:/collection \
+        --workdir /workdir \
+        --rm \
+        --env "ANSIBLE_FORCE_COLOR=true" \
+        --env "CI_COMMIT_SHA=${CI_COMMIT_SHA}" \
+        --env "ANSIBLE_LOG_PATH=/workdir/ansible.log" \
+        --env "PIP_BREAK_SYSTEM_PACKAGES=1" \
+        --name test_image_${CI_JOB_ID} \
+        nofusscomputing/ansible-docker-os:dev-${test_image}
+
+    - | # enter test container
+      docker exec -i test_image_${CI_JOB_ID} ps aux
+    - docker ps
+    - docker exec -i test_image_${CI_JOB_ID} apt update
+    - docker exec -i test_image_${CI_JOB_ID} apt install -y --no-install-recommends python3-pip net-tools dnsutils iptables
+    - |
+      if [ "${test_image}" == 'debian-12' ]; then
+
+        echo "Debian 12":
+
+        docker exec -i test_image_${CI_JOB_ID} pip install ansible-core --break-system-packages;
+
+        docker exec -i test_image_${CI_JOB_ID} mkdir -p /etc/iptables;
+
+        docker exec -i test_image_${CI_JOB_ID} touch /etc/iptables/rules.v6;
+
+        docker exec -i test_image_${CI_JOB_ID} update-alternatives --set iptables /usr/sbin/iptables-legacy;
+
+      else
+
+        echo " Not Debian 12":
+
+        docker exec -i test_image_${CI_JOB_ID} pip install ansible-core;
+
+      fi
+
+    - docker exec -i test_image_${CI_JOB_ID} cat /etc/hosts
+    - docker exec -i test_image_${CI_JOB_ID} cat /etc/resolv.conf
+    - | # check if DNS working
+      docker exec -i test_image_${CI_JOB_ID} nslookup google.com
+  script:
+    - | # inside container?
+      docker exec -i test_image_${CI_JOB_ID} ls -l /collection;
+      docker exec -i test_image_${CI_JOB_ID} echo $PWD;
+
+    - | # Show Network Interfaces
+      docker exec -i test_image_${CI_JOB_ID} ifconfig;
+
+    - | # Install the collection
+      docker exec -i test_image_${CI_JOB_ID} bash -c 'ansible-galaxy collection install $(ls /collection/*.tar.gz)'
+
+    - | # output ansible vars
+      docker exec -i test_image_${CI_JOB_ID} ansible -m setup localhost
+
+    - | # run the collection
+      docker exec -i test_image_${CI_JOB_ID} \
+        ${test_command} \
+        --extra-vars "nfc_role_firewall_policy_input=ACCEPT" \
+        --extra-vars "nfc_role_firewall_policy_forward=ACCEPT" \
+        -vv
+
+    - | # Create test.yaml
+      mkdir -p test_results;
+      cat <<EOF > test_results/${test_image}.json
+      {
+        "$( echo ${test_image}  | sed -e 's/\./_/')": "Pass"
+      }
+
+      EOF
+
+  after_script:
+    - | # Create test.yaml if not exists
+      if [ ! -f test_results/${test_image}.json ]; then
+
+      echo "[TRACE] Test has failed"
+
+      mkdir -p test_results;
+
+      cat <<EOF > test_results/${test_image}.json
+      {
+        "$( echo ${test_image}  | sed -e 's/\./_/')": "Fail"
+      }
+
+      EOF
+
+      fi
+
+    - | # Run trace script for debugging
+      chmod +x ./.gitlab/integration_test_trace.sh;
+
+      ./.gitlab/integration_test_trace.sh;
+
+  artifacts:
+    untracked: false
+    paths:
+      - ansible.log
+      - test_results/*
+    when: always
+
+  rules:
+
+    - if: $CI_COMMIT_TAG
+      allow_failure: true
+      when: on_success
+
+    # Needs to run, even by bot as the test results need to be available
+    # - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+    #   when: never
+
+    - if: # Occur on merge
+        $CI_COMMIT_BRANCH
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      allow_failure: true
+      when: on_success
+
+    # - if:
+    #     $CI_COMMIT_BRANCH != "development"
+    #       &&
+    #     $CI_COMMIT_BRANCH != "master"
+    #       &&
+    #     $CI_PIPELINE_SOURCE == "push"
+    #   allow_failure: true
+    #   when: always
+
+    - when: never
+
+
+
+Playbook - Install:
+  extends: .integration_test
+  parallel:
+    matrix:
+      - test_image: debian-11
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+      - test_image: debian-12
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+      - test_image: ubuntu-20.04
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+      - test_image: ubuntu-22.04
+        test_command: ansible-playbook nofusscomputing.kubernetes.install
+
+
+
+test_results:
+  stage: test
+
+  extends: .ansible_playbook
+
+  variables:
+    ansible_playbook: .gitlab/test_results.yaml
+    ANSIBLE_PLAYBOOK_DIR: $CI_PROJECT_DIR
+
+  needs:
+    - Playbook - Install
+
+  artifacts:
+    untracked: false
+    when: always
+    access: all
+    expire_in: "3 days"
+    paths:
+      - test_results.json
+
+  rules:
+
+    - if: $CI_COMMIT_TAG
+      allow_failure: true
+      when: on_success
+
+    # Needs to run, even by bot as the test results need to be available
+    # - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+    #   when: never
+
+    - if: # Occur on merge
+        $CI_COMMIT_BRANCH
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      allow_failure: true
+      when: on_success
+
+    # - if:
+    #     $CI_COMMIT_BRANCH != "development"
+    #       &&
+    #     $CI_COMMIT_BRANCH != "master"
+    #       &&
+    #     $CI_PIPELINE_SOURCE == "push"
+    #   allow_failure: true
+    #   when: always
+
+    - when: never

.gitlab/integration_test_trace.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# colour ref: https://stackoverflow.com/a/28938235
+
+NC='\033[0m'       # Text Reset
+
+# Regular Colors
+Black='\033[0;30m'        # Black
+Red='\033[0;31m'          # Red
+Green='\033[0;32m'        # Green
+Yellow='\033[0;33m'       # Yellow
+Blue='\033[0;34m'         # Blue
+Purple='\033[0;35m'       # Purple
+Cyan='\033[0;36m'         # Cyan
+
+
+cmd() {
+
+  echo -e "${Yellow}[TRACE] ${Green}executing ${Cyan}'$1'${NC}"
+
+  docker exec -i test_image_${CI_JOB_ID} $1 || true
+
+}
+
+
+cmd "journalctl -xeu netfilter-persistent.service";
+
+cmd "journalctl -xeu iptables.service"
+
+cmd "journalctl -xeu k3s.service"
+
+cmd "systemctl status netfilter-persistent.service"
+
+cmd "systemctl status iptables.service"
+
+cmd "systemctl status k3s.service"
+
+cmd "kubectl get po -A -o wide"
+
+cmd "kubectl get no -o wide"
+
+cmd "iptables -nvL --line-numbers"

.gitlab/merge_request_templates/default.md

@@ -0,0 +1,22 @@
+### :books: Summary
+<!-- your summary here emojis ref: https://github.com/yodamad/gitlab-emoji -->
+
+
+
+### :link: Links / References
+<!-- using a list as any links to other references or links as required. if relevent, describe the link/reference -->
+
+
+### :construction_worker: Tasks
+
+ - [ ] Add your tasks here if required (delete)
+
+<!-- dont remove tasks below strike through including the checkbox by enclosing in double tidle '~~' -->
+
+- [ ] Playbook Update
+
+    This collection has a [corresponding playbook](https://gitlab.com/nofusscomputing/projects/ansible/ansible_playbooks/-/blob/development/role.yaml) that may need to be updated (Ansible Role), specifically [Role Validation](https://gitlab.com/nofusscomputing/projects/ansible/ansible_playbooks/-/blob/development/tasks/role/validation/nfc_kubernetes.yaml).
+
+- [ ] NetBox Rendered Config Update
+
+    This Collection has a [NetBox Rendered Config template](https://gitlab.com/nofusscomputing/infrastructure/configuration-management/netbox/-/blob/development/templates/cluster.json.j2) that may need to be updated. Specifically Section `cluster.type == 'kubernetes'`

.gitlab/test_results.yaml

@@ -0,0 +1,19 @@
+---
+
+- name: Create Test Results File
+  hosts: localhost
+  gather_facts: false
+
+
+  tasks:
+    
+
+    - name: Load Test Results
+      ansible.builtin.include_vars:
+        dir: ../test_results
+        name: test_results
+
+    - name: Create Results file
+      ansible.builtin.copy:
+        content: "{{ (test_results) | to_nice_json }}"
+        dest: ../test_results.json

.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+    "yaml.schemas": {
+        "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/tasks": [
+            "roles/nfc_kubernetes/tasks/*.yaml",
+            "roles/nfc_kubernetes/tasks/*/*.yaml",
+            "roles/nfc_kubernetes/tasks/*/*/*.yaml"
+        ],
+        "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/vars.json": [
+            "roles/nfc_kubernetes/variables/**.yaml"
+        ],
+        "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible.json#/$defs/playbook": ".gitlab/test_results.yaml"
+    },
+    "gitlab.aiAssistedCodeSuggestions.enabled": false,
+    "gitlab.duoChat.enabled": false,
+}

CHANGELOG.md

@@ -1,162 +1,91 @@
-## 0.2.0 (2024-02-03)
+## 1.8.1-a2 (2024-05-02)
 
-### Bug Fixes
+### Fix
 
-- **config**: [623d1781](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/623d1781960afa81ac62dc4479f52297eb75b721) - set external ip if set or node ip if not set [ [!23](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/23) ]
-- **install**: [506385f3](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/506385f3d84c9b419f9ae3ad6ec5e3424c922a3f) - don't attempt to reinstall the cluster if already installed [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **prime_install**: [ccf5c03a](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/ccf5c03a4c55d626ed3bf009d9111aac823ef51c) - requires cluster init for prime install [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **install**: [bed1bf30](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/bed1bf30957f050b2748c2282b123400533c29ee) - only configure network manager if installed [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- [9cdc89b1](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/9cdc89b1ec3961f93d0c656f0b6ef2beae0336a7) - always install and configure regardless of tags [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **common**: [5edfdf4f](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/5edfdf4faf71ebccd0d5890e70ef0d9a566ae6f6) - don't run soe task on software install [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **new_cluster**: [5ffbd78e](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/5ffbd78e2b34288d8d93efda1e30ad662653fbfd) - correct config for new cluster install [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **node_restart**: [c6ff60bb](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/c6ff60bb14f8a0820b97d8e5e988fe8da65e7f35) - use correct variables [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **restart_k3s**: [aa2d858e](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/aa2d858ede6a426ce0e4885260275b5be1207abe) - use correct group var [ [!16](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/16) ]
-- **token_fetch**: [88d57588](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/88d57588fcecbd10d1e058a26711d32e0371b38c) - only fetch token after prime installed [ [!16](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/16) ]
-- **handler**: [26120c3e](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/26120c3e9846c11c45f5c8d2257cf29e63be86a6) - kubernetes restart when clause corrected [ [!15](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/15) [#16](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/16) [#21](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/21) [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **audit_log**: [ea38ddf2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/ea38ddf22b6e2c6ceda4c1cccb31f3fad126b058) - max age not backup [ [!12](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/12) ]
-- **config**: [56bb4557](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/56bb4557b523b24c27ce9c8f2e998f06f768e2c2) - ensure server var is list not csv string [ [!11](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/11) [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
-- **handler**: [0fb5e276](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/0fb5e27612cc18dcf8027c8e65cac11a9ca607bc) - restart kubernetes implementation was flawed [ [!11](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/11) ]
-- **config**: [301ed9ad](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/301ed9ad3fc269f5b0fd625c30da53e714a70e80) - ensure join token is included in config [ [!11](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/11) [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
-- **multi-master**: [4ce5f372](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/4ce5f3722365d9584ac7da29a9903d7d8d71f414) - ensure that all master nodes are added during config [ [!11](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/11) [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
-- **play**: [86af4606](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/86af4606d76fc007dfdfaa9d84037d8b225ab35d) - k3s download task doen't require content returned [ [!11](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/11) ]
-- **k3s_multi_master**: [c41e1254](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/c41e12544b4eff2ce9e62e2209c5b402f7b5a71f) - adjusted config so multi-master install works [ [!9](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/9) [!4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/4) ]
-- **olm**: [915cdf5e](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/915cdf5e1e279a357f4664cc1d0c6dd5d1374c71) - dont fail if already installed [ [!9](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/9) [!4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/4) ]
-- **cluster_encryption**: [f0cf4cd0](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/f0cf4cd00ce5184ab5c242059ccd5c8a072c50a5) - use correct variable path to check if required [ [!9](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/9) [!4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/4) ]
-- **firewall**: [ed1a1acf](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/ed1a1acf7ece02c2adb2d226f249e84ee0c2016e) - if host value enpty, don't create rule [ [!9](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/9) [!4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/4) ]
-- **kubelet_args**: [59a5e0aa](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/59a5e0aacf8dd1c61ae4d4e6b8aa4a5b3cde3e6a) - ensure correct indentation for kubelet args [ [!9](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/9) [!4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/4) ]
-- **node_labels**: [20dae6ba](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/20dae6ba4d569eb55479dbe291fb578f48af4029) - ensure correct indentation for label list [ [!9](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/9) [!4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/4) ]
-- **config**: [fac3ace5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/fac3ace5f5907946b5f9b82ad94b07e97fece7c9) - ensure config option servicelb-namespace only deployed to prime node [ [!8](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/8) [#12](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/12) ]
+- **nfc_kubernetes**: cast url var as list
 
-### Code Refactor
+## 1.8.1-a1 (2024-05-02)
 
-- **config**: [b350b2e1](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/b350b2e188cc9b3274d7b04cd3f2dcce8e4cf505) - use jinja to construct data then pretty print it [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- [478e4ccf](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/478e4ccfa51746e9e34ed9bdc30b8ab2142624e1) - dont show changes for prep work [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **tasks**: [c04b12a7](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/c04b12a714a998dbe50256fb4f7f0c7f54f56536) - ensure module FQCN is used [ [!15](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/15) [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **node_labels**: [58a95e67](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/58a95e6781c5f16e0453409c9e58c55d6aa41312) - removed from config.yaml and set to be a manifest on prime node [ [!10](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/10) [!4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/4) ]
-- **config**: [1a0407a9](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/1a0407a901408caf35d254a6de4a90dadf63769a) - only populate cluster-domain if defined [ [!7](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/7) [#12](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/12) ]
+### Fix
 
-### Continious Integration
+- **nfc_kubernetes**: correct url build to loop through all cpu arch
 
-- [7a017c4e](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/7a017c4e2916034dceda51e07cf3f09c7bbf1a2f) - add documentation deployment [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
+## 1.8.0 (2024-05-02)
 
-### Documentaton / Guides
+### Feat
 
-- [3004f998](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/3004f998bf95345af90569d5e6741a52f102bf2a) - add release notes [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- [6d974083](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/6d974083cf923df6f06fa2a26983b129deac45e0) - fix github badges links [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- [b2f9e5d3](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/b2f9e5d3caf99bf32ca040fd6f0d902309946beb) - update [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- [3198b5d2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/3198b5d2f9c06bb119d8549eca16bf1c9cd29607) - added recently configurable items [ [!7](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/7) ]
+- **nfc_kubernetes**: build url and on use cast as string
 
-### Features
+## 1.7.2 (2024-04-25)
 
-- **calico**: [abb7042c](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/abb7042cbd415ee9692911a5324355a424c70239) - turn bpf off [ [!23](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/23) ]
-- **calico**: [a45fe0c9](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/a45fe0c9f96f8c4b0571829c3a39edb189d12a5e) - set tolerations for typha "CriticalAddonsOnly" [ [!23](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/23) ]
-- **config**: [45863ecf](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/45863ecff37480d9c45a12efa714c3856eb59f21) - for server self. use internal ip to connect instead of external [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **config**: [b43e1dbb](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/b43e1dbb80c5edcd1fc18f9674f1fcd0ee4ec00d) - dont set external-ip if it matches node-ip [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **config**: [efba1ff6](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/efba1ff6c74a5e57f98f2a9d78a765212f6795df) - set value `node-ip` [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **calico**: [2e136ee0](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/2e136ee08838d16e7695da469bb55e796c7089b1) - use vxlan instead of ipip [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **olm**: [384ef924](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/384ef924ca7a3de51db16b84790e2b3842a7c83c) - uninstall olm if tag `olm_uninstall` specified [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **calico**: [54f9ec0c](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/54f9ec0c950825b0a81fe4b19efe5e8ddc41f88c) - add job tag calico_manifest to enable rollback [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **cni**: [26c0ab12](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/26c0ab123628898096cf7767298dd16684db173f) - enable eBPF [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **metallb**: [be1ddecc](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/be1ddecc33925f39c860e9d0177eb3bc79941ba6) - MetalLB Installable instead of ServiceLB [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **calico_operator**: [79d89b3b](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/79d89b3b3a094bbea10448c23a979807be7644fb) - once installed or new cluster always run calico operator deploy [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **calico_operator**: [8d8ba095](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/8d8ba0951e173536ae4b3cc708f56c507a349942) - install before prime check as networking is required [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **firewall**: [8919486b](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/8919486b6b83b3429b30936318e7be8bbff158d3) - add required rules for calico operator [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **calico_operator**: [5925a26c](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/5925a26c608d497dc09b4e9cecb1a13b5be5f849) - store ansible manifests on node [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **calico_operator**: [f48f6454](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/f48f645468f82c2c7c9aa62c86e66d52fee9002d) - deploy the operator to configure calico cni [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) [#3](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/3) ]
-- **networ_manager**: [7049c57b](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/7049c57bd062a277236fb93c8e3eff4b34e271ac) - setup calico interfaces as unmanaged [ [!17](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/17) ]
-- **install**: [e1220b0d](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/e1220b0dac4129169445c30f1e9867cbe5ceff5d) - enable k3s module metrics-server [ [!16](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/16) ]
-- **olm**: [681b52b3](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/681b52b31a654040cd5e2c23a52f2b8631b3085b) - dont install by default [ [!16](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/16) ]
-- **calico**: [60f7c2d6](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/60f7c2d6b6de0cbda1b43f08060f87fe2e1dc6ed) - disable vxlan [ [!16](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/16) ]
-- **calico**: [a54fbe26](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/a54fbe26f372e2c5cccfc193b71f007ae2e441c9) - use vxlan overlay [ [!16](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/16) ]
-- **calico**: [0bdd5c66](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/0bdd5c66c262d9afc0226a8e201170b0b9188ea3) - IP AUTO-detection set to kubernetes-internal-ip [ [!14](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/14) ]
-- [74cc2079](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/74cc20794764bb2e1b042ceb641b0e515703884a) - feature gate added to prevent restart of kubernetes service [ [!14](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/14) [!13](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/13) ]
-- **node**: [c28f0b8e](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/c28f0b8ee301e1c76553476a0808b45efa417bd9) - ability to configure node taints [ [!13](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/13) [!9](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/9) [!7](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/7) ]
-- **config**: [4a41f7e3](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/4a41f7e348d00b91a39160aaa9d8c1cd8e338cb4) - set node name to inventory_hostname [ [!12](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/12) [!7](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/7) ]
-- **firewall**: [a31837c8](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/a31837c803902e6bcfd80704286c62ec9536e422) - add vxlan rules [ [!12](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/12) ]
-- **audit_logs**: [73691631](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/7369163195a06dac4f07a01bd84db64867480561) - keep two days by default [ [!12](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/12) ]
-- **firewall**: [59699afb](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/59699afb4402b518c9bd2103384affd382761ea2) - allow hosts external IP [ [!12](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/12) ]
-- **olm**: [021e54f3](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/021e54f32864bfb2dff4933d3053aa50a6f20000) - install olm operator [ [!9](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/9) [!4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/4) ]
-- **config**: [c7cd1da4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/c7cd1da4312092c5afe7b093d39471488100ab1c) - support setting node labels [ [!7](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/7) [#12](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/12) ]
-- **config**: [cdc06363](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/cdc06363aad63a72cb7287596f4f340b25226f54) - support setting the service [ [!7](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/7) [#12](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/12) ]
-- **config**: [34432433](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/34432433f3127763d1def38d479b869e1a84d6c8) - support setting the cluster domain [ [!7](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/7) [#12](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/12) ]
-- [21cef1f4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/21cef1f4c32777245b56edb232cc0528dacb94db) - enable installation for arm64 architectures [ [!7](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/7) [#11](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/11) [!29](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/29) [!1](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/1) ]
+### Fix
 
-## 0.2.0rc1 (2023-11-06)
+- **nfc_kubernetes**: adjust some tasks to run during checkmode
 
-### Bug Fixes
+## 1.7.1 (2024-04-24)
 
-- **config**: [f09737b2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/f09737b21f1bac4d505207136e9505c83140b4cf) - typo in config template [ [!6](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/6) ]
-- **rbac**: [6ab17bdc](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/6ab17bdc3c660e704ce7319a21a517f38907a541) - authorization:namespace:owner ns owner not to remove cluster resources [ [!6](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/6) ]
-- **container_registries**: [9936cd44](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/9936cd449911a0e9612309690fbca82e8565c2b3) - these are optional, dont attempt to configure if non specified [ [!6](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/6) ]
-- **config**: [5278a499](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/5278a4996e0370bbe14ba08189879c19f6b1f983) - server attribute not required for prime master node [ [!6](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/6) ]
+### Fix
 
-### Documentaton / Guides
+- add role readme
 
-- [0acc7a3c](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/0acc7a3cc2b93f94fc92839e34dec02cfe4d5ec2) - added note on config for system resources [ [!6](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/6) [#7](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/7) ]
+## 1.7.0 (2024-04-24)
 
-### Features
+### Feat
 
-- **rbac**: [59f50d53](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/59f50d53df4e28ef114d734f652cb25d396a381d) - new cluster role and binding ingress-admin [ [!6](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/6) ]
-- **config**: [d2081284](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/d2081284d1d526a57a34082f5ad944dc12671750) - configurabl system reserved cpu,memory,storage [ [!6](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/6) ]
-- **install**: [42ac18e0](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/42ac18e05768e61f0d960fb3afe2482431bd5fce) - disable swap [ [!6](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/6) ]
+- **kubernetes_netbox**: custom field bug work around
+- **services**: add netbox service fields
+- **role**: New role kubernetes_netbox
 
-## 0.2.0rc0 (2023-11-02)
+### Fix
 
-### Bug Fixes
+- **nfc_kubernetes**: ensure install tasks run when job_tags specified
+- **facts**: gather required facts if not already available
+- **install**: correct template installed var
+- **install**: as part of install check, confirm service
 
-- **firewall**: [9c420475](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/9c4204751e00c6d8a36c0fcb1ff66f0a87ba574e) - use correct chain name [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) [#2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/2) ]
-- **firewall**: [ba59dd30](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/ba59dd305728b15801aca3aef81ea8aa35e5d402) - correct rules so that kubernetes hosts are added to firewall allowed hosts [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) [#2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/2) ]
-- **wireguard**: [4a9d9839](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/4a9d98394e3fd327a0f681e42b5415e25520cff4) - install before k3s [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
-- **templates**: [89b65732](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/89b65732478883a5ccc29c9100c30840bf08d55e) - add clause to filter for prime master only [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
-- **install**: [b77cc6a8](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/b77cc6a8e902739f43519d4e9a0c0dcfca0cc30f) - install prime then wait till ready before install other nodes [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
-- **install**: [26f1f2ef](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/26f1f2efe63b47f7ab569f99aa0c1b5f28ff461b) - server install to use version variables [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
-- **configure**: [db515d2c](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/db515d2c1d921950c034911fad6622f969232895) - move required config files to install portion [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
-- **config**: [1b62a666](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/1b62a6663285b6c62323ca387ba176cc9d7eff74) - cluster-init only required for master nodes [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
+## 1.6.0 (2024-03-29)
 
-### Code Refactor
+### Feat
 
-- [4d9f9dcd](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/4d9f9dcdff5d07bb9f0722ddc22ff061ffa24f20) - use kubernetes_config for cluster config [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
-- **template**: [50f48ab5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/50f48ab5a1a5983ed7465fefda7119df5d4fff96) - config.yaml rearranged orderd, and updated for kube role [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
-- **install**: [4465bcd2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/4465bcd2c4f4a94cacd0f35025d63f4f858ee58d) - clean wait for k3s to be ready [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) ]
+- **test**: add integration test. playbook install
+- add retry=3 delay=10 secs to all ansible url modules
+- **upgrade**: If upgrade occurs, dont run remaining tasks
+- support upgrading cluster
 
-### Features
+### Fix
 
-- **install**: [c7907bf5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/c7907bf585d182b5e68d241428dd69d7c1af1be7) - install additional master nodes [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) [#2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/2) ]
-- **install**: [fd547a4c](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/fd547a4c0fbfa34fca6ec6860d294c1713594e41) - install worker nodes [ [!5](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/5) [#2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/2) ]
-- **install**: [76e48fd9](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/76e48fd965b9113d7aa69679524bc119f45db6ac) - wait for cluster to be ready before continuing [ [!4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/4) ]
-- **config**: [abc01ce4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/abc01ce48cbd586dfec067b3fcdbca0d204843d4) - enable configering etc cron and retention [ [!4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/4) ]
+- **docs**: use correct badge query url
 
-## 0.1.0 (2023-10-29)
+### Refactor
 
-### Bug Fixes
+- **galaxy**: for dependent collections prefix with `>=` so as to not cause version lock
 
-- **k3s**: [9ac52ee1](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/9ac52ee165fd364c7091ab3f1e14df365270f532) - use correct variables in conditional clauses [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) ]
-- [8272b250](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/8272b2507b298ccec05e6dbaa2a526b5136b8d2d) - uncommented hash tasks as they are required [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) ]
-- **install**: [57d268ec](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/57d268ec3cd990ea21979cbafe7421a0af04ea91) - config files only required for prime master [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) ]
-- **install**: [0f4a02ca](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/0f4a02cadd24dc1890e57bba5266f17dd44e9766) - restructure and uncommented install steps [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) ]
-- [49087753](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/4908775367a657867878111ad7e8a75e5203e492) - dont flush handlers [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) ]
+## 1.5.0 (2024-03-21)
 
-### Code Refactor
+### Feat
 
-- [93897ea7](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/93897ea7d5d8e11725aa1c285fac64388215d00b) - moved config file deploy to be part of install tasks [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) ]
+- **collection**: nofusscomputing.firewall update 1.0.1 -> 1.1.0
 
-### Continious Integration
+## 1.4.0 (2024-03-20)
 
-- [55d5c5d6](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/55d5c5d6943a0794bd73f8701667e85dd653c5ea) - add initial jobs [ [!1](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/1) ]
+### Feat
 
-### Documentaton / Guides
+- **install**: "ansible_check_mode=true" no hostname check
 
-- [779be020](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/779be0200e71956a3125332d57ac6e0dc7a4914a) - add to feature list openid [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) ]
-- [c3843dde](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/c3843ddef0a6d4f885a989675b79ac5861e21138) - role workflow [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) [#4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/4) ]
-- [b69d5b8a](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/b69d5b8a358e6b024b0afda819af0082c0b87a48) - feature list [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) [#4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/4) ]
-- [60392a56](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/60392a565c53010faca6c6eda15d2c386133a8f7) - restructure for seperate role index and ansible setup [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) [#4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/4) ]
-- [bbfbbedd](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/bbfbbedd11ea5b1fde199899b70cd87119e3a989) - initial docs for rbac [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) [#4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/4) ]
-- [3e785d7d](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/3e785d7db158e41744ad19c4fcab1c11aa23823f) - added other projects to blurb [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) [#4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/4) ]
-- [7abfb703](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/7abfb70320419ab1e98666a16453bb1b0a48426e) - Ansible setup [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) [#4](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/4) ]
-- [b588b038](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/b588b0383d3e353a8e487d06b787aed2e28de2d8) - added docs layout [ [!1](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/1) ]
+## 1.3.0 (2024-03-18)
 
-### Features
+### Fix
 
-- **networking**: [60fd25df](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/commit/60fd25df8ec897e74c164d9cc0e49ed07d002d0e) - install and configure wireguard encryption [ [!2](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/2) [#3](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues/3) ]
+- **handler**: add missing 'reboot_host' handler
+- **firewall**: ensure slave nodes can access ALL masters API point
+- **firewall**: dont add rules for disabled features
 
-## 0.0.1 (2023-10-28)
+## 1.2.0 (2024-03-16)
+
+### Feat
+
+- **firewall**: use collection nofusscomputing.firewall to configure kubernetes firewall
+
+### Fix
+
+- **config**: use correct var name when setting node name

CONTRIBUTING.md

@@ -0,0 +1,24 @@
+# Contribution Guide
+
+
+
+## Updating components with a remote source
+
+Some components within this role are sourced from a remote source. To update them to the latest release use the following commands.
+
+> Ensure that before committing the update remote files to the repository, that no features have been removed that were added.
+
+
+### Kubevirt
+
+``` bash
+
+export KUBEVIRT_RELEASE='<kubevirt release i.e. v1.2.0>'
+
+# From within roles/nfc_kubernetes/templates directory
+wget https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_RELEASE}/kubevirt-operator.yaml -O kubevirt-operator.yaml.j2
+
+# From within the roles/nfc_kubernetes/templates directory
+wget https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_RELEASE}/kubevirt-cr.yaml -O kubevirt-cr.yaml.j2
+
+```

README-orig.md

@@ -1,39 +0,0 @@
-# Kubernetes Ansible Playbook
-
-
-## Additional changes
-
-- `SystemdCgroup = false` -> `SystemdCgroup = true` [See this comment](https://github.com/kubernetes/kubernetes/issues/110177#issuecomment-1161647736)
-
-## Tags
-
-This role has been setup to take advantage of Ansible tags. The use of these tags enables finer control over what tasks are run. By design, when you set a task, only what is required for the tag is run.
-
-available tags are as follows
-
-- `containerregistry` apply container/docker registry settings
-- `firewall` apply firewall settings (firewall name/type independent)
-- `install` Run every task within the role. this is the same as omitting `--tags`
-- `iptables` apply iptables settings
-- `manifest` Apply/remove kubernetes manifests
-- `namespace` Apply/remove kubernetes namespaces
-- `nodelabels` Apply/remove kubernetes node labels
-- `taints` Apply/remove kubernetes taints
-
-!!! tip
-    if you intend on running the `install` tag, you can omit the `--tags` flag from the ansible tag all togther
-
-!!! alert
-    the first time this playbook is run if cli switch `--extra-vars "init=true"` is used with `init` either a bool true/false, the manifests will not be applied. this is to enable the kubernetes to be fully setup prior to applying manifests that may prevent successful completion of the play.
-
-## command Cheatsheet
-
-- `crictl --runtime-endpoint unix:///run/containerd/containerd.sock images` list all container images on the host
-
-## Links / References
-
-- ContainerD Configuration
-
-  - [Registry Configuration](https://github.com/containerd/containerd/blob/7cd72cce99c8d3b938c1b763c2744a0b699028ab/docs/cri/config.md#registry-configuration)
-
-  - [Configuring ContainerD registries](https://github.com/containerd/containerd/blob/7cd72cce99c8d3b938c1b763c2744a0b699028ab/docs/hosts.md#cri)

README.md

@@ -1,34 +1,44 @@
-<div align="center" width="100%">
+<span style="text-align: center;">
 
-
-# No Fuss Computing - Ansible Role: nfc_kubernetes
+# No Fuss Computing - Ansible Collection Kubernetes
 
 <br>
 
 ![Project Status - Active](https://img.shields.io/badge/Project%20Status-Active-green?logo=gitlab&style=plastic) 
 
+
+[![Downloads](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fkubernetes%2F&query=%24.download_count&style=plastic&logo=ansible&logoColor=white&label=Galaxy%20Downloads&labelColor=black&color=cyan)](https://galaxy.ansible.com/ui/repo/published/nofusscomputing/kubernetes/)
+
+
+----
+
 <br>
 
-![Gitlab forks count](https://img.shields.io/badge/dynamic/json?label=Forks&query=%24.forks_count&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2F&color=ff782e&logo=gitlab&style=plastic) ![Gitlab stars](https://img.shields.io/badge/dynamic/json?label=Stars&query=%24.star_count&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2F&color=ff782e&logo=gitlab&style=plastic) [![Open Issues](https://img.shields.io/badge/dynamic/json?color=ff782e&logo=gitlab&style=plastic&label=Open%20Issues&query=%24.statistics.counts.opened&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fissues_statistics)](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues)
+![Gitlab forks count](https://img.shields.io/badge/dynamic/json?label=Forks&query=%24.forks_count&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2F&color=ff782e&logo=gitlab&style=plastic) ![Gitlab stars](https://img.shields.io/badge/dynamic/json?label=Stars&query=%24.star_count&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2F&color=ff782e&logo=gitlab&style=plastic) [![Open Issues](https://img.shields.io/badge/dynamic/json?color=ff782e&logo=gitlab&style=plastic&label=Open%20Issues&query=%24.statistics.counts.opened&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fissues_statistics)](https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes/-/issues)
 
 
 
-![GitHub forks](https://img.shields.io/github/forks/NofussComputing/ansible_role_nfc_kubernetes?logo=github&style=plastic&color=000000&labell=Forks) ![GitHub stars](https://img.shields.io/github/stars/NofussComputing/ansible_role_nfc_kubernetes?color=000000&logo=github&style=plastic) ![Github Watchers](https://img.shields.io/github/watchers/NofussComputing/ansible_role_nfc_kubernetes?color=000000&label=Watchers&logo=github&style=plastic)
+![GitHub forks](https://img.shields.io/github/forks/NofussComputing/ansible_collection_kubernetes?logo=github&style=plastic&color=000000&labell=Forks) ![GitHub stars](https://img.shields.io/github/stars/NofussComputing/ansible_collection_kubernetes?color=000000&logo=github&style=plastic) ![Github Watchers](https://img.shields.io/github/watchers/NofussComputing/ansible_collection_kubernetes?color=000000&label=Watchers&logo=github&style=plastic)
 <br>
 
-This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes) and has a read-only copy hosted on [Github](https://github.com/NofussComputing/ansible_role_nfc_kubernetes).
+This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes) and has a read-only copy hosted on [Github](https://github.com/NofussComputing/ansible_collection_kubernetes).
 
 ----
 
 **Stable Branch**
 
-![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/kubernetes%2F-%2Fraw%2Fmaster%2F.cz.yaml) 
+![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fmaster%2F.cz.yaml) 
+
+![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814)
+
 
 ----
 
 **Development Branch** 
 
-![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/kubernetes%2F-%2Fraw%2Fdevelopment%2F.cz.yaml)
+![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes%2F-%2Fraw%2Fdevelopment%2F.cz.yaml)
+
+![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fdevelopment%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fdevelopment%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fdevelopment%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fdevelopment%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814)
 
 ----
 <br>
@@ -37,14 +47,14 @@ This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/a
 
 links:
 
-- [Issues](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/issues)
+- [Issues](https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes/-/issues)
 
-- [Merge Requests (Pull Requests)](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests)
+- [Merge Requests (Pull Requests)](https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes/-/merge_requests)
 
 
 
 ## Contributing
-All contributions for this project must conducted from [Gitlab](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes).
+All contributions for this project must conducted from [Gitlab](https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes).
 
 For further details on contributing please refer to the [contribution guide](CONTRIBUTING.md).
 

docs/projects/ansible/collection/firewall/index.md

@@ -0,0 +1 @@
+linked to

docs/projects/ansible/collection/kubernetes/index.md

@@ -0,0 +1,83 @@
+---
+title: Kubernetes
+description: No Fuss Computings Ansible Collection Kubernetes
+date: 2024-03-13
+template: project.html
+about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
+---
+
+<span style="text-align: center;">
+
+![Project Status - Active](https://img.shields.io/badge/Project%20Status-Active-green?logo=gitlab&style=plastic)
+
+
+![Gitlab build status - stable](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Dmaster&logo=gitlab&style=plastic) ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic)
+
+![Debian 11](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-11'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2011&color=a80030) ![Debian 12](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'debian-12'%5D&style=plastic&logo=debian&logoColor=a80030&label=Debian%2012&color=a80030) ![Ubuntu 20.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-20_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2020&color=dd4814) ![Ubuntu 22.04](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F51640029%2Fjobs%2Fartifacts%2Fmaster%2Fraw%2Ftest_results.json%3Fjob%3Dtest_results&query=%24%5B'ubuntu-22_04'%5D&style=plastic&logo=ubuntu&logoColor=dd4814&label=Ubuntu%2022&color=dd4814)
+
+
+[![Downloads](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fkubernetes%2F&query=%24.download_count&style=plastic&logo=ansible&logoColor=white&label=Galaxy%20Downloads&labelColor=black&color=cyan)](https://galaxy.ansible.com/ui/repo/published/nofusscomputing/kubernetes/)
+
+
+</span>
+
+This Ansible Collection is for installing a K3s Kubernetes cluster, both single and multi-node cluster deployments are supported. In addition to installing and configuring the firewall for the node. for further information on the firewall config please see the [firewall docs](../firewall/index.md)
+
+
+## Installation
+
+To install this collection use `ansible-galaxy collection install nofusscomputing.kubernetes`
+
+
+## Features
+
+- Install k3s cluster. Both Single and multi-node clusters
+
+- Configure the cluster
+
+- Upgrade a cluster
+
+For a more detailed list of featured checkout the roles [documentation](roles/nfc_kubernetes/index.md).
+
+
+## Using this collection
+
+This collection has been designed to be a complete and self-contained management tool for a K3s kubernetes cluster.
+
+## Cluster Installation
+
+By default the install playbook will install to localhost.
+
+``` bash
+
+ansible-playbook nofusscomputing.kubernetes.install
+
+```
+
+!!! danger
+    By default when the install task is run, The firewall is also configured. The default sets the `FORWARD` and `INPUT` tables to have a policy of `DROP`. Failing to add any required additional rules before installing/configuring kubernetes will cause you to not have remote access to the machine. 
+    
+    You are encouraged to run `ansible-playbook nofusscomputing.firewall.install` with your rules configured within your inventory first. see the [firewall docs](../firewall/index.md) for more information.
+
+The install playbook has a dynamic `hosts` key. This has been done to specifically support running the playbook from AWX and being able to populate the field from the survey feature. Order of precedence for the host variable is as follows:
+
+- `nfc_pb_host` set to any valid value that a playbook `hosts` key can accept
+
+- `nfc_pb_kubernetes_cluster_name` with the name of the cluster. This variable is appended to string `kubernetes_cluster_` to serve as a group name for the cluster to be installed. i.e. for a cluster called `prime`, the group name would be set to `kubernetes_cluster_prime`
+
+- `--limit` specified at runtime
+
+- `localhost`
+
+For the available variables please view the [nfc_kubernetes role docs](roles/nfc_kubernetes/index.md#default-variables)
+
+
+## Cluster Upgrade
+
+[In place cluster upgrades](https://docs.k3s.io/upgrades/manual#upgrade-k3s-using-the-binary) is the method used to conduct the cluster upgrades. The logic for the upgrades first confirms that K3s is installed and that the local binary and running k3s version are the desired versions. If they are not, they will be updated to the desired version. On completion of this the node has its `k3s` service restarted which completes the upgrade process.
+
+!!! info
+    If an upgrade occurs, no other task within the play will run. This is by design. if you have further tasks to be run in addition to the upgrade, run the play again.
+
+!!! danger
+    not following the [Kubernetes version skew policy](https://kubernetes.io/releases/version-skew-policy/) when upgrading your cluster may break your cluster.

docs/projects/ansible/collection/kubernetes/roles/kubernetes_netbox/index.md

@@ -0,0 +1,46 @@
+---
+title: NetBox Kubernetes
+description: No Fuss Computings Ansible role kubernetes_netbox
+date: 2023-10-24
+template: project.html
+about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
+---
+
+This Ansible role as part of our collection `nofusscomputing.kubernetes` is intended to be used to setup NetBox so that the settings for deploying a kubernetes cluster can be stored within NetBox.
+
+
+## Role Details
+
+| Item| Value | Description |
+|:---|:---:|:---|
+| Dependent Roles | _None_ | |
+| Optional Roles | _None_ | |
+| Idempotent | _Yes_ |  |
+| Stats Available | _Not Yet_ |  |
+| Tags | _Nil_ |  |
+| Requirements | _None_  |  |
+
+
+## Features
+
+- Adds custom fields to `cluster` object within NetBox that this collection can use to deploy a kubernetes cluster.
+
+!!! info
+    Due to a bug in ansible module `netbox.netbox.netbox_custom_field` The fields are not created as they should be. For example, the fields are supposed to be set to only display when not empty. for more information see [Github #1210](https://github.com/netbox-community/ansible_modules/issues/1210). We have [added a workaround](https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes/-/merge_requests/56#note_1876912267) so the fields are created.
+    
+    Other than that, the fields are created as they should.
+
+
+## Usage
+
+To configure NetBox, ensure that the NetBox Access variables are set and run playbook `nofusscomputing.netbox.kubernetes_netbox`. This will setup NetBox with the required fields that role [nfc_kubernetes](../nfc_kubernetes/index.md) uses.
+
+
+## Default Variables
+
+
+``` yaml title="defaults/main.yaml" linenums="1"
+
+--8<-- "roles/kubernetes_netbox/defaults/main.yaml"
+
+```

docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/ansible.md

@@ -3,7 +3,7 @@ title: Ansible
 description: No Fuss Computings Ansible role nfc_kubernetes Ansible docs
 date: 2023-10-24
 template: project.html
-about: https://gitlab.com/nofusscomputing/projects/ansible/roles/kubernetes
+about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
 ---
 
 This page intends to describe/explain the setup of ansible for this role. 
@@ -25,7 +25,7 @@ There are many ways to layout your inventory within Ansible. To take full advant
     !!! info Info
         The nfc_kubernetes role uses this field for any configuration that requires a hostname. You are strongly encouraged to use DNS name and the DNS name be resolveable for each host accessing to the host in question. Using DNS host name is of paramount importance for a host with dynamic DHCP being used.
 
-- variable `Kubernetes_Master` _boolean_ set for all host that are master nodes.
+- variable `nfc_role_kubernetes_master` _boolean_ set for all host that are master nodes.
 
 - hosts that require Kubernetes API access added to variable `kubernetes_config.cluster.access`
 

docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/firewall.md

@@ -3,14 +3,14 @@ title: Firewall
 description: No Fuss Computings Ansible role nfc_kubernetes
 date: 2023-10-24
 template: project.html
-about: https://gitlab.com/nofusscomputing/projects/ansible/roles/kubernetes
+about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
 ---
 
 This role include logic to generate firewall rules for iptables. Both IPv4 and IPv6 rules are generated. to survive reboots or network cable disconects, a script is created and added to the `if-up.d.` This enables that each time the interface is brought up, the firewall rules are applied. For a list of the firewall rules applied see the [K3s documentation](https://docs.k3s.io/installation/requirements#inbound-rules-for-k3s-server-nodes)
 
 Rules generation workflow:
 
-- itertes over all kubernetes hosts
+- iterates over all kubernetes hosts
 
 - adds rules if host is masters for worker access
 

docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md

@@ -3,23 +3,40 @@ title: Kubernetes
 description: No Fuss Computings Ansible role nfc_kubernetes
 date: 2023-10-24
 template: project.html
-about: https://gitlab.com/nofusscomputing/projects/ansible/roles/kubernetes
+about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
 ---
 
-This Ansible role is designed to deploy a K3s Kubernetes cluster. After adding your configuration, the cluster will deploy and have a configured CNI (calico) and be in a state ready to use. This role can be used with our [our playbooks](../../playbooks/index.md) or comes included, along with the playbook within our [Ansible Execution Environment](../../execution_environment/index.md).
+This Ansible role is designed to deploy a K3s Kubernetes cluster. Without adding cluster configuration this role will install K3s as a single node cluster. To deploy a multi-node cluster add your configuration, K3s will be installed on all nodes. On completion you will have fully configured cluster in a state ready to use. This role can be used with our [our playbooks](../../../../playbooks/index.md) or comes included, along with the playbook within our [Ansible Execution Environment](../../../../execution_environment/index.md).
+
+
+## Role Details
+
+| Item| Value | Description |
+|:---|:---:|:---|
+| Dependent Roles | _None_ | |
+| Optional Roles | _nfc_firewall_ | Used to setup the firewall for kubernetes. |
+| Idempotent | _Yes_ |  |
+| Stats Available | _Not Yet_ |  |
+| Tags | _Nil_ |  |
+| Requirements | _Gather Facts_  |  |
+|   | _become_ |  |
 
 
 ## Features
 
-- CNI Setup
+- CNI Setup, calico including `calicoctl` plugin
+
+    > `kubectl calico ....` instead of `calicoctl ....`
 
 - Configurable:
 
     - Container Registries
 
-    - etcd snapshot cron schedule
+    - ectd deployment
+    
+        - etcd snapshot cron schedule
 
-    - etcd snapshot retention
+        - etcd snapshot retention
     
     - Cluster Domain
 
@@ -47,6 +64,14 @@ This Ansible role is designed to deploy a K3s Kubernetes cluster. After adding y
 
 - Install MetalLB
 
+- Install KubeVirt including `virtctl` plugin
+
+    > `kubectl virt ....` instead of `virtctl ....`
+
+- Install the Helm Binary
+
+- Upgrade cluster
+
 
 ## Role Workflow
 
@@ -66,6 +91,8 @@ For a more probable than not success this role first installs/configures prime m
 
 1. Configure Kubernetes
 
+1. Install Kubevirt
+
 If the playbook is setup as per [our recommendation](ansible.md) step 2 onwards is first done on master nodes then worker nodes.
 
 !!! tip
@@ -79,11 +106,14 @@ If the playbook is setup as per [our recommendation](ansible.md) step 2 onwards
     ```
     _See default variables below for explanation of each variable if it's not evident enough._
 
+
 ## Default Variables
 
+On viewing these variables you will notice there are single dictionary keys prefixed `nfc_role_kubernetes_` and a dictionary of dictionaries `kubernetes_config`. variables prefixed with `nfc_role_kubernetes_` are for single node installs with the `kubernetes_config` dictionary containing all of the information for an entire cluster. The `kubernetes_config` dictionary variables take precedence. Even if you are installing a cluster on multiple nodes, you are still advised to review the variables prefixed with `nfc_role_kubernetes_` as they may still be needed. i.e. setting a node type use keys `nfc_role_kubernetes_prime`, `nfc_role_kubernetes_master` and `nfc_role_kubernetes_worker`.
+
 
 ``` yaml title="defaults/main.yaml" linenums="1"
 
---8<-- "defaults/main.yml"
+--8<-- "roles/nfc_kubernetes/defaults/main.yml"
 
 ```

docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/rbac.md

@@ -3,7 +3,7 @@ title: RBAC
 description: No Fuss Computings Ansible role nfc_kubernetes RBAC documentation.
 date: 2023-10-29
 template: project.html
-about: https://gitlab.com/nofusscomputing/projects/ansible/roles/kubernetes
+about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
 ---
 
 As part of this roles workflow, A set of Clester Roles and Cluster Bindings are deployed and ready to use. The intent of these roles is to create a default set of roles that only require the authorization system to provide the users groups. As they have been defined as Cluster Roles you can bind to both cluster and/or namespace.  

docs/projects/ansible/collection/kubernetes/roles/nfc_kubernetes/release_notes.md

@@ -3,7 +3,7 @@ title: Release Notes
 description: No Fuss Computings Ansible role nfc_kubernetes
 date: 2024-01-31
 template: project.html
-about: https://gitlab.com/nofusscomputing/projects/ansible/roles/kubernetes
+about: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
 ---
 
 This document details any changes that have occured that may impact users of this role. It's a rolling document and will be amended from time to time.
@@ -11,6 +11,12 @@ This document details any changes that have occured that may impact users of thi
 
 ## Changes with an impact
 
+- _**13 Mar 2024**_ Container Images now a dictionary. This role has two images `kubevirt_operator` and `tigera_operator`.
+
+    - All Images are stored in dictionary `nfc_role_kubernetes_container_images` with each image using its own dictionary with mandatory keys `registry`, `image` and `tag`. This change has been made to cater for those whom store their images within their inventory as a dict of dict. For instance to use your inventory image declare variable `nfc_role_kubernetes_container_images.kubevirt_operator: my_images.my_kubevirt_dict` as an example.
+
+    - A lot of variables have been updated. To view what has changed, please see `defaults/main.yaml` in [MR !35](https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/merge_requests/35)
+
 - _**31 Jan 2024**_ Calico CNI deployment has been migrated to use the calico operator. 
 
     - All new cluster installations will be deployed with the operator

galaxy.yml

@@ -0,0 +1,87 @@
+### REQUIRED
+# The namespace of the collection. This can be a company/brand/organization or product namespace under which all
+# content lives. May only contain alphanumeric lowercase characters and underscores. Namespaces cannot start with
+# underscores or numbers and cannot contain consecutive underscores
+namespace: nofusscomputing
+
+# The name of the collection. Has the same character restrictions as 'namespace'
+name: kubernetes
+
+# The version of the collection. Must be compatible with semantic versioning
+version: 1.8.1-a2
+
+# The path to the Markdown (.md) readme file. This path is relative to the root of the collection
+readme: README.md
+
+# A list of the collection's content authors. Can be just the name or in the format 'Full Name <email> (url)
+# @nicks:irc/im.site#channel'
+authors:
+  - No Fuss Computing
+
+
+### OPTIONAL but strongly recommended
+# A short summary description of the collection
+description: Install a K3s Kubernetes Cluster
+
+# Either a single license or a list of licenses for content inside of a collection. Ansible Galaxy currently only
+# accepts L(SPDX,https://spdx.org/licenses/) licenses. This key is mutually exclusive with 'license_file'
+license:
+  - MIT
+
+# The path to the license file for the collection. This path is relative to the root of the collection. This key is
+# mutually exclusive with 'license'
+license_file: ''
+
+# A list of tags you want to associate with the collection for indexing/searching. A tag name has the same character
+# requirements as 'namespace' and 'name'
+tags:
+  - k3s
+  - kubernetes
+  - tools
+
+# Collections that this collection requires to be installed for it to be usable. The key of the dict is the
+# collection label 'namespace.name'. The value is a version range
+# L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version
+# range specifiers can be set and are separated by ','
+dependencies:
+  ansible.posix: '>=1.5.4'
+  kubernetes.core: '>=3.0.0'
+  nofusscomputing.firewall: '>=1.1.0'
+  netbox.netbox: '>=3.16.0'
+
+
+# The URL of the originating SCM repository
+repository: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
+
+# The URL to any online docs
+documentation: https://nofusscomputing.com/projects/ansible/collection/kubernetes/
+
+# The URL to the homepage of the collection/project
+# homepage: https://example.com
+
+# The URL to the collection issue tracker
+issues: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes/-/issues
+
+# A list of file glob-like patterns used to filter any files or directories that should not be included in the build
+# artifact. A pattern is matched from the relative path of the file or directory of the collection directory. This
+# uses 'fnmatch' to match the files or directories. Some directories and files like 'galaxy.yml', '*.pyc', '*.retry',
+# and '.git' are always filtered. Mutually exclusive with 'manifest'
+build_ignore:
+  - .vscode
+  - artifacts
+  - docs
+  - .git*
+  - gitlab-ci
+  - website-template
+  - .ansible-lint-ignore
+  - .cz.yaml
+  - .nfc_automation.yaml
+  - dockerfile
+  - mkdocs.yml
+
+# A dict controlling use of manifest directives used in building the collection artifact. The key 'directives' is a
+# list of MANIFEST.in style
+# L(directives,https://packaging.python.org/en/latest/guides/using-manifest-in/#manifest-in-commands). The key
+# 'omit_default_directives' is a boolean that controls whether the default directives are used. Mutually exclusive
+# with 'build_ignore'
+# manifest: null

handlers/main.yml

@@ -1,53 +0,0 @@
----
-- name: "restart ContainerD"
-  service:
-    name: containerd
-    state: restarted
-  when: >
-    containerd_config.changed | default(false) | bool
-      and
-    containerd_installed.rc | default(1) | int == 0
-      and
-    kubernetes_type == 'k8s'
-  tags:
-    - configure
-    - install
-
-
-- name: Restart Kubernetes
-  ansible.builtin.service:
-    name: |-
-      {%- if kubernetes_type == 'k3s' -%}
-        {%- if Kubernetes_Master | default(false) | bool -%}
-          k3s
-        {%- else -%}
-          k3s-agent
-        {%- endif -%}
-      {%- else -%}
-        kubelet
-      {%- endif %}
-    state: restarted
-  listen: kubernetes_restart
-  when: |-
-    not (
-      nfc_kubernetes_no_restart
-        or
-      (
-        inventory_hostname in groups['kubernetes_master']
-          and
-        nfc_kubernetes_no_restart_master
-      )
-        or
-      (
-        inventory_hostname == kubernetes_config.cluster.prime.name
-          and
-        nfc_kubernetes_no_restart_prime
-      )
-        or
-      (
-        inventory_hostname in groups['kubernetes_worker']
-          and
-        nfc_kubernetes_no_restart_slave
-      )
-    )
-

meta/runtime.yml

@@ -0,0 +1,52 @@
+---
+# Collections must specify a minimum required ansible version to upload
+# to galaxy
+requires_ansible: '>=2.14.0'
+
+# Content that Ansible needs to load from another location or that has
+# been deprecated/removed
+# plugin_routing:
+#   action:
+#     redirected_plugin_name:
+#       redirect: ns.col.new_location
+#     deprecated_plugin_name:
+#       deprecation:
+#         removal_version: "4.0.0"
+#         warning_text: |
+#           See the porting guide on how to update your playbook to
+#           use ns.col.another_plugin instead.
+#     removed_plugin_name:
+#       tombstone:
+#         removal_version: "2.0.0"
+#         warning_text: |
+#           See the porting guide on how to update your playbook to
+#           use ns.col.another_plugin instead.
+#   become:
+#   cache:
+#   callback:
+#   cliconf:
+#   connection:
+#   doc_fragments:
+#   filter:
+#   httpapi:
+#   inventory:
+#   lookup:
+#   module_utils:
+#   modules:
+#   netconf:
+#   shell:
+#   strategy:
+#   terminal:
+#   test:
+#   vars:
+
+# Python import statements that Ansible needs to load from another location
+# import_redirection:
+#   ansible_collections.ns.col.plugins.module_utils.old_location:
+#     redirect: ansible_collections.ns.col.plugins.module_utils.new_location
+
+# Groups of actions/modules that take a common set of options
+# action_groups:
+#   group_name:
+#     - module1
+#     - module2

mkdocs.yml

@@ -2,9 +2,9 @@ INHERIT: website-template/mkdocs.yml
 
 docs_dir: 'docs'
 
-repo_name: Kubernetes Ansible Role
-repo_url: https://gitlab.com/nofusscomputing/projects/ansible/kubernetes
-edit_uri: '/-/ide/project/nofusscomputing/projects/ansible/kubernetes/edit/development/-/docs/'
+repo_name: Kubernetes Ansible Collection
+repo_url: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
+edit_uri: '/-/ide/project/nofusscomputing/projects/ansible/collections/kubernetes/edit/development/-/docs/'
 
 nav:
 - Home: index.md
@@ -29,22 +29,29 @@ nav:
 
       - projects/ansible/playbooks/index.md
 
-    - Roles:
+    - Collections:
 
-      - projects/ansible/roles/index.md
+      - projects/ansible/collection/index.md
 
       - Kubernetes:
 
-        - projects/ansible/roles/kubernetes/index.md
+        - projects/ansible/collection/kubernetes/index.md
 
-        - projects/ansible/roles/kubernetes/ansible.md
+        - Role nfc_kubernetes:
 
-        - projects/ansible/roles/kubernetes/firewall.md
+          - projects/ansible/collection/kubernetes/roles/nfc_kubernetes/index.md
 
-        - projects/ansible/roles/kubernetes/rbac.md
+          - projects/ansible/collection/kubernetes/roles/nfc_kubernetes/ansible.md
 
-        - projects/ansible/roles/kubernetes/release_notes.md
+          - projects/ansible/collection/kubernetes/roles/nfc_kubernetes/firewall.md
 
+          - projects/ansible/collection/kubernetes/roles/nfc_kubernetes/rbac.md
+
+          - projects/ansible/collection/kubernetes/roles/nfc_kubernetes/release_notes.md
+
+        - Role kubernetes_netbox:
+
+          - projects/ansible/collection/kubernetes/roles/kubernetes_netbox/index.md
 
 - Operations: 
 

playbooks/install.yaml

@@ -0,0 +1,64 @@
+---
+- name: Install K3s Kubernetes
+  hosts: |-
+    {%- if nfc_pb_host is defined -%}
+
+      {{ nfc_pb_host }}
+
+    {%- elif nfc_pb_kubernetes_cluster_name is defined -%}
+
+      kubernetes_cluster_{{ nfc_pb_kubernetes_cluster_name | lower }}
+
+    {%- else -%}
+
+      {%- if ansible_limit is defined -%}
+
+        {{ ansible_limit }}
+
+      {%- else -%}
+
+        localhost
+
+      {%- endif -%}
+
+    {%- endif %}
+  become: true
+  gather_facts: true
+
+
+  tasks:
+
+
+    - name: Install/Configure Kubernetes
+      ansible.builtin.include_role:
+        name: nfc_kubernetes
+      tags:
+        - always
+
+
+  # vars:
+
+  #
+  #   Future feature, add playbook to import to awx
+  #
+  #   nfc_pb_awx_tower_template:
+
+
+  #     - name: "Collection/NoFussComputing/Kubernetes/Install"
+  #       ask_credential_on_launch: true
+  #       ask_job_type_on_launch: true
+  #       ask_limit_on_launch: true
+  #       ask_tags_on_launch: true
+  #       ask_variables_on_launch: true
+  #       description: |
+  #         Playbook to Install/Configure Kubernetes using configuration
+  #         from code.
+  #       execution_environment: "No Fuss Computing EE"
+  #       job_type: "check"
+  #       labels:
+  #         - cluster
+  #         - k3s
+  #         - kubernetes
+  #       verbosity: 2
+  #       use_fact_cache: true
+  #       survey_enabled: false

playbooks/netbox.yaml

@@ -0,0 +1,64 @@
+---
+- name: Install K3s Kubernetes
+  hosts: |-
+    {%- if nfc_pb_host is defined -%}
+
+      {{ nfc_pb_host }}
+
+    {%- elif nfc_pb_kubernetes_cluster_name is defined -%}
+
+      kubernetes_cluster_{{ nfc_pb_kubernetes_cluster_name | lower }}
+
+    {%- else -%}
+
+      {%- if ansible_limit is defined -%}
+
+        {{ ansible_limit }}
+
+      {%- else -%}
+
+        localhost
+
+      {%- endif -%}
+
+    {%- endif %}
+  become: false
+  gather_facts: false
+
+
+  tasks:
+
+
+    - name: Configure NetBox for Kubernetes Deployment(s)
+      ansible.builtin.include_role:
+        name: kubernetes_netbox
+      tags:
+        - always
+
+
+  # vars:
+
+  #
+  #   Future feature, add playbook to import to awx
+  #
+  #   nfc_pb_awx_tower_template:
+
+
+  #     - name: "Collection/NoFussComputing/Kubernetes/NetBox/Configure"
+  #       ask_credential_on_launch: true
+  #       ask_job_type_on_launch: true
+  #       ask_limit_on_launch: true
+  #       ask_tags_on_launch: true
+  #       ask_variables_on_launch: true
+  #       description: |
+  #         Playbook to Install/Configure Kubernetes using configuration
+  #         from code.
+  #       execution_environment: "No Fuss Computing EE"
+  #       job_type: "check"
+  #       labels:
+  #         - cluster
+  #         - k3s
+  #         - kubernetes
+  #       verbosity: 2
+  #       use_fact_cache: true
+  #       survey_enabled: false

requirements.txt

@@ -0,0 +1,2 @@
+pynetbox
+pytz

roles/defaults/main.yaml

@@ -0,0 +1,9 @@
+---
+
+#
+# NetBox Access Variables. Required
+#
+
+# nfc_pb_api_netbox_url:                   # ENV [NETBOX_API]
+# nfc_pb_api_netbox_token:                 # ENV [NETBOX_TOKEN]
+# nfc_pb_api_netbox_validate_cert: true    # ENV [NETBOX_VALIDATE_CERT]

roles/kubernetes_netbox/README.md

@@ -0,0 +1,3 @@
+## No Fuss Computing - Ansible Role kubernetes_netbox
+
+Nothing to see here

roles/kubernetes_netbox/meta/main.yaml

@@ -0,0 +1,30 @@
+galaxy_info:
+
+  role_name: kubernetes_netbox
+
+  author: No Fuss Computing
+
+  description: Configure the required items within Netbox to support deploying kubernetes from Netbox configuration.
+
+  issue_tracker_url: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
+
+  license: MIT
+
+  min_ansible_version: '2.15'
+
+  platforms:
+
+    - name: Debian
+      versions:
+        - bullseye
+        - bookworm
+
+    - name: Ubuntu
+      versions:
+        - 21
+
+  galaxy_tags:
+    - cluster
+    - k3s
+    - kubernetes
+    - netbox

roles/kubernetes_netbox/tasks/cluster.yaml

@@ -0,0 +1,255 @@
+---
+
+# add cluster type kubernetes
+
+- name: Create Custom Field - Configure Firewall
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Configure Firewall
+      name: nfc_role_kubernetes_configure_firewall
+      type: boolean
+      ui_visibility: 'hidden-ifunset'
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - ETCD Enabled
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: ETCD Enabled
+      name: nfc_role_kubernetes_etcd_enabled
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Install OLM
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Install OLM
+      name: nfc_role_kubernetes_install_olm
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Install Helm
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Install Helm
+      name: nfc_role_kubernetes_install_helm
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Install KubeVirt
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Install KubeVirt
+      name: nfc_role_kubernetes_install_kubevirt
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - KubeVirt Operator Replicas
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: KubeVirt Operator Replicas
+      name: nfc_role_kubernetes_kubevirt_operator_replicas
+      type: integer
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      validation_minimum: 1
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Enable MetalLB
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Enable MetalLB
+      name: nfc_kubernetes_enable_metallb
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Enable ServiceLB (klipper)
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Enable ServiceLB (klipper)
+      name: nfc_kubernetes_enable_servicelb
+      type: boolean
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Pod Subnet
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Pod Subnet
+      name: nfc_role_kubernetes_pod_subnet
+      object_type: ipam.prefix
+      type: object
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Service Subnet
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - virtualization.cluster
+      default: null
+      group_name: Kubernetes
+      label: Service Subnet
+      name: nfc_role_kubernetes_service_subnet
+      object_type: ipam.prefix
+      type: object
+      ui_visibility: hidden-ifunset
+      # is_cloneable: false
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp

roles/kubernetes_netbox/tasks/main.yaml

@@ -0,0 +1,21 @@
+---
+
+- name: Setup NetBox for Kubernetes Cluster Deployments
+  ansible.builtin.include_tasks:
+    file: cluster.yaml
+    apply:
+      tags:
+        - always
+  tags:
+    - always
+
+
+- name: Setup NetBox for Kubernetes Service Deployments
+  ansible.builtin.include_tasks:
+    file: services.yaml
+    apply:
+      tags:
+        - always
+  tags:
+    - never
+    - services

roles/kubernetes_netbox/tasks/services.yaml

@@ -0,0 +1,50 @@
+---
+
+- name: Create Custom Field - Instance
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - ipam.service
+      group_name: Kubernetes
+      label: Instance Name
+      description: "Name of the Instance to be deployed"
+      name: service_kubernetes_instance
+      type: text
+      ui_visibility: hidden-ifunset
+      # is_cloneable: true
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp
+
+
+- name: Create Custom Field - Namespace
+  netbox.netbox.netbox_custom_field:
+    netbox_url: "{{ lookup('env', 'NETBOX_API') | default(nfc_pb_api_netbox_url) }}"
+    netbox_token: "{{ lookup('env', 'NETBOX_TOKEN') | default(nfc_pb_api_netbox_token) }}"
+    data:
+      content_types:
+        - ipam.service
+      group_name: Kubernetes
+      label: Service Namespace
+      description: "Deployment Namespace"
+      name: service_kubernetes_namespace
+      type: text
+      ui_visibility: hidden-ifunset
+      # is_cloneable: true
+      weight: 100
+    state: present
+    validate_certs: "{{ lookup('env', 'NETBOX_VALIDATE_CERT') | default(nfc_pb_api_netbox_validate_cert) | default(true) | bool }}"
+  delegate_to: localhost
+  failed_when: >
+    custom_field_tmp.msg != 'ui_visibility does not exist on existing object. Check to make sure valid field.'
+      and
+    custom_field_tmp.diff is not defined
+  register: custom_field_tmp

roles/nfc_kubernetes/README.md

@@ -0,0 +1,3 @@
+## No Fuss Computing - Ansible Role nfc_kubernetes
+
+Nothing to see here

roles/nfc_kubernetes/defaults/main.yml

@@ -6,10 +6,10 @@
 calico_image_tag: v3.25.0 # Depreciated
 # EoF Depreciated
 # SoF New Variables
-nfc_kubernetes_calico_version: v3.27.0
-nfc_kubernetes_tigera_operator_registry: quay.io
-nfc_kubernetes_tigera_operator_image: tigera/operator
-nfc_kubernetes_tigera_operator_tag: v1.32.3               # Calico v3.27.0
+nfc_role_kubernetes_calico_version: v3.27.0
+# nfc_kubernetes_tigera_operator_registry: quay.io
+# nfc_kubernetes_tigera_operator_image: tigera/operator
+# nfc_kubernetes_tigera_operator_tag: v1.32.3               # Calico v3.27.0
 # EoF New Variables, EEoF Depreciated
 
 
@@ -17,26 +17,58 @@ nfc_kubernetes_enable_metallb: false
 nfc_kubernetes_enable_servicelb: false
 
 
+nfc_role_kubernetes_container_images:
+
+  kubevirt_operator:
+    name: Kubevirt Operator
+    registry: quay.io
+    image: kubevirt/virt-operator
+    tag: v1.2.0
+
+  tigera_operator:
+    name: Tigera Operator
+    registry: quay.io
+    image: tigera/operator
+    tag: v1.32.3    # Calico v3.27.0
+
+
+nfc_role_kubernetes_cluster_domain: cluster.local
+
+nfc_role_kubernetes_configure_firewall: true
+
+nfc_role_kubernetes_etcd_enabled: false
+
+nfc_role_kubernetes_install_olm: false
+
+nfc_role_kubernetes_install_helm: true
+
+nfc_role_kubernetes_install_kubevirt: false
+
+nfc_role_kubernetes_kubevirt_operator_replicas: 1
+
+nfc_role_kubernetes_oidc_enabled: false
+
+nfc_role_kubernetes_resolv_conf_file: /etc/resolv.conf
+
+nfc_role_kubernetes_pod_subnet: 172.16.248.0/21
+nfc_role_kubernetes_service_subnet: 172.16.244.0/22
+
+nfc_role_kubernetes_prime: true                         # Mandatory for a node designated as the prime master node
+nfc_role_kubernetes_master: true                        # Mandatory for a node designated as a master node and the prime master node
+nfc_role_kubernetes_worker: false                       # Mandatory for a node designated as a worker node
+
 ############################################################################################################
 #
 #                     Old Vars requiring refactoring
 #
 # ############################################################################################################
-# KubernetesPodSubnet: 10.85.0.0/16
-# KubernetesServiceSubnet: 10.86.0.0/16
 
 
-Kubernetes_Prime: false                                    # Optional, Boolean. Is the current host the Prime master?
-Kubernetes_Master: false                                   # Optional, Boolean. Is the current host a master host?
-
-ContainerDioVersion: 1.6.20-1
-KubernetesVersion: '1.26.2'                                # must match the repository release version
-kubernetes_version_olm: '0.26.0'
+KubernetesVersion: '1.26.12'                                # must match the repository release version
+kubernetes_version_olm: '0.27.0'
 
 
 
-
-KubernetesVersion_k8s_prefix: '-00'
 KubernetesVersion_k3s_prefix: '+k3s1'
 
 kubernetes_private_container_registry: []                  # Optional, Array. if none use `[]`
@@ -46,19 +78,15 @@ kubernetes_etcd_snapshot_retention: 5
 
 # host_external_ip: ''                                     # Optional, String. External IP Address for host.
 
-kubernetes_type: k8s                                       # Mandatory, String. choice K8s | k3s
-
 kube_apiserver_arg_audit_log_maxage: 2
 
 kubelet_arg_system_reserved_cpu: 450m
 kubelet_arg_system_reserved_memory: 512Mi
 kubelet_arg_system_reserved_storage: 8Gi
 
-# kubernetes_olm_install: true # optional, boolean. default=true
-
 
 nfc_kubernetes:
-  enable_firewall: true             # Optional, bool enable firewall rules from role 'nfc_firewall'
+  enable_firewall: false             # Optional, bool enable firewall rules from role 'nfc_firewall'
 
 nfc_kubernetes_no_restart: false         # Set to true to prevent role from restarting kubernetes on the host(s)
 nfc_kubernetes_no_restart_master: false  # Set to true to prevent role from restarting kubernetes on master host(s)
@@ -76,7 +104,7 @@ k3s:
         kind: Policy
         rules:
         - level: Request
-      when: "{{ Kubernetes_Master | default(false) }}"
+      when: "{{ nfc_role_kubernetes_master }}"
 
     - name: 90-kubelet.conf
       path: /etc/sysctl.d
@@ -108,7 +136,7 @@ k3s:
         #       usernames: []
         #       runtimeClasses: []
         #       namespaces: [kube-system]
-      when: "{{ kubernetes_config.cluster.prime.name == inventory_hostname }}"
+      when: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname }}"
 
 
 #############################################################################################
@@ -123,7 +151,7 @@ k3s:
 #       - 'my.dnshostname.com'
 #       - '2001:4860:4860::8888'
 #       - '192.168.1.1'
-#     Name: earth                       # Mandatory, String. Cluster Name
+#     domain_name: earth                # Mandatory, String. Cluster Domain Name
 #     group_name:                       # Mandatory, String. name of the ansible inventory group containg all cluster hosts
 #     prime:
 #       name: k3s-prod                  # Mandatory, String. Ansible inventory_host that will
@@ -132,10 +160,21 @@ k3s:
 #       encrypt: true                   # Optional, Boolean. default `false`. Install wireguard for inter-node encryption
 #       podSubnet: 172.16.70.0/24       # Mandatory, String. CIDR
 #       ServiceSubnet: 172.16.72.0/24   # Mandatory, String. CIDR
-#                                       # Mandatory, String. Token to join nodes to the cluster
-#   node_token: !vault |
-#     $ANSIBLE_VAULT;1.2;AES256;kubernetes/cluster/production
-#     {rest_of encrypted key}
+#
+#
+#  helm:
+#    enabled: true       # Optional, Boolean. default=false. Install Helm Binary
+#
+#
+#  kube_virt:
+#    enabled: false      # Optional, Boolean. default=false. Install KubeVirt
+#
+#    nodes: []           # Optional, List of String. default=inventory_hostname. List of nodes to install kibevirt on.
+#
+#    operator:
+#      replicas: 2       # Optional, Integer. How many virt_operators to deploy.
+#
+#
 #  oidc:                                                    # Used to configure Kubernetes with OIDC Authentication.
 #    enabled: true                                          # Mandatory, boolen. speaks for itself.
 #    issuer_url: https://domainname.com/realms/realm-name   # Mandatory, String. URL of OIDC Provider
@@ -144,14 +183,14 @@ k3s:
 #    username_prefix: oidc                                  # Optional, String. What to prefix to username
 #    groups_claim: roles                                    # Mandatory, String. Claim name containing groups
 #    groups_prefix: ''                                      # Optional, String. string to append to groups
-
-#     hosts: 
-
-#       my-host-name:
-#         labels:
-#           mylabel: myvalue
-
-#         taints:
-#           - effect: NoSchedule
-#             key: taintkey
-#             value: taintvalue
+#
+#   hosts:
+#
+#     my-host-name:
+#       labels:
+#         mylabel: myvalue
+#
+#       taints:
+#         - effect: NoSchedule
+#           key: taintkey
+#           value: taintvalue

roles/nfc_kubernetes/handlers/main.yml

@@ -0,0 +1,41 @@
+---
+
+- name: Reboot Node
+  ansible.builtin.reboot:
+    reboot_timeout: 300
+  listen: reboot_host
+  when: ansible_connection == 'ssh'
+  
+
+- name: Restart Kubernetes
+  ansible.builtin.service:
+    name: |-
+      {%- if nfc_role_kubernetes_master | default(false) | bool -%}
+        k3s
+      {%- else -%}
+        k3s-agent
+      {%- endif -%}
+    state: restarted
+  listen: kubernetes_restart
+  when: |-
+    not (
+      nfc_kubernetes_no_restart
+        or
+      (
+        nfc_role_kubernetes_master
+          and
+        nfc_kubernetes_no_restart_master
+      )
+        or
+      (
+        inventory_hostname == kubernetes_config.cluster.prime.name | default(inventory_hostname)
+          and
+        nfc_kubernetes_no_restart_prime
+      )
+        or
+      (
+        nfc_role_kubernetes_worker
+          and
+        nfc_kubernetes_no_restart_slave
+      )
+    )

roles/nfc_kubernetes/meta/main.yml

@@ -1,24 +1,29 @@
 galaxy_info:
+
   role_name: nfc_kubernetes
+
   author: No Fuss Computing
-  description: template role to install and configure Kubernetes on a host
 
-  issue_tracker_url: https://gitlab.com/nofusscomputing/projects/ansible/kubernetes
+  description: Install and configure single and multi-node K3s Kubernetes cluster.
 
-  license: https://gitlab.com/nofusscomputing/projects/ansible/kubernetes/-/blob/master/LICENSE
+  issue_tracker_url: https://gitlab.com/nofusscomputing/projects/ansible/collections/kubernetes
+
+  license: MIT
 
   min_ansible_version: '2.15'
 
   platforms:
+
     - name: Debian
       versions:
         - bullseye
+        - bookworm
+
     - name: Ubuntu
       versions:
         - 21
 
   galaxy_tags:
+    - cluster
     - k3s
-    - k8s
     - kubernetes
-    - container

roles/nfc_kubernetes/tasks/helm/main.yaml

@@ -0,0 +1,30 @@
+---
+
+- name: Fetch Helm APT Key
+  ansible.builtin.get_url:
+    url: https://baltocdn.com/helm/signing.asc
+    dest: /usr/share/keyrings/helm.asc
+    mode: 740
+  changed_when: not ansible_check_mode
+  delay: 10
+  retries: 3
+
+
+- name: Add Helm Repository
+  ansible.builtin.apt_repository:
+    repo: >-
+      deb [arch={%- if ansible_architecture == 'aarch64' -%}
+        arm64
+      {%- else -%}
+        amd64
+      {%- endif %} signed-by=/usr/share/keyrings/helm.asc] http://baltocdn.com/helm/stable/{{
+      ansible_os_family | lower }}/ all main
+    state: present
+    filename: helm
+
+
+- name: Install Helm
+  ansible.builtin.apt:
+    package:
+      - helm
+    state: present

roles/nfc_kubernetes/tasks/install.yaml

@@ -0,0 +1,147 @@
+---
+
+- name: Get Hostname
+  ansible.builtin.command:
+    cmd: hostname
+  changed_when: false
+  check_mode: false
+  register: hostname_to_check
+  tags:
+    - always
+
+
+- name: Hostname Check
+  ansible.builtin.assert:
+    that:
+      - hostname_to_check.stdout == inventory_hostname
+    msg: The hostname must match the inventory_hostname
+  tags:
+    - always
+  when: >
+    inventory_hostname != 'localhost'
+
+
+- name: Testing Env Variables
+  ansible.builtin.set_fact:
+    ansible_default_ipv4: {
+      "address": "127.0.0.1"
+    }
+  check_mode: false
+  tags:
+    - always
+  when: >
+    lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != ''
+
+
+- name: Gather Facts required by role
+  ansible.builtin.setup:
+    gather_subset:
+      - all_ipv4_addresses
+      - os_family
+      - processor
+  tags:
+    - always
+  when: >
+    ansible_architecture is not defined
+      or
+    ansible_default_ipv4 is not defined
+      or
+    ansible_os_family is not defined
+
+
+- name: Check Machine Architecture
+  ansible.builtin.set_fact:
+    nfc_kubernetes_install_architectures: "{{ nfc_kubernetes_install_architectures | default({}) | combine({ansible_architecture: ''}) }}"
+  tags:
+    - always
+
+
+- name: Configure Kubernetes Firewall Rules
+  ansible.builtin.include_role:
+    name: nofusscomputing.firewall.nfc_firewall
+  vars:
+    nfc_role_firewall_firewall_type: iptables
+    nfc_role_firewall_additional_rules: "{{ ( lookup('template', 'vars/firewall_rules.yaml') | from_yaml ).kubernetes_chains }}"
+  tags:
+    - always
+  when: >
+    nfc_role_kubernetes_configure_firewall
+
+
+- name: Install required software
+  ansible.builtin.apt:
+    name: python3-pip
+    install_recommends: false
+    state: present
+  when: >
+    install_kubernetes | default(true) | bool
+      and
+    not kubernetes_installed | default(false) | bool
+  tags:
+    - always
+
+
+- name: K3s Install
+  ansible.builtin.include_tasks:
+    file: k3s/install.yaml
+    apply:
+      tags:
+        - always
+  when: >
+    install_kubernetes | default(true) | bool
+      and
+    not kubernetes_installed | default(false) | bool
+  tags:
+    - always
+
+
+- name: K3s Configure
+  ansible.builtin.include_tasks:
+    file: k3s/configure.yaml
+    apply:
+      tags:
+        - always
+  when: >
+    install_kubernetes | default(true) | bool
+      and
+    kubernetes_installed | default(false) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
+  tags:
+    - always
+
+
+- name: Kubevert
+  ansible.builtin.include_tasks:
+    file: kubevirt/main.yaml
+    apply:
+      tags:
+        - always
+  when: >
+    kubernetes_installed | default(false) | bool
+      and
+    kubernetes_config.kube_virt.enabled | default(nfc_role_kubernetes_install_kubevirt)
+      and
+    inventory_hostname in kubernetes_config.kube_virt.nodes | default([ inventory_hostname ]) | list
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
+  tags:
+    - always
+
+
+- name: Helm
+  ansible.builtin.include_tasks:
+    file: helm/main.yaml
+    apply:
+      tags:
+        - always
+  when: >
+    kubernetes_installed | default(false) | bool
+      and
+    kubernetes_config.helm.enabled | default(nfc_role_kubernetes_install_helm)
+      and
+    nfc_role_kubernetes_master
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
+  tags:
+    - always

roles/nfc_kubernetes/tasks/k3s/configure.yaml

@@ -12,6 +12,12 @@
   when: item.when | default(false) | bool
 
 
+- name: Check if FW dir exists
+  ansible.builtin.stat:
+    name: /etc/iptables-reloader/rules.d
+  register: firewall_rules_dir_metadata
+
+
 - name: Copy Templates
   ansible.builtin.template:
     src: "{{ item.src }}"
@@ -28,11 +34,21 @@
 
       - src: kubernetes-manifest-rbac.yaml.j2
         dest: /var/lib/rancher/k3s/server/manifests/rbac-authorization-common.yaml
-        when: "{{ kubernetes_config.cluster.prime.name == inventory_hostname }}"
+        when: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname }}"
 
       - src: iptables-kubernetes.rules.j2
-        dest: "/etc/iptables.rules.d/iptables-kubernetes.rules"
+        dest: "/etc/iptables-reloader/rules.d/iptables-kubernetes.rules"
         notify: firewall_reloader
+        when: |-
+          {%- if firewall_installed -%}
+
+            {{ firewall_rules_dir_metadata.stat.exists }}
+
+          {%- else -%}
+
+            false
+
+          {%- endif %}
 
 
 - name: Add Kubernetes Node Labels

roles/nfc_kubernetes/tasks/k3s/install.yaml

@@ -1,30 +1,42 @@
 ---
 
+- name: Install required python modules
+  ansible.builtin.pip:
+    name: "{{ item }}"
+    state: present
+  loop: "{{ pip_packages }}"
+  vars:
+    pip_packages:
+      - kubernetes>=12.0.0
+      - PyYAML>=3.11
+
+
 - name: Check for calico deployment manifest
   ansible.builtin.stat:
     name: /var/lib/rancher/k3s/server/manifests/calico.yaml
-  become: true
   register: file_calico_yaml_metadata
 
 
 - name: Check for calico Operator deployment manifest
   ansible.builtin.stat:
     name: /var/lib/rancher/k3s/ansible/deployment-manifest-calico_operator.yaml
-  become: true
   register: file_calico_operator_yaml_metadata
 
 
-- name: Install Software
-  ansible.builtin.include_role:
-    name: nfc_common
+- name: Install dependent packages
+  ansible.builtin.apt:
+    name: "{{ package }}"
+    state: present
+  loop: "{{ packages }}"
+  loop_control:
+    loop_var: package
   vars:
-    common_gather_facts: false
-    initial_common_tasks: true # Don't run init tasks
-    aptInstall:
-      - name: curl
-      - name: iptables
-      - name: jq
-      - name: wireguard
+    packages:
+      - wget
+      - curl
+      - iptables
+      - jq
+      - wireguard
 
 
 - name: Remove swapfile from /etc/fstab
@@ -41,14 +53,29 @@
     - install
 
 
-- name: Disable swap
-  ansible.builtin.command:
-    cmd: swapoff -a
-  changed_when: false
-  when:
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
+- name: Testing Environment try/catch
+  block:
+
+
+    - name: Disable swap
+      ansible.builtin.command:
+        cmd: swapoff -a
+      changed_when: false
+      when:
+        - ansible_os_family == 'Debian'
+      tags:
+        - install
+
+
+  rescue:
+
+    - name: Check if inside Gitlab CI
+      ansible.builtin.assert:
+        that:
+          - lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != ''
+        success_msg: "Inside testing enviroment, 'Disable swap' error OK"
+        fail_msg: "You should figure out what went wrong"
+
 
 - name: Check an armbian os system
   ansible.builtin.stat:
@@ -106,6 +133,8 @@
         value: '524288'
       - name: fs.inotify.max_user_instances
         value: '512'
+      - name: net.ipv6.conf.all.disable_ipv6
+        value: '1'
   when:
     - ansible_os_family == 'Debian'
 
@@ -113,7 +142,6 @@
 - name: Check for Network Manager Directory
   ansible.builtin.stat:
     name: /etc/NetworkManager/conf.d
-  become: true
   register: directory_network_manager_metadata
 
 
@@ -133,24 +161,249 @@
     mode: '770'
     owner: root
     group: root
-  become: true
   diff: true
   when: directory_network_manager_metadata.stat.exists
 
 
-- name: Check if K3s Installed
+- name: File Metadata - k3s binary
+  ansible.builtin.stat:
+    checksum_algorithm: sha256
+    name: /usr/local/bin/k3s
+  register: metadata_file_k3s_existing_binary
+
+
+- name: File Metadata - k3s[-agent].service
+  ansible.builtin.stat:
+    checksum_algorithm: sha256
+    name: |-
+      /etc/systemd/system/k3s
+      {%- if not nfc_role_kubernetes_master | default(false) | bool -%}
+        -agent
+      {%- endif -%}
+      .service
+  register: metadata_file_k3s_service
+
+
+- name: Directory Metadata - /etc/rancher/k3s/k3s.yaml
+  ansible.builtin.stat:
+    name: /etc/rancher/k3s/k3s.yaml
+  register: metadata_dir_etc_k3s
+
+
+- name: File Metadata - /var/lib/rancher/k3s/server/token
+  ansible.builtin.stat:
+    checksum_algorithm: sha256
+    name: /var/lib/rancher/k3s/server/token
+  register: metadata_file_var_k3s_token
+
+
+- name: Config Link
+  ansible.builtin.shell:
+    cmd: >
+      ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config
+    executable: bash
+    creates: ~/.kube/config
+  when: >
+    nfc_role_kubernetes_master | default(false) | bool
+      and
+    metadata_dir_etc_k3s.stat.exists | default(false) | bool
+
+
+- name: Fetch Kubernetes Node Object
+  kubernetes.core.k8s_info:
+    kind: Node
+    name: "{{ inventory_hostname }}"
+  register: kubernetes_node
+  when: >
+    metadata_file_k3s_existing_binary.stat.exists | default(false) | bool
+      and
+    metadata_file_k3s_service.stat.exists | default(false) | bool
+      and
+    metadata_dir_etc_k3s.stat.exists | default(false) | bool
+      and
+    metadata_file_var_k3s_token.stat.exists | default(false) | bool
+
+
+- name: Fetch Installed K3s Metadata
   ansible.builtin.shell:
     cmd: |
-      if [[ $(service k3s status) ]]; then exit 0; else exit 1; fi
+      export installed_version=$(k3s --version | grep k3s | awk '{print $3}');
+      export installed="
+      {%- if
+        metadata_file_k3s_existing_binary.stat.exists | default(false) | bool
+          and
+        metadata_file_k3s_service.stat.exists | default(false) | bool
+          and
+        metadata_dir_etc_k3s.stat.exists | default(false) | bool
+          and
+        metadata_file_var_k3s_token.stat.exists | default(false) | bool
+      -%}
+        true
+      {%- else -%}
+        false
+      {%- endif -%}";
+
+      if ! service k3s status > /dev/null; then
+
+        export installed='false';
+
+      fi
+
+      export running_version="{{ kubernetes_node.resources[0].status.nodeInfo.kubeletVersion | default('0') }}";
+
+      export correct_hash=$(wget -q https://github.com/k3s-io/k3s/releases/download/v
+        {{-KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}}
+          /sha256sum-
+          {%- if ansible_architecture | lower == 'x86_64' -%}
+            amd64
+          {%- elif ansible_architecture | lower == 'aarch64' -%}
+            arm64
+          {%- endif %}.txt -O - | grep -v 'images' | awk '{print $1}');
+
+      cat <<EOF
+      {
+        "current_hash": "{{ metadata_file_k3s_existing_binary.stat.checksum | default('') }}",
+        "current_version": "${installed_version}",
+        "desired_hash": "${correct_hash}",
+        "desired_version": "v{{ KubernetesVersion + KubernetesVersion_k3s_prefix | default('') }}",
+        "installed": ${installed},
+        "running_version": "${running_version}"
+      }
+
+      EOF
+
     executable: /bin/bash
   changed_when: false
+  check_mode: false
   failed_when: false
-  register: k3s_installed
+  register: k3s_metadata
 
 
-- name: Check Machine Architecture
+- name: K3s Metadata Fact
   ansible.builtin.set_fact:
-    nfc_kubernetes_install_architectures: "{{ nfc_kubernetes_install_architectures | default({}) | combine({ansible_architecture: ''}) }}"
+    node_k3s: "{{ k3s_metadata.stdout | from_yaml }}"
+
+
+- name: Cached K3s Binary Details
+  ansible.builtin.stat:
+    path: "/tmp/k3s.{{ ansible_architecture }}"
+    checksum_algorithm: sha256
+  delegate_to: localhost
+  register: file_cached_k3s_binary
+  vars:
+    ansible_connection: local
+
+
+- name: Remove Cached K3s Binaries
+  ansible.builtin.file:
+    path: "/tmp/k3s.{{ ansible_architecture }}"
+    state: absent
+  delegate_to: localhost
+  vars:
+    ansible_connection: local
+  when: >
+    file_cached_k3s_binary.stat.checksum | default('0') != node_k3s.desired_hash
+
+  # Workaround. See: https://github.com/ansible/awx/issues/15161
+- name: Build K3s Download URL
+  ansible.builtin.set_fact:
+    cacheable: false
+    url_download_k3s: |-
+      [
+        {%- for key, value in nfc_kubernetes_install_architectures | dict2items -%}
+        "https://github.com/k3s-io/k3s/releases/download/
+          {{- node_k3s.desired_version | urlencode -}}
+        /k3s
+        {%- if key == 'aarch64' -%}
+        -arm64
+        {%- endif %}",
+        {%- endfor -%}
+      ]
+  changed_when: false
+  check_mode: false
+  delegate_to: localhost
+  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
+  loop_control:
+    loop_var: cpu_arch
+  vars:
+    ansible_connection: local
+
+
+- name: Download K3s Binary
+  ansible.builtin.uri:
+    url: "{{ url | string }}"
+    method: GET
+    return_content: false
+    status_code:
+      - 200
+      - 304
+    dest: "/tmp/k3s.{{ ansible_architecture }}"
+    mode: "744"
+  changed_when: not ansible_check_mode
+  check_mode: false
+  delay: 10
+  retries: 3
+  register: k3s_download_files
+  delegate_to: localhost
+  failed_when: >
+    (lookup('ansible.builtin.file', '/tmp/k3s.' + ansible_architecture) | hash('sha256') | string) != node_k3s.desired_hash
+      and
+    (
+      k3s_download_files.status | int != 200
+        or
+      k3s_download_files.status | int != 304
+    )
+  run_once: true
+  when: ansible_os_family == 'Debian'
+  loop: "{{ url_download_k3s | from_yaml }}"
+  loop_control:
+    loop_var: url
+  vars:
+    ansible_connection: local
+
+
+- name: Copy K3s binary to Host
+  ansible.builtin.copy:
+    src: "/tmp/k3s.{{ ansible_architecture }}"
+    dest: "/usr/local/bin/k3s"
+    mode: '741'
+    owner: root
+    group: root
+  register: k3s_binary_copy
+  when: >
+    node_k3s.current_hash != node_k3s.desired_hash
+
+
+- name: K3s Binary Upgrade
+  ansible.builtin.service:
+    name: |-
+      {%- if nfc_role_kubernetes_master | default(false) | bool -%}
+        k3s
+      {%- else -%}
+        k3s-agent
+      {%- endif %}
+    state: restarted
+  register: k3s_upgrade_service_restart
+  when: >
+    (
+      k3s_binary_copy.changed | default(false) | bool
+        and
+      node_k3s.installed | default(false) | bool
+    )
+      or
+    (
+      node_k3s.running_version != node_k3s.desired_version
+        and
+      node_k3s.installed | default(false) | bool
+    )
+
+
+- name: Create Fact - cluster_upgraded
+  ansible.builtin.set_fact:
+    nfc_role_kubernetes_cluster_upgraded: true
+    cacheable: true
+  when: >
+    k3s_upgrade_service_restart.changed | default(false) | bool
 
 
 - name: Download Install Scripts
@@ -163,12 +416,20 @@
       - 304
     dest: "{{ item.dest }}"
     mode: "744"
+  check_mode: false
   changed_when: false
+  delay: 10
+  retries: 3
   register: k3s_download_script
   delegate_to: localhost
   run_once: true
   # no_log: true
-  when: ansible_os_family == 'Debian'
+  when: >
+    ansible_os_family == 'Debian'
+      and
+    item.when | default(true) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
   loop: "{{ download_files }}"
   vars:
     ansible_connection: local
@@ -177,66 +438,13 @@
         url: https://get.k3s.io
       - dest: /tmp/install_olm.sh
         url: https://raw.githubusercontent.com/operator-framework/operator-lifecycle-manager/v{{ kubernetes_version_olm }}/scripts/install.sh
+        when: "{{ nfc_role_kubernetes_install_olm }}"
 
 
-- name: Download K3s Binary
-  ansible.builtin.uri:
-    url: |-
-      https://github.com/k3s-io/k3s/releases/download/v
-        {{- KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}}
-      /k3s
-      {%- if cpu_arch.key == 'aarch64' -%}
-      -arm64
-      {%- endif %}
-    method: GET
-    return_content: false
-    status_code:
-      - 200
-      - 304
-    dest: "/tmp/k3s.{{ cpu_arch.key }}"
-    mode: "744"
-  changed_when: false
-  register: k3s_download_files
-  delegate_to: localhost
-  run_once: true
-  # no_log: true
-  when: ansible_os_family == 'Debian'
-  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
-  loop_control:
-    loop_var: cpu_arch
-  vars:
-    ansible_connection: local
-
-
-- name: "[TRACE] Downloaded File SHA256"
-  ansible.builtin.set_fact:
-    hash_sha256_k3s_downloaded_binary: "{{ lookup('ansible.builtin.file', '/tmp/k3s.' + cpu_arch.key) | hash('sha256') | string }}"
-  delegate_to: localhost
-  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
-  loop_control:
-    loop_var: cpu_arch
-
-
-- name: Existing k3s File hash
-  ansible.builtin.stat:
-    checksum_algorithm: sha256
-    name: /usr/local/bin/k3s
-  register: hash_sha256_k3s_existing_binary
-
-
-- name: Copy K3s binary to Host
-  ansible.builtin.copy:
-    src: "/tmp/k3s.{{ ansible_architecture }}"
-    dest: "/usr/local/bin/k3s"
-    mode: '740'
-    owner: root
-    group: root
-  when: hash_sha256_k3s_existing_binary.stat.checksum | default('0') != hash_sha256_k3s_downloaded_binary
-
 - name: Copy install scripts to Host
   ansible.builtin.copy:
-    src: "{{ item }}"
-    dest: "{{ item }}"
+    src: "{{ item.path }}"
+    dest: "{{ item.path }}"
     mode: '755'
     owner: root
     group: root
@@ -244,9 +452,14 @@
   loop: "{{ install_scripts }}"
   vars:
     install_scripts:
-      - "/tmp/install.sh"
-      - "/tmp/install_olm.sh"
-  # when: hash_sha256_k3s_existing_binary.stat.checksum | default('0') != hash_sha256_k3s_downloaded_binary
+      - path: "/tmp/install.sh"
+      - path: "/tmp/install_olm.sh"
+        when: "{{ nfc_role_kubernetes_install_olm }}"
+  when: >
+    item.when | default(true) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
+
 
 - name: Required Initial config files
   ansible.builtin.copy:
@@ -259,7 +472,8 @@
   loop: "{{ k3s.files }}"
   when: >
     item.when | default(true) | bool
-  #   kubernetes_config.cluster.prime.name == inventory_hostname
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Copy Intial required templates
@@ -274,6 +488,8 @@
   diff: true
   when: >
     item.when | default(true) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
   vars:
     templates_to_apply:
       - src: k3s-config.yaml.j2
@@ -283,7 +499,7 @@
         dest: /var/lib/rancher/k3s/server/manifests/calico.yaml
         when: >
           {{
-            kubernetes_config.cluster.prime.name == inventory_hostname
+            kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
               and
             (
               (
@@ -291,7 +507,7 @@
                   and
                 file_calico_yaml_metadata.stat.exists
                   and
-                k3s_installed.rc == 0
+                not node_k3s.installed | bool
               )
                 or
               'calico_manifest' in ansible_run_tags
@@ -318,19 +534,23 @@
   ansible.builtin.command:
     cmd: update-alternatives --set iptables /usr/sbin/iptables-legacy
   changed_when: false
+  when: >
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install K3s (prime master)
   ansible.builtin.shell:
     cmd: |
       INSTALL_K3S_SKIP_DOWNLOAD=true \
-      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
-      /tmp/install.sh --cluster-init
+      INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \
+      /tmp/install.sh {% if nfc_role_kubernetes_etcd_enabled %}--cluster-init{% endif %}
   changed_when: false
   when: >
-    kubernetes_config.cluster.prime.name == inventory_hostname
+    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
       and
-    k3s_installed.rc == 1
+    not node_k3s.installed | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install Calico Operator
@@ -352,7 +572,9 @@
       and
     'calico_manifest' not in ansible_run_tags
       and
-    kubernetes_config.cluster.prime.name == inventory_hostname
+    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install MetalLB Operator
@@ -371,7 +593,9 @@
   when: >-
     nfc_kubernetes_enable_metallb | default(false) | bool
       and
-    kubernetes_config.cluster.prime.name == inventory_hostname
+    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Wait for kubernetes prime to be ready
@@ -385,7 +609,7 @@
         exit 127;
       fi
     executable: /bin/bash
-  delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
+  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
   run_once: true
   register: kubernetes_ready_check
   retries: 30
@@ -396,6 +620,22 @@
     kubernetes_ready_check.rc != 0
   changed_when: false
   failed_when: kubernetes_ready_check.rc != 0
+  when: >
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
+      and
+    not ansible_check_mode
+
+
+- name: Config Link
+  ansible.builtin.shell:
+    cmd: >
+      ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config
+    executable: bash
+    creates: ~/.kube/config
+  when: >
+    nfc_role_kubernetes_master | default(false) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install olm
@@ -409,9 +649,11 @@
     install_olm.rc == 1
   register: install_olm
   when: >
-    kubernetes_config.cluster.prime.name == inventory_hostname
+    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
       and
-    kubernetes_olm_install | default(false) | bool
+    nfc_role_kubernetes_install_olm | default(false) | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Uninstall OLM
@@ -422,7 +664,7 @@
       kubectl delete -n olm deployment olm-operator;
 
       kubectl delete crd catalogsources.operators.coreos.com;
-      kubectl delete` crd clusterserviceversions.operators.coreos.com;
+      kubectl delete crd clusterserviceversions.operators.coreos.com;
       kubectl delete crd installplans.operators.coreos.com;
       kubectl delete crd olmconfigs.operators.coreos.com;
       kubectl delete crd operatorconditions.operators.coreos.com;
@@ -437,9 +679,11 @@
   failed_when: false
   register: install_olm
   when: >
-    kubernetes_config.cluster.prime.name == inventory_hostname
+    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
       and
-    'olm_uninstall' not in ansible_run_tags
+    'olm_uninstall' in ansible_run_tags
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Enable Cluster Encryption
@@ -448,10 +692,12 @@
   changed_when: false
   failed_when: false    # New cluster will fail
   when: >
-    kubernetes_config.cluster.prime.name == inventory_hostname
+    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
       and
     kubernetes_config.cluster.networking.encrypt | default(false) | bool
       and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
+      and
     (
       'calico_manifest' in ansible_run_tags
         or
@@ -466,18 +712,22 @@
 - name: Fetch Join Token
   ansible.builtin.slurp:
     src: /var/lib/rancher/k3s/server/token
-  delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
+  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
   run_once: true
   register: k3s_join_token
   no_log: true # Value is sensitive
+  when: >
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Create Token fact
   ansible.builtin.set_fact:
     k3s_join_token: "{{ k3s_join_token.content | b64decode | replace('\n', '') }}"
-  delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
+  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
   run_once: true
   no_log: true # Value is sensitive
+  when: >
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install K3s (master nodes)
@@ -485,17 +735,19 @@
     cmd: |
       INSTALL_K3S_EXEC="server" \
       INSTALL_K3S_SKIP_DOWNLOAD=true \
-      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
+      INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \
       K3S_TOKEN="{{ k3s_join_token }}" \
       /tmp/install.sh
     executable: /bin/bash
   changed_when: false
   when: >
-    Kubernetes_Master | default(false) | bool
+    nfc_role_kubernetes_master | default(false) | bool
       and
-    not kubernetes_config.cluster.prime.name == inventory_hostname
+    not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
       and
-    k3s_installed.rc == 1
+    not node_k3s.installed | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Install K3s (worker nodes)
@@ -504,16 +756,20 @@
       set -o pipefail
       INSTALL_K3S_EXEC="agent" \
       INSTALL_K3S_SKIP_DOWNLOAD=true \
-      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
+      INSTALL_K3S_VERSION="v{{ node_k3s.desired_version }}" \
       K3S_TOKEN="{{ k3s_join_token }}" \
-      K3S_URL="https://{{ hostvars[kubernetes_config.cluster.prime.name].ansible_host }}:6443" \
+      K3S_URL="https://{{ hostvars[kubernetes_config.cluster.prime.name | default(inventory_hostname)].ansible_host }}:6443" \
       /tmp/install.sh -
     executable: /bin/bash
   changed_when: false
   when: >
-    not Kubernetes_Master | default(false) | bool
+    not nfc_role_kubernetes_master | default(false) | bool
       and
-    k3s_installed.rc == 1
+    not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+      and
+    not node_k3s.installed | bool
+      and
+    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
 
 
 - name: Set Kubernetes Final Install Fact
@@ -521,4 +777,3 @@
     kubernetes_installed: true
     # Clear Token as no llonger required and due to being a sensitive value
     k3s_join_token: null
-    nfc_kubernetes_install_architectures: {}

roles/nfc_kubernetes/tasks/k3s/migrate_to_operator.yaml

@@ -96,7 +96,7 @@
     - name: Fetch Calico Kubectl Plugin
       ansible.builtin.uri:
         url: |-
-          https://github.com/projectcalico/calico/releases/download/{{ nfc_kubernetes_calico_version }}/calicoctl-linux-
+          https://github.com/projectcalico/calico/releases/download/{{ nfc_role_kubernetes_calico_version }}/calicoctl-linux-
           {%- if cpu_arch.key == 'aarch64' -%}
             arm64
           {%- else -%}
@@ -110,6 +110,7 @@
         owner: root
         group: 'root'
       changed_when: false
+      check_mode: false
       become: true
       delegate_to: localhost
       loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
@@ -127,7 +128,7 @@
         owner: root
         group: 'root'
       become: true
-      when: inventory_hostname in groups['kubernetes_master']
+      when: nfc_role_kubernetes_master
 
 
     - name: Setup Automagic Host Endpoints
@@ -150,10 +151,10 @@
             projectcalico.org/operator-node-migration-
         executable: bash
       become: true
-      delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
+      delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
       changed_when: false
       failed_when: false
-      loop: "{{ groups[kubernetes_config.cluster.group_name] }}"
+      loop: "{{ groups[kubernetes_config.cluster.group_name | default('make_me_optional')] | default([]) }}"
 
     # kubectl label node ip-10-229-92-202.eu-west-1.compute.internal projectcalico.org/operator-node-migration-
     # migration started

roles/nfc_kubernetes/tasks/kubevirt/main.yaml

@@ -0,0 +1,72 @@
+---
+
+- name: Validate Virtualization Support
+  ansible.builtin.include_tasks:
+    file: kubevirt/validate.yaml
+    apply:
+      tags:
+        - always
+  tags:
+    - always
+
+
+- name: Deploy KubeVirt
+  ansible.builtin.template:
+    src: "{{ item }}"
+    dest: "/var/lib/rancher/k3s/server/manifests/{{ item | replace('.j2', '') | lower }}"
+    owner: root
+    mode: '700'
+    force: true
+  notify: "{{ item.notify | default(omit) }}"
+  loop: "{{ templates_to_apply }}"
+  diff: true
+  vars:
+    templates_to_apply:
+      - kubevirt-operator.yaml.j2
+      - kubevirt-cr.yaml.j2
+
+
+- name: Fetch virtctl Kubectl Plugin
+  ansible.builtin.uri:
+    url: |-
+      https://github.com/kubevirt/kubevirt/releases/download/{{
+        nfc_role_kubernetes_container_images.kubevirt_operator.tag }}/virtctl-{{
+          nfc_role_kubernetes_container_images.kubevirt_operator.tag }}-linux-
+      {%- if cpu_arch.key == 'aarch64' -%}
+        arm64
+      {%- else -%}
+        amd64
+      {%- endif %}
+    status_code:
+      - 200
+      - 304
+    dest: "/tmp/kubectl-virtctl.{{ cpu_arch.key }}"
+    mode: '777'
+    owner: root
+    group: 'root'
+  changed_when: false
+  become: true
+  delegate_to: localhost
+  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
+  loop_control:
+    loop_var: cpu_arch
+  vars:
+    ansible_connection: local
+
+
+- name: Add virtctl Plugin
+  ansible.builtin.copy:
+    src: "/tmp/kubectl-virtctl.{{ ansible_architecture }}"
+    dest: /usr/local/bin/kubectl-virt
+    mode: '770'
+    owner: root
+    group: 'root'
+  become: true
+  when: nfc_role_kubernetes_master
+
+
+- name: Wait for KubeVirt to initialize
+  ansible.builtin.command:
+    cmd: kubectl -n kubevirt wait kv kubevirt --for condition=Available
+  changed_when: false
+  failed_when: false

roles/nfc_kubernetes/tasks/kubevirt/validate.yaml

@@ -0,0 +1,25 @@
+---
+
+- name: Install LibVirt-Clients
+  ansible.builtin.apt:
+    name: libvirt-clients
+    state: present
+
+
+- name: Confirm Virtualization Support
+  ansible.builtin.command:
+    cmd: virt-host-validate qemu
+  changed_when: false
+  failed_when: false
+  register: virt_support_check_command
+
+
+- name: Confirm No QEMU failures
+  ansible.builtin.assert:
+    that:
+      - (": FAIL" | string) not in (item | string)
+      - |
+        (": PASS" | string) in (item | string)
+          or
+        (": WARN" | string) in (item | string)
+  loop: "{{ virt_support_check_command.stdout_lines }}"

roles/nfc_kubernetes/tasks/main.yaml

@@ -0,0 +1,41 @@
+---
+
+- name: Install/Configure Kubernetes Prime Master Node
+  ansible.builtin.include_tasks:
+    file: install.yaml
+  tags:
+    - always
+  when:
+    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+      and
+    nfc_role_kubernetes_prime | bool
+      and
+    not kubernetes_installed | default(false)
+
+
+- name: Install/Configure Kubernetes on remaining Master Nodes
+  ansible.builtin.include_tasks:
+    file: install.yaml
+  tags:
+    - always
+  when:
+    kubernetes_config.cluster.prime.name | default(inventory_hostname) != inventory_hostname
+      and
+    nfc_role_kubernetes_master | bool
+      and
+    not kubernetes_installed | default(false)
+
+
+- name: Install/Configure Kubernetes on Worker Nodes
+  ansible.builtin.include_tasks:
+    file: install.yaml
+  tags:
+    - always
+  when: >
+    nfc_role_kubernetes_worker | bool
+      and
+    not nfc_role_kubernetes_prime | bool
+      and
+    not nfc_role_kubernetes_master | bool
+      and
+    not kubernetes_installed | default(false)

roles/nfc_kubernetes/templates/ConfigMap-manifest-Calico_Service_Endpoint.yaml.j2

@@ -6,6 +6,6 @@ metadata:
   namespace: tigera-operator
 data:
   KUBERNETES_SERVICE_HOST: "
-    {%- set octet = kubernetes_config.cluster.networking.ServiceSubnet | split('.') -%}
+    {%- set octet = kubernetes_config.cluster.networking.ServiceSubnet | default(nfc_role_kubernetes_service_subnet) | split('.') -%}
     {{- octet[0] }}.{{- octet[1] }}.{{- octet[2] }}.1"
   KUBERNETES_SERVICE_PORT: '443'

roles/nfc_kubernetes/templates/Deployment-manifest-Calico_Operator.yaml.j2

@@ -25272,7 +25272,10 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: tigera-operator
-          image: {{ nfc_kubernetes_tigera_operator_registry }}/{{ nfc_kubernetes_tigera_operator_image}}:{{ nfc_kubernetes_tigera_operator_tag }}
+          image: {{ 
+            nfc_role_kubernetes_container_images.tigera_operator.registry }}/{{ 
+              nfc_role_kubernetes_container_images.tigera_operator.image}}:{{ 
+                nfc_role_kubernetes_container_images.tigera_operator.tag }}
           imagePullPolicy: IfNotPresent
           command:
             - operator
@@ -25290,7 +25293,7 @@ spec:
             - name: OPERATOR_NAME
               value: "tigera-operator"
             - name: TIGERA_OPERATOR_INIT_IMAGE_VERSION
-              value: {{ nfc_kubernetes_tigera_operator_tag }}
+              value: {{ nfc_role_kubernetes_container_images.tigera_operator.tag }}
           envFrom:
             - configMapRef:
                 name: kubernetes-services-endpoint

roles/nfc_kubernetes/templates/IPPool-manifest-Calico_Cluster.yaml.j2

@@ -9,7 +9,7 @@ spec:
     - Workload
     - Tunnel
   blockSize: 26
-  cidr: {{ kubernetes_config.cluster.networking.podSubnet }}
+  cidr: {{ kubernetes_config.cluster.networking.podSubnet | default(nfc_role_kubernetes_pod_subnet) }}
   ipipMode: Never
   natOutgoing: true
   nodeSelector: all()

roles/nfc_kubernetes/templates/Installation-manifest-Calico_Cluster.yaml.j2

@@ -11,7 +11,7 @@ spec:
     hostPorts: Enabled
     ipPools:
     - blockSize: 26
-      cidr: {{ kubernetes_config.cluster.networking.podSubnet }}
+      cidr: {{ kubernetes_config.cluster.networking.podSubnet | default(nfc_role_kubernetes_pod_subnet) }}
       disableBGPExport: false
       encapsulation: VXLAN
       natOutgoing: Enabled
@@ -41,7 +41,7 @@ spec:
     type: RollingUpdate
   nonPrivileged: Disabled
   serviceCIDRs:
-    - {{ kubernetes_config.cluster.networking.ServiceSubnet }}
+    - {{ kubernetes_config.cluster.networking.ServiceSubnet | default(nfc_role_kubernetes_service_subnet) }}
   typhaDeployment:
     spec:
       template:

roles/nfc_kubernetes/templates/calico.yaml.j2

@@ -4810,7 +4810,7 @@ spec:
             # chosen from this range. Changing this value after installation will have
             # no effect. This should fall within `--cluster-cidr`.
             - name: CALICO_IPV4POOL_CIDR
-              value: "{{ KubernetesPodSubnet }}"
+              value: "{{ nfc_role_kubernetes_pod_subnet }}"
             # Disable file logging so `kubectl logs` works.
             - name: CALICO_DISABLE_FILE_LOGGING
               value: "true"

roles/nfc_kubernetes/templates/iptables-kubernetes.rules.j2

@@ -1,7 +1,7 @@
 #
 # IP Tables Firewall Rules for Kubernetes
 # 
-# Managed By ansible/role/nfc_kubernetes
+# Managed By ansible/collection/kubernetes
 #
 # Dont edit this file directly as it will be overwritten. To grant a host API access
 # edit the cluster config, adding the hostname/ip to path kubernetes_config.cluster.access
@@ -31,7 +31,7 @@
 
 {%- endif -%}
 
-{%- for kubernetes_host in groups[kubernetes_config.cluster.group_name] -%}
+{%- for kubernetes_host in groups[kubernetes_config.cluster.group_name | default('me_is_optional')] | default([]) -%}
 
     {%- set kubernetes_host = hostvars[kubernetes_host].ansible_host -%}
 
@@ -61,9 +61,9 @@
 
   {%- if kubernetes_host != '' -%}
 
-    {%- for master_host in groups['kubernetes_master'] -%}
+    {%- for master_host in groups['kubernetes_master'] | default([]) -%}
 
-      {%- if master_host in groups[kubernetes_config.cluster.group_name] -%}
+      {%- if master_host in groups[kubernetes_config.cluster.group_name | default('me_is_optional')] | default([]) -%}
 
         {%- set master_host = hostvars[master_host].ansible_host -%}
 
@@ -88,7 +88,7 @@
         {%- endif -%}
 
 
-        {%- if Kubernetes_Master | default(false) | bool -%}
+        {%- if nfc_role_kubernetes_master | default(false) | bool -%}
 
           {%- if 
               master_host == kubernetes_host
@@ -149,8 +149,13 @@
 
       {#- All cluster Hosts -#}
 
-
-      {%- if Kubernetes_Master | default(false) | bool -%}
+      {%- if 
+        nfc_role_kubernetes_master | default(false) | bool
+          and
+        kubernetes_host not in groups['kubernetes_master']
+          and
+        '-I kubernetes-api -s ' + kubernetes_host + ' -j ACCEPT' not in data.firewall_rules
+      -%}
 
         {%- set data.firewall_rules = data.firewall_rules + ['-I kubernetes-api -s ' + kubernetes_host + ' -j ACCEPT'] -%}
 
@@ -162,9 +167,17 @@
 
       {%- set data.firewall_rules = data.firewall_rules + ['-I kubernetes-flannel-wg-four -s ' + kubernetes_host + ' -j ACCEPT'] -%}
 
-      {%- set data.firewall_rules = data.firewall_rules + ['-I kubernetes-flannel-wg-six -s ' + kubernetes_host + ' -j ACCEPT'] -%}
+      {%- if false -%}{# see IPv6 is disabled #}
+        
+        {%- set data.firewall_rules = data.firewall_rules + ['-I kubernetes-flannel-wg-six -s ' + kubernetes_host + ' -j ACCEPT'] -%}
+      
+      {%- endif -%}
 
-      {%- set data.firewall_rules = data.firewall_rules + ['-I kubernetes-calico-bgp -s ' + kubernetes_host + ' -j ACCEPT'] -%}
+      {%- if false -%}{# see Installation-manifest-Calico_Cluster.yaml.j2 bgp is disabled #}
+      
+        {%- set data.firewall_rules = data.firewall_rules + ['-I kubernetes-calico-bgp -s ' + kubernetes_host + ' -j ACCEPT'] -%}
+
+      {%- endif -%}
 
       {%- set data.firewall_rules = data.firewall_rules + ['-I kubernetes-calico-typha -s ' + kubernetes_host + ' -j ACCEPT'] -%}
 
@@ -182,7 +195,7 @@
 
 {%- endfor -%}
 
-{%- if Kubernetes_Master | default(false) | bool -%}
+{%- if nfc_role_kubernetes_master | default(false) | bool -%}
 
   {%- if host_external_ip is defined -%}
 

roles/nfc_kubernetes/templates/k3s-config.yaml.j2

@@ -6,7 +6,12 @@
 # Dont edit this file directly as it will be overwritten.
 #
 
-{%- if inventory_hostname in groups['kubernetes_master'] -%}
+{%- if 
+  nfc_role_kubernetes_master
+    or
+  kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+-%}
+
   {%
 
   set kube_apiserver_arg = [
@@ -19,31 +24,38 @@
   -%}
     {%
     set servers_config = {
-        "cluster-cidr": KubernetesPodSubnet,
+        "cluster-cidr": nfc_role_kubernetes_pod_subnet,
         "disable": [
           "traefik"
         ],
         "disable-network-policy": true,
-        "etcd-snapshot-retention": kubernetes_etcd_snapshot_retention | int,
-        "etcd-snapshot-schedule-cron":  kubernetes_etcd_snapshot_cron_schedule | string,
         "flannel-backend": "none",
-        "service-cidr": KubernetesServiceSubnet
+        "service-cidr": nfc_role_kubernetes_service_subnet
       }
     -%}
 
-    {%- if 
-          kubernetes_config.cluster.domain_name is defined
-            and
-          kubernetes_config.cluster.domain_name | default('') != ''
-    -%}
+    {%- if nfc_role_kubernetes_etcd_enabled -%}
 
         {%- set servers_config = servers_config | combine({
-          "cluster-domain": kubernetes_config.cluster.domain_name
+          "etcd-snapshot-retention": kubernetes_etcd_snapshot_retention | int,
+          "etcd-snapshot-schedule-cron":  kubernetes_etcd_snapshot_cron_schedule | string,
         }) -%}
 
     {%- endif -%}
 
-    {%- if kubernetes_config.cluster.oidc.enabled | default(false) | bool -%}
+    {%- if 
+          kubernetes_config.cluster.domain_name | default(nfc_role_kubernetes_cluster_domain) is defined
+            and
+          kubernetes_config.cluster.domain_name | default(nfc_role_kubernetes_cluster_domain) != ''
+    -%}
+
+        {%- set servers_config = servers_config | combine({
+          "cluster-domain": kubernetes_config.cluster.domain_name | default(nfc_role_kubernetes_cluster_domain)
+        }) -%}
+
+    {%- endif -%}
+
+    {%- if kubernetes_config.cluster.oidc.enabled | default(nfc_role_kubernetes_oidc_enabled) | default(false) | bool -%}
 
       {%- 
         set kube_apiserver_arg = kube_apiserver_arg + [
@@ -116,6 +128,16 @@
 
 {# SoF All Nodes #}
 
+{%- if inventory_hostname == 'localhost' -%}
+
+  {%- set node_name = hostname_to_check.stdout -%}
+
+{%- else -%}
+
+  {%- set node_name = inventory_hostname -%}
+
+{%- endif -%}
+
 {%
 
   set all_nodes_config = {
@@ -123,21 +145,22 @@
       "system-reserved=cpu=" + kubelet_arg_system_reserved_cpu  + ",memory=" + kubelet_arg_system_reserved_memory + 
       ",ephemeral-storage=" + kubelet_arg_system_reserved_storage
     ],
-    "node-name": inventory_hostname,
+    "node-name": node_name,
+    "resolv-conf": nfc_role_kubernetes_resolv_conf_file,
   }
 
 -%}
 
 
-{%- if groups[kubernetes_config.cluster.group_name] | default([]) | list | length > 0 -%}
+{%- if groups[kubernetes_config.cluster.group_name | default('make_me_optional')] | default([]) | list | length > 0 -%}
   
-  {%- if k3s_installed.rc == 0 -%}
+  {%- if node_k3s.installed -%}
 
     {%- set ns = namespace(server=[]) -%}
 
     {%- for cluster_node in groups[kubernetes_config.cluster.group_name] -%}
 
-      {%- if cluster_node in groups['kubernetes_master'] -%}
+      {%- if cluster_node in groups['kubernetes_master'] | default([]) -%}
 
         {%- if hostvars[cluster_node].host_external_ip is defined -%}
 
@@ -176,7 +199,7 @@
   {%- elif 
     kubernetes_config.cluster.prime.name != inventory_hostname
       and
-    k3s_installed.rc == 1
+    not node_k3s.installed
   -%}
 
     {%- set server = (server | default([])) + [
@@ -215,7 +238,11 @@
 {# EoF All Nodes #}
 
 
-{%- if inventory_hostname in groups['kubernetes_master'] -%}
+{%- if 
+  nfc_role_kubernetes_master
+    or
+  kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
+-%}
 
   {%- set servers_config = servers_config | combine( all_nodes_config ) -%}
 

roles/nfc_kubernetes/templates/kubevirt-cr.yaml.j2

@@ -0,0 +1,16 @@
+---
+apiVersion: kubevirt.io/v1
+kind: KubeVirt
+metadata:
+  name: kubevirt
+  namespace: kubevirt
+spec:
+  certificateRotateStrategy: {}
+  configuration:
+    developerConfiguration:
+      featureGates: []
+  customizeComponents: {}
+  imagePullPolicy: IfNotPresent
+  workloadUpdateStrategy:
+    workloadUpdateMethods:
+      - LiveMigrate

roles/nfc_kubernetes/vars/firewall_rules.yaml

@@ -0,0 +1,90 @@
+---
+
+kubernetes_chains:
+
+  - name: kubernetes-embedded-etcd
+    chain: true
+    table: INPUT
+    protocol: tcp
+    dest:
+      port:
+        - '2379'
+        - '2380'
+    comment: etcd. Servers only
+    when: "{{ nfc_role_kubernetes_etcd_enabled }}"
+
+  - name: kubernetes-api
+    chain: true
+    table: INPUT
+    protocol: tcp
+    dest:
+      port: '6443'
+    comment: Kubernetes API access. All Cluster hosts and end users
+
+  - name: kubernetes-calico-bgp
+    chain: true
+    table: INPUT
+    protocol: tcp
+    dest:
+      port: '179'
+    comment: Kubernetes Calico BGP. All Cluster hosts and end users
+    when: false    # currently hard set to false. see Installation-manifest-Calico_Cluster.yaml.j2
+
+  - name: kubernetes-flannel-vxlan
+    chain: true
+    table: INPUT
+    protocol: udp
+    dest:
+      port: '4789'
+    comment: Flannel. All cluster hosts
+
+  - name: kubernetes-kubelet-metrics
+    chain: true
+    table: INPUT
+    protocol: tcp
+    dest:
+      port: '10250'
+    comment: Kubernetes Metrics. All cluster hosts
+
+  - name: kubernetes-flannel-wg-four
+    chain: true
+    table: INPUT
+    protocol: udp
+    dest:
+      port: '51820'
+    comment: Flannel Wiregaurd IPv4. All cluster hosts
+
+  - name: kubernetes-flannel-wg-six
+    chain: true
+    table: INPUT
+    protocol: udp
+    dest:
+      port: '51821'
+    comment: Flannel Wiregaurd IPv6. All cluster hosts
+    when: false    # ipv6 is disabled. see install.yaml sysctrl
+
+  - name: kubernetes-calico-typha
+    chain: true
+    table: INPUT
+    protocol: tcp
+    dest:
+      port: '5473'
+    comment: Calico networking with Typha enabled. Typha agent hosts.
+
+  - name: metallb-l2-tcp
+    chain: true
+    table: INPUT
+    protocol: tcp
+    dest:
+      port: '7946'
+    comment: MetalLB Gossip
+    when: "{{ nfc_kubernetes_enable_metallb }}"
+
+  - name: metallb-l2-udp
+    chain: true
+    table: INPUT
+    protocol: udp
+    dest:
+      port: '7946'
+    comment: MetalLB Gossip
+    when: "{{ nfc_kubernetes_enable_metallb }}"

tasks/common.yaml

@@ -1,303 +0,0 @@
----
-
-- name: "{{ role_name }} Install Software"
-  include_role: 
-    name: nfc_common
-  vars:
-    common_gather_facts: false
-    aptSigningKeys:
-      - name: docker
-        url: https://download.docker.com/linux/debian/gpg
-        save_directory: /usr/share/keyrings
-        file_extension: asc
-
-      - name: kubernetes
-        url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
-        save_directory: /usr/share/keyrings
-        file_extension: asc
-
-    aptRepositories:
-      - name: docker
-        repo: deb [arch={{ dynamic_processor_architecture }} signed-by=/usr/share/keyrings/docker.asc] http://download.docker.com/linux/{{ ansible_os_family | lower }}   {{ ansible_lsb.codename | lower }} stable
-      - name: kubernetes
-        repo: deb [signed-by=/usr/share/keyrings/kubernetes.asc] http://apt.kubernetes.io/ kubernetes-xenial main
-
-    aptInstall:
-      - name: gnupg2 
-      - name: apt-transport-https
-      - name: software-properties-common
-      - name: ca-certificates
-      - name: iptables
-      - name: python3-pip
-      - name: python3-virtualenv
-
-      - name: containerd.io
-        version: "{{ ContainerDioVersion }}"
-
-      - name: kubectl
-        version: "{{ KubernetesVersion }}"
-      - name: kubelet
-        version: "{{ KubernetesVersion }}"
-      - name: kubeadm
-        version: "{{ KubernetesVersion }}"
-  tags:
-    - install
-
-# containerd.io=1.6.22-1 kubectl=1.26.9-00 kubelet=1.26.9-00 kubeadm=1.26.9-00
-
-- name: Remove swapfile from /etc/fstab
-  mount:
-    name: "{{ item }}"
-    fstype: swap
-    state: absent
-  with_items:
-    - swap
-    - none
-  when: 
-    - ansible_os_family == 'Debian' # ansible_lsb.codename = bullseye, ansible_lsb.major_release = 11
-  tags:
-    - install
-
-
-- name: Disable swap
-  command: swapoff -a
-  changed_when: true == false
-  when: 
-    #- ansible_swaptotal_mb > 0
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
-
-- name: Check an armbian os system
-  stat:
-    path: /etc/default/armbian-zram-config
-  register: armbian_stat_result
-
-
-- name: Armbian Disable Swap
-  ansible.builtin.shell:
-    cmd: |
-      sed -i 's/\# SWAP=false/SWAP=false/g' /etc/default/armbian-zram-config;
-      sed -i 's/ENABLED=true/ENABLED=false/g' /etc/default/armbian-zram-config;
-  args:
-    executable: bash
-  changed_when: false
-  # failed_when: false
-  #notify: RebootHost # doesnt need to reboot as swapoff -a covers the deployment
-  when: armbian_stat_result.stat.exists
-
-
-- name: Add the overlay module
-  community.general.modprobe:
-    name: overlay
-    state: present
-  when:
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
-
-
-- name: Add the br_netfilter module
-  community.general.modprobe:
-    name: br_netfilter
-    state: present
-  when:
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
-
-- name: check if containerd installed
-  ansible.builtin.shell:
-    cmd: which containerd
-  failed_when: false
-  changed_when: false
-  register: containerd_installed
-
-
-- name: "Containerd.io Started?"
-  service:
-    name: containerd
-    state: started
-  tags:
-    - configure
-    - install
-  when: >
-    ansible_os_family == 'Debian'
-      and
-    containerd_installed.rc | default(1) | int == 0
-
-
-- name: containerd load modules config
-  template:
-    src: "etc_module_containerd.conf"
-    dest: /etc/modules-load.d/containerd.conf
-    owner: root
-    mode: 0700
-  notify: "restart ContainerD"
-  when: 
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
-
-
-- name: Create containerD host directories.
-  become_method: sudo
-  become: yes
-  file: 
-    path: /etc/containerd/certs.d/{{ item.name }}
-    state: directory
-    owner: "{{ ansible_user }}"
-    group: "{{ ansible_user }}"
-    mode: 0700
-  with_items: "{{ containerd.repositories }}"
-  tags:
-    - install
-    - containerRegistry
-
-
-- name: containerD registry host
-  template:
-    src: "containerd-registry-hosts.toml.j2"
-    dest: /etc/containerd/certs.d/{{ item.name }}/hosts.toml
-    owner: root
-    mode: 0700
-  notify: "restart ContainerD"
-  with_items: "{{ containerd.repositories }}"
-  when: 
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
-    - containerRegistry
-
-
-- name: containerD default config
-  template:
-    src: "etc_containerd_containerd.toml"
-    dest: /etc/containerd/config.toml
-    owner: root
-    mode: 0700
-  notify: "restart ContainerD"
-  register: containerd_config
-  when: 
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
-    - containerRegistry
-
-
-- name: Install required python modules
-  ansible.builtin.pip:
-    name: kubernetes
-    state: forcereinstall
-    #virtualenv: /tmp/venv_ansible
-  when: inventory_hostname != 'op1'
-  tags:
-    - install
-
-
-- name: sysctl net.bridge.bridge-nf-call-ip6tables
-  sysctl:
-    name: net.bridge.bridge-nf-call-ip6tables
-    value: '1'
-    sysctl_set: yes
-    state: present
-    reload: yes
-  when:
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
-
-
-- name: sysctl net.bridge.bridge-nf-call-iptables
-  sysctl:
-    name: net.bridge.bridge-nf-call-iptables
-    value: '1'
-    sysctl_set: yes
-    state: present
-    reload: yes
-  when:
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
-
-
-- name: sysctl net.ipv4.ip_forward
-  sysctl:
-    name: net.ipv4.ip_forward
-    value: '1'
-    sysctl_set: yes
-    state: present
-    reload: yes
-  when:
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
-
-
-# - name: Check if kubernetes has been Initialized
-#   stat:
-#     path: /etc/kubernetes/admin.conf
-#   register: KubernetesInit
-#   when: 
-#     - kubernetes_config.cluster.prime.name == inventory_hostname
-
-
-- name: check if iptables is installed
-  ansible.builtin.shell: |-
-    dpkg -s iptables &> /dev/null
-  changed_when: true == false
-  register: iptables_installed
-  when: 
-    - ansible_os_family == 'Debian'
-  tags:
-    - install
-    - iptables
-    - firewall
-
-
-- name: Add kubernetes Firewall Rules - '/etc/iptables-kubernetes.rules'
-  template:
-    src: iptables-kubernetes.rules.j2
-    dest: "/etc/iptables-kubernetes.rules"
-    owner: root
-    mode: 0700
-    force: yes
-  notify: "Apply Firewall Rules"
-  when: 
-    - ansible_os_family == 'Debian'
-    - iptables_installed.rc == 0
-  tags:
-    - install
-    - iptables
-    - firewall
-
-
-- name: File - '/etc/network/if-pre-up.d/firewall-kubernetes'
-  template:
-    src: firewall-kubernetes.j2
-    dest: "/etc/network/if-pre-up.d/firewall-kubernetes"
-    owner: root
-    mode: 0700
-    force: yes
-  when: 
-    - ansible_os_family == 'Debian'
-    - iptables_installed.rc == 0
-  tags:
-    - install
-    - iptables
-    - firewall
-
-
-- name:  Create local workdir
-  file:
-    path: "{{ item }}"
-    state: directory
-    mode: 0700
-  delegate_to: localhost
-  connection: local
-  run_once: true
-  changed_when: true == false
-  with_items:
-    - /tmp/ansible/
-  tags:
-    - always

tasks/k3s.yaml

@@ -1,29 +0,0 @@
----
-# kubernetes_installed
-
-- name: K3s Install
-  ansible.builtin.include_tasks:
-    file: k3s/install.yaml
-    apply:
-      tags:
-        - always
-  when: >
-    install_kubernetes | default(true) | bool
-      and
-    not kubernetes_installed | default(false) | bool
-  tags:
-    - always
-
-
-- name: K3s Configure
-  ansible.builtin.include_tasks:
-    file: k3s/configure.yaml
-    apply:
-      tags:
-        - always
-  when: >
-    install_kubernetes | default(true) | bool
-      and
-    kubernetes_installed | default(false) | bool
-  tags:
-    - always

tasks/k8s.yaml

@@ -1,103 +0,0 @@
----
-- name: Common Tasks
-  include_tasks: common.yaml
-  # tags:
-  #   - install
-
-- name: Check if kubernetes has been Initialized
-  stat:
-    path: /etc/kubernetes/admin.conf
-  register: KubernetesInitialized
-  tags:
-    - always
-
-- name: kubernetes prime
-  include_tasks: prime.yaml
-  when: kubernetes_config.cluster.prime.name == inventory_hostname
-
-
-- name: kubernetes workers
-  include_tasks: workers.yaml
-  when: kubernetes_config.cluster.prime.name != inventory_hostname
-
-
-- name: Add Kubernetes Node Labels
-  kubernetes.core.k8s:
-    definition:
-      apiVersion: v1
-      kind: Node
-      metadata:
-        name: "{{ inventory_hostname }}"
-        labels: 
-          "{{ item | from_yaml_all }}"
-  delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
-  with_items:
-    - "{{ kubernetes_config.hosts[inventory_hostname].labels }}"
-  when: 
-    - ( kubernetes_config.hosts[inventory_hostname].labels is defined and 
-      kubernetes_config.hosts[inventory_hostname].labels|default('')|length > 0 )
-  tags:
-    - install
-    - nodelabels
-
-
-- name: Add Node Taints
-  kubernetes.core.k8s_taint:
-    state: "present"
-    name: "{{ inventory_hostname }}"
-    taints:
-      - "{{ item | from_yaml_all }}"
-  delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
-  with_items:
-    - "{{ kubernetes_config.hosts[inventory_hostname].taints.present }}"
-  when: 
-    - (kubernetes_config.hosts[inventory_hostname].taints.present is defined and 
-      kubernetes_config.hosts[inventory_hostname].taints.present|default('')|length > 0 )
-  tags:
-    - install
-    - taints
-
-
-- name: Remove Node Taints
-  kubernetes.core.k8s_taint:
-    state: "absent"
-    name: "{{ inventory_hostname }}"
-    taints:
-      - "{{ item | from_yaml_all }}"
-  delegate_to: "{{ kubernetes_config.cluster.prime.name }}"  
-  with_items:
-    - "{{ kubernetes_config.hosts[inventory_hostname].taints.absent }}"
-  when: 
-    - ( kubernetes_config.hosts[inventory_hostname].taints.absent is defined and 
-      kubernetes_config.hosts[inventory_hostname].taints.absent|default('')|length > 0 )
-  tags:
-    - install
-    - taints
-
-
-
-
-- name: Create Cluster Namespaces
-  kubernetes.core.k8s:
-    state: present
-    definition:
-      apiVersion: v1
-      kind: Namespace
-      metadata:
-        name: "{{ item.name }}"
-        labels:
-          #app.kubernetes.io/version: # App version
-          #app.kubernetes.io/component:
-          #app.kubernetes.io/part-of:
-          app.kubernetes.io/managed-by: Ansible
-          #meta.kubernetes.io/description: "{{ item.description | default('') }}"
-          meta.kubernetes.io/version: "{{ deployment_git_current_short_hash | default('') }}"
-  with_items:
-    - "{{ kubernetes_config.namespaces }}"
-  when: 
-    ( kubernetes_config.namespaces is defined and 
-      kubernetes_config.namespaces | default('') | length > 0 and
-      kubernetes_config.cluster.prime.name == inventory_hostname )
-  tags:
-    - install
-    - namespaces

tasks/main.yml

@@ -1,38 +0,0 @@
----
-
-- name: Firewall Rules
-  ansible.builtin.include_role:
-    name: nfc_firewall
-  vars:
-    nfc_firewall_enabled_kubernetes: "{{ nfc_kubernetes.enable_firewall | default(false) | bool }}"
-  tags:
-    - never
-    - install
-
-
-# fix, reload firewall `iptables-reloader`
-- name: Reload iptables
-  ansible.builtin.command:
-    cmd: bash -c /usr/bin/iptables-reloader
-  changed_when: false
-  tags:
-    - never
-    - install
-
-
-- name: K8s Cluster
-  ansible.builtin.include_tasks: k8s.yaml
-  when: kubernetes_type == 'k8s'
-  tags:
-    - never
-    - install
-
-
-- name: K3s Cluster
-  ansible.builtin.include_tasks: k3s.yaml
-  when: kubernetes_type == 'k3s'
-  tags:
-    - never
-    - install
-    - operator_calico
-    - operator_migrate_calico

tasks/prime.yaml

@@ -1,146 +0,0 @@
----
-
-- name: initialize Kubernetes cluster
-  block:
-    - name: Intilizing Kubernetes Cluster
-      #command: kubeadm init --pod-network-cidr "{{ KubernetesPodSubnet }}" --apiserver-advertise-address "{{ ansible_default_ipv4.address }}" --ignore-preflight-errors Mem --cri-socket=unix:///var/run/crio/crio.sock
-      command: kubeadm init --pod-network-cidr "{{ KubernetesPodSubnet }}" --service-cidr "{{ KubernetesServiceSubnet }}" --apiserver-advertise-address "0.0.0.0" --ignore-preflight-errors Mem #--cri-socket=unix:///var/run/containerd/containerd.sock
-      when: 
-        - not KubernetesInitialized.stat.exists
-
-  rescue:
-    - name: Reset Kubeadmn
-      ansible.builtin.shell: "{{ item }}"
-      #register: kube_reset
-      failed_when: item.rc != 0
-      with_items:
-        - kubeadm reset --force
-        - rm -Rf /etc/cni/net.d
-
-
-- name: Check if kubernetes has been Initialized
-  stat:
-    path: /etc/kubernetes/admin.conf
-  register: KubernetesInitialized
-  tags:
-    - always
-
-
-- name: fetch kubernetes health
-  ansible.builtin.shell: " wget http://localhost:10248/healthz -q -O - || true"
-  register: KubernetesHealth
-  changed_when: true == false
-  when: KubernetesInitialized.stat.exists
-  tags:
-    - always
-
-
-- name: set kubeernetes health fact
-  set_fact:
-    kube_health: "{{ KubernetesHealth.stdout | default(false) == 'ok' }}"
-  changed_when: true == false
-  tags:
-    - always
-
-
-- name: Create directory for kube config.
-  become_method: sudo
-  become: yes
-  file: 
-    #path: /home/{{ ansible_user }}/.kube
-    path: ~/.kube
-    state: directory
-    owner: "{{ ansible_user }}"
-    group: "{{ ansible_user }}"
-    mode: 0700
-  # when: Kubernetes_Master
-  tags:
-    - always
-
-
-- name: Copy Kube config for local user
-  copy:
-    remote_src: yes
-    src: /etc/kubernetes/admin.conf
-    #dest: /home/{{ ansible_user }}/.kube/config
-    dest: ~/.kube/config
-    owner: "{{ ansible_user }}"
-    group: "{{ ansible_user }}"
-    mode: 0700
-  tags:
-    - always
-
-
-- name: Add calico networking.
-  template:
-    src: "calico.yaml.j2"
-    dest: /etc/kubernetes/manifests/calico.yaml
-    owner: root
-    mode: 0744
-
-
-- name: apply calico manifest
-  command: kubectl apply -f /etc/kubernetes/manifests/calico.yaml
-  tags:
-    - install
-    - manifest
-
-
-- name: create remote workdir
-  file:
-    path: "{{ item }}"
-    state: directory
-    mode: 0700
-  with_items:
-    - /tmp/ansible/
-  tags:
-    - always
-
-
-- name:  Create local workdir
-  file:
-    path: "{{ item }}"
-    state: directory
-    mode: 0700
-  delegate_to: localhost
-  connection: local
-  with_items:
-    - /tmp/ansible/
-  tags:
-    - always
-
-
-- name: get join command
-  ansible.builtin.shell: kubeadm token create --print-join-command > /tmp/ansible/join_kubernetes.sh
-  changed_when: true == false
-  tags:
-    - always
-
-
-- name: download join command
-  fetch:
-    src: /tmp/ansible/join_kubernetes.sh
-    dest: /tmp/ansible/
-    flat: yes
-  changed_when: true == false
-  tags:
-    - always
-
-
-  # always:
-
-  #   - name:  remove remote workdir
-  #     file:
-  #       path: "{{ item }}"
-  #       state: absent
-  #     with_items:
-  #       - /tmp/ansible/join_kubernetes.sh
-  #     changed_when: true == false
-
-  # when: 
-  #   #- Kubernetes_Prime
-  #   #- KubernetesInit.stat.exists
-  #   - kubernetes_config.cluster.prime.name == inventory_hostname
-
-
-

tasks/workers.yaml

@@ -1,46 +0,0 @@
----
-# - name: configure non-prime nodes - check node health
-#   shell: "curl http://localhost:10248/healthz || true"
-#   register: health
-#   changed_when: true == false
-
-# - set_fact:
-#     kube_joined: "{{ health.stdout == 'ok' }}"
-#   changed_when: true == false
-#   # when: 
-#   #     - not Kubernetes_Prime
-
-- name: configure non-prime nodes - create remote workdir
-  file:
-    path: "{{ item }}"
-    state: directory
-    mode: 0700
-  changed_when: true == false
-  with_items:
-    - /tmp/ansible/
-
-
-- ansible.builtin.shell: " wget http://localhost:10248/healthz -q -O - || true"
-  register: health
-  changed_when: true == false
-
-
-- set_fact:
-    kube_joined: "{{ health.stdout == 'ok' }}"
-  changed_when: true == false
-
-
-- name: get join command from ansible controller
-  copy:
-    src: /tmp/ansible/join_kubernetes.sh
-    dest: /tmp/ansible/join_kubernetes.sh
-    mode: 0700
-  changed_when: true == false
-  when: 
-    - not kube_joined
-
-
-- name: configure non-prime nodes - join node to kubernetes cluster
-  command: sh /tmp/ansible/join_kubernetes.sh
-  when: 
-    - not kube_joined

templates/containerd-registry-hosts.toml.j2

@@ -1,10 +0,0 @@
-#
-# {{ item.name }} Container Registry Configuration
-# Managed by: Ansible
-#
-
-server = "{{ item.server }}"
-
-[host."{{ item.url }}"]
-  capabilities = {{ item.capabilities | from_yaml_all }}
-  skip_verify = {{ item.skip_verify | default(false) | lower }}

templates/etc_containerd_containerd.toml

@@ -1,250 +0,0 @@
-disabled_plugins = []
-imports = []
-oom_score = 0
-plugin_dir = ""
-required_plugins = []
-root = "/var/lib/containerd"
-state = "/run/containerd"
-temp = ""
-version = 2
-
-[cgroup]
-  path = ""
-
-[debug]
-  address = ""
-  format = ""
-  gid = 0
-  level = ""
-  uid = 0
-
-[grpc]
-  address = "/run/containerd/containerd.sock"
-  gid = 0
-  max_recv_message_size = 16777216
-  max_send_message_size = 16777216
-  tcp_address = ""
-  tcp_tls_ca = ""
-  tcp_tls_cert = ""
-  tcp_tls_key = ""
-  uid = 0
-
-[metrics]
-  address = ""
-  grpc_histogram = false
-
-[plugins]
-
-  [plugins."io.containerd.gc.v1.scheduler"]
-    deletion_threshold = 0
-    mutation_threshold = 100
-    pause_threshold = 0.02
-    schedule_delay = "0s"
-    startup_delay = "100ms"
-
-  [plugins."io.containerd.grpc.v1.cri"]
-    device_ownership_from_security_context = false
-    disable_apparmor = false
-    disable_cgroup = false
-    disable_hugetlb_controller = true
-    disable_proc_mount = false
-    disable_tcp_service = true
-    enable_selinux = false
-    enable_tls_streaming = false
-    enable_unprivileged_icmp = false
-    enable_unprivileged_ports = false
-    ignore_image_defined_volumes = false
-    max_concurrent_downloads = 3
-    max_container_log_line_size = 16384
-    netns_mounts_under_state_dir = false
-    restrict_oom_score_adj = false
-    sandbox_image = "registry.k8s.io/pause:3.6"
-    selinux_category_range = 1024
-    stats_collect_period = 10
-    stream_idle_timeout = "4h0m0s"
-    stream_server_address = "127.0.0.1"
-    stream_server_port = "0"
-    systemd_cgroup = false
-    tolerate_missing_hugetlb_controller = true
-    unset_seccomp_profile = ""
-
-    [plugins."io.containerd.grpc.v1.cri".cni]
-      bin_dir = "/opt/cni/bin"
-      conf_dir = "/etc/cni/net.d"
-      conf_template = ""
-      ip_pref = ""
-      max_conf_num = 1
-
-    [plugins."io.containerd.grpc.v1.cri".containerd]
-      default_runtime_name = "runc"
-      disable_snapshot_annotations = true
-      discard_unpacked_layers = false
-      ignore_rdt_not_enabled_errors = false
-      no_pivot = false
-      snapshotter = "overlayfs"
-
-      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
-        base_runtime_spec = ""
-        cni_conf_dir = ""
-        cni_max_conf_num = 0
-        container_annotations = []
-        pod_annotations = []
-        privileged_without_host_devices = false
-        runtime_engine = ""
-        runtime_path = ""
-        runtime_root = ""
-        runtime_type = ""
-
-        [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]
-
-      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
-
-        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-          base_runtime_spec = ""
-          cni_conf_dir = ""
-          cni_max_conf_num = 0
-          container_annotations = []
-          pod_annotations = []
-          privileged_without_host_devices = false
-          runtime_engine = ""
-          runtime_path = ""
-          runtime_root = ""
-          runtime_type = "io.containerd.runc.v2"
-
-          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
-            BinaryName = ""
-            CriuImagePath = ""
-            CriuPath = ""
-            CriuWorkPath = ""
-            IoGid = 0
-            IoUid = 0
-            NoNewKeyring = false
-            NoPivotRoot = false
-            Root = ""
-            ShimCgroup = ""
-            SystemdCgroup = true
-
-      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
-        base_runtime_spec = ""
-        cni_conf_dir = ""
-        cni_max_conf_num = 0
-        container_annotations = []
-        pod_annotations = []
-        privileged_without_host_devices = false
-        runtime_engine = ""
-        runtime_path = ""
-        runtime_root = ""
-        runtime_type = ""
-
-        [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]
-
-    [plugins."io.containerd.grpc.v1.cri".image_decryption]
-      key_model = "node"
-
-    [plugins."io.containerd.grpc.v1.cri".registry]
-      config_path = "/etc/containerd/certs.d"
-
-      [plugins."io.containerd.grpc.v1.cri".registry.auths]
-
-      [plugins."io.containerd.grpc.v1.cri".registry.configs]
-
-      [plugins."io.containerd.grpc.v1.cri".registry.headers]
-
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-
-    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
-      tls_cert_file = ""
-      tls_key_file = ""
-
-  [plugins."io.containerd.internal.v1.opt"]
-    path = "/opt/containerd"
-
-  [plugins."io.containerd.internal.v1.restart"]
-    interval = "10s"
-
-  [plugins."io.containerd.internal.v1.tracing"]
-    sampling_ratio = 1.0
-    service_name = "containerd"
-
-  [plugins."io.containerd.metadata.v1.bolt"]
-    content_sharing_policy = "shared"
-
-  [plugins."io.containerd.monitor.v1.cgroups"]
-    no_prometheus = false
-
-  [plugins."io.containerd.runtime.v1.linux"]
-    no_shim = false
-    runtime = "runc"
-    runtime_root = ""
-    shim = "containerd-shim"
-    shim_debug = false
-
-  [plugins."io.containerd.runtime.v2.task"]
-    platforms = ["linux/amd64"]
-    sched_core = false
-
-  [plugins."io.containerd.service.v1.diff-service"]
-    default = ["walking"]
-
-  [plugins."io.containerd.service.v1.tasks-service"]
-    rdt_config_file = ""
-
-  [plugins."io.containerd.snapshotter.v1.aufs"]
-    root_path = ""
-
-  [plugins."io.containerd.snapshotter.v1.btrfs"]
-    root_path = ""
-
-  [plugins."io.containerd.snapshotter.v1.devmapper"]
-    async_remove = false
-    base_image_size = ""
-    discard_blocks = false
-    fs_options = ""
-    fs_type = ""
-    pool_name = ""
-    root_path = ""
-
-  [plugins."io.containerd.snapshotter.v1.native"]
-    root_path = ""
-
-  [plugins."io.containerd.snapshotter.v1.overlayfs"]
-    root_path = ""
-    upperdir_label = false
-
-  [plugins."io.containerd.snapshotter.v1.zfs"]
-    root_path = ""
-
-  [plugins."io.containerd.tracing.processor.v1.otlp"]
-    endpoint = ""
-    insecure = false
-    protocol = ""
-
-[proxy_plugins]
-
-[stream_processors]
-
-  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
-    accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
-    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
-    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
-    path = "ctd-decoder"
-    returns = "application/vnd.oci.image.layer.v1.tar"
-
-  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
-    accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
-    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
-    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
-    path = "ctd-decoder"
-    returns = "application/vnd.oci.image.layer.v1.tar+gzip"
-
-[timeouts]
-  "io.containerd.timeout.bolt.open" = "0s"
-  "io.containerd.timeout.shim.cleanup" = "5s"
-  "io.containerd.timeout.shim.load" = "5s"
-  "io.containerd.timeout.shim.shutdown" = "3s"
-  "io.containerd.timeout.task.state" = "2s"
-
-[ttrpc]
-  address = ""
-  gid = 0
-  uid = 0

templates/etc_module_containerd.conf

@@ -1,2 +0,0 @@
-overlay
-br_netfilter