kubernetes/roles/nfc_kubernetes/tasks/k3s/install.yaml

---

- name: Check for calico deployment manifest
  ansible.builtin.stat:
    name: /var/lib/rancher/k3s/server/manifests/calico.yaml
  register: file_calico_yaml_metadata


- name: Check for calico Operator deployment manifest
  ansible.builtin.stat:
    name: /var/lib/rancher/k3s/ansible/deployment-manifest-calico_operator.yaml
  register: file_calico_operator_yaml_metadata


- name: Install dependent packages
  ansible.builtin.apt:
    name: "{{ package }}"
    state: present
  loop: "{{ packages }}"
  loop_control:
    loop_var: package
  vars:
    packages:
      - curl
      - iptables
      - jq
      - wireguard


- name: Remove swapfile from /etc/fstab
  ansible.posix.mount:
    name: "{{ item }}"
    fstype: swap
    state: absent
  with_items:
    - swap
    - none
  when:
    - ansible_os_family == 'Debian' # ansible_lsb.codename = bullseye, ansible_lsb.major_release = 11
  tags:
    - install


- name: Disable swap
  ansible.builtin.command:
    cmd: swapoff -a
  changed_when: false
  when:
    - ansible_os_family == 'Debian'
  tags:
    - install

- name: Check an armbian os system
  ansible.builtin.stat:
    path: /etc/default/armbian-zram-config
  register: armbian_stat_result


- name: Armbian Disable Swap
  ansible.builtin.shell:
    cmd: |
      sed -i 's/\# SWAP=false/SWAP=false/g' /etc/default/armbian-zram-config;
      sed -i 's/ENABLED=true/ENABLED=false/g' /etc/default/armbian-zram-config;
  args:
    executable: bash
  changed_when: false
  when: armbian_stat_result.stat.exists


- name: Create Required directories
  ansible.builtin.file:
    name: "{{ item.name }}"
    state: "{{ item.state }}"
    mode: "{{ item.mode }}"
  loop: "{{ dirs }}"
  vars:
    dirs:
      - name: /etc/rancher/k3s
        state: directory
        mode: 700
      - name: /var/lib/rancher/k3s/server/logs
        state: directory
        mode: 700
      - name: /var/lib/rancher/k3s/server/manifests
        state: directory
        mode: 700
      - name: /var/lib/rancher/k3s/ansible
        state: directory
        mode: 700


- name: Add sysctl net.ipv4.ip_forward
  ansible.posix.sysctl:
    name: "{{ item.name }}"
    value: "{{ item.value }}"
    sysctl_set: true
    state: present
    reload: true
  loop: "{{ settings }}"
  notify: reboot_host    # On change reboot
  vars:
    settings:
      - name: net.ipv4.ip_forward
        value: '1'
      - name: fs.inotify.max_user_watches
        value: '524288'
      - name: fs.inotify.max_user_instances
        value: '512'
      - name: net.ipv6.conf.all.disable_ipv6
        value: '1'
  when:
    - ansible_os_family == 'Debian'


- name: Check for Network Manager Directory
  ansible.builtin.stat:
    name: /etc/NetworkManager/conf.d
  register: directory_network_manager_metadata


- name: Network Manager Setup
  ansible.builtin.copy:
    content: |-
      #
      # K3s Configuration for Network Manager
      #
      # Managed By ansible/role/nfc_kubernetes
      #
      # Dont edit this file directly as it will be overwritten.
      #
      [keyfile]
      unmanaged-devices=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:vxlan-v6.calico;interface-name:wireguard.cali;interface-name:wg-v6.cali
    dest: /etc/NetworkManager/conf.d/calico.conf
    mode: '770'
    owner: root
    group: root
  diff: true
  when: directory_network_manager_metadata.stat.exists


- name: Check if K3s Installed
  ansible.builtin.shell:
    cmd: |
      if [[ $(service k3s status) ]]; then exit 0; else exit 1; fi
    executable: /bin/bash
  changed_when: false
  failed_when: false
  register: k3s_installed
  when: >
    nfc_role_kubernetes_master | default(false) | bool


- name: Check if K3s Installed
  ansible.builtin.shell:
    cmd: |
      if [[ $(service k3s-agent status) ]]; then exit 0; else exit 1; fi
    executable: /bin/bash
  changed_when: false
  failed_when: false
  register: k3s_installed
  when: >
    not nfc_role_kubernetes_worker | default(false) | bool


- name: Download Install Scripts
  ansible.builtin.uri:
    url: "{{ item.url }}"
    method: GET
    return_content: true
    status_code:
      - 200
      - 304
    dest: "{{ item.dest }}"
    mode: "744"
  changed_when: false
  register: k3s_download_script
  delegate_to: localhost
  run_once: true
  # no_log: true
  when: >
    ansible_os_family == 'Debian'
      and
    item.when | default(true) | bool
  loop: "{{ download_files }}"
  vars:
    ansible_connection: local
    download_files:
      - dest: /tmp/install.sh
        url: https://get.k3s.io
      - dest: /tmp/install_olm.sh
        url: https://raw.githubusercontent.com/operator-framework/operator-lifecycle-manager/v{{ kubernetes_version_olm }}/scripts/install.sh
        when: "{{ nfc_role_kubernetes_install_olm }}"


- name: Download K3s Binary
  ansible.builtin.uri:
    url: |-
      https://github.com/k3s-io/k3s/releases/download/v
        {{- KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}}
      /k3s
      {%- if cpu_arch.key == 'aarch64' -%}
      -arm64
      {%- endif %}
    method: GET
    return_content: false
    status_code:
      - 200
      - 304
    dest: "/tmp/k3s.{{ cpu_arch.key }}"
    mode: "744"
  changed_when: false
  register: k3s_download_files
  delegate_to: localhost
  run_once: true
  # no_log: true
  when: ansible_os_family == 'Debian'
  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
  loop_control:
    loop_var: cpu_arch
  vars:
    ansible_connection: local


- name: "[TRACE] Downloaded File SHA256"
  ansible.builtin.set_fact:
    hash_sha256_k3s_downloaded_binary: "{{ lookup('ansible.builtin.file', '/tmp/k3s.' + cpu_arch.key) | hash('sha256') | string }}"
  delegate_to: localhost
  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
  loop_control:
    loop_var: cpu_arch


- name: Existing k3s File hash
  ansible.builtin.stat:
    checksum_algorithm: sha256
    name: /usr/local/bin/k3s
  register: hash_sha256_k3s_existing_binary


- name: Copy K3s binary to Host
  ansible.builtin.copy:
    src: "/tmp/k3s.{{ ansible_architecture }}"
    dest: "/usr/local/bin/k3s"
    mode: '741'
    owner: root
    group: root
  when: hash_sha256_k3s_existing_binary.stat.checksum | default('0') != hash_sha256_k3s_downloaded_binary


- name: Copy install scripts to Host
  ansible.builtin.copy:
    src: "{{ item.path }}"
    dest: "{{ item.path }}"
    mode: '755'
    owner: root
    group: root
  changed_when: false
  loop: "{{ install_scripts }}"
  vars:
    install_scripts:
      - path: "/tmp/install.sh"
      - path: "/tmp/install_olm.sh"
        when: "{{ nfc_role_kubernetes_install_olm }}"
  when: >
    item.when | default(true) | bool


- name: Required Initial config files
  ansible.builtin.copy:
    content: |
      {{  item.content }}
    dest: "{{  item.path }}/{{  item.name }}"
    mode: '740'
    owner: root
    group: root
  loop: "{{ k3s.files }}"
  when: >
    item.when | default(true) | bool
  #   kubernetes_config.cluster.prime.name == inventory_hostname


- name: Copy Intial required templates
  ansible.builtin.template:
    src: "{{ item.src }}"
    dest: "{{ item.dest }}"
    owner: root
    mode: '700'
    force: true
  notify: "{{ item.notify | default(omit) }}"
  loop: "{{ templates_to_apply }}"
  diff: true
  when: >
    item.when | default(true) | bool
  vars:
    templates_to_apply:
      - src: k3s-config.yaml.j2
        dest: /etc/rancher/k3s/config.yaml
        notify: kubernetes_restart
      - src: "calico.yaml.j2"
        dest: /var/lib/rancher/k3s/server/manifests/calico.yaml
        when: >
          {{
            kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
              and
            (
              (
                not file_calico_operator_yaml_metadata.stat.exists
                  and
                file_calico_yaml_metadata.stat.exists
                  and
                k3s_installed.rc == 0
              )
                or
              'calico_manifest' in ansible_run_tags
            )
          }}
      - src: k3s-registries.yaml.j2
        dest: /etc/rancher/k3s/registries.yaml
        notify: kubernetes_restart
        when: "{{ (kubernetes_private_container_registry | default([])) | from_yaml | list | length > 0 }}"


# - name: Templates IPv6
#   ansible.builtin.template:
#     src: iptables-kubernetes.rules.j2
#     dest: "/etc/ip6tables.rules.d/ip6tables-kubernetes.rules"
#     owner: root
#     mode: '700'
#     force: true
#   vars:
#     ipv6: true


- name: Set IPTables to legacy mode
  ansible.builtin.command:
    cmd: update-alternatives --set iptables /usr/sbin/iptables-legacy
  changed_when: false


- name: Install K3s (prime master)
  ansible.builtin.shell:
    cmd: |
      INSTALL_K3S_SKIP_DOWNLOAD=true \
      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
      /tmp/install.sh {% if nfc_role_kubernetes_etcd_enabled %}--cluster-init{% endif %}
  changed_when: false
  when: >
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    k3s_installed.rc == 1


- name: Install Calico Operator
  ansible.builtin.include_tasks:
    file: migrate_to_operator.yaml
    apply:
      tags:
        - always
  when: >-
    (
      (
        'operator_migrate_calico' in ansible_run_tags
          or
        'operator_calico' in ansible_run_tags
      )
        or
      not file_calico_yaml_metadata.stat.exists
    )
      and
    'calico_manifest' not in ansible_run_tags
      and
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname


- name: Install MetalLB Operator
  ansible.builtin.include_tasks:
    file: manifest_apply.yaml
    apply:
      tags:
        - always
  loop: "{{ manifests }}"
  loop_control:
    loop_var: manifest
  vars:
    manifests:
      - name: MetalLB Operator
        template: Deployment-manifest-MetalLB_Operator.yaml
  when: >-
    nfc_kubernetes_enable_metallb | default(false) | bool
      and
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname


- name: Wait for kubernetes prime to be ready
  ansible.builtin.shell:
    cmd: |
      set -o pipefail
      if [ `which jq` ]; then
        echo $(kubectl get no $(hostname) -o json | jq .status.conditions[4].status | tr -d '"');
      else
        echo jq command not found;
        exit 127;
      fi
    executable: /bin/bash
  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
  run_once: true
  register: kubernetes_ready_check
  retries: 30
  delay: 10
  until: >
    kubernetes_ready_check.stdout | default(false) | bool
      or
    kubernetes_ready_check.rc != 0
  changed_when: false
  failed_when: kubernetes_ready_check.rc != 0


- name: Install olm
  ansible.builtin.shell:
    cmd: |
      /tmp/install_olm.sh v{{ kubernetes_version_olm }}
  changed_when: false
  failed_when: >
    'already installed' not in install_olm.stdout
      and
    install_olm.rc == 1
  register: install_olm
  when: >
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    nfc_role_kubernetes_install_olm | default(false) | bool


- name: Uninstall OLM
  ansible.builtin.shell:
    cmd: |
      kubectl delete -n olm deployment packageserver;
      kubectl delete -n olm deployment catalog-operator;
      kubectl delete -n olm deployment olm-operator;

      kubectl delete crd catalogsources.operators.coreos.com;
      kubectl delete crd clusterserviceversions.operators.coreos.com;
      kubectl delete crd installplans.operators.coreos.com;
      kubectl delete crd olmconfigs.operators.coreos.com;
      kubectl delete crd operatorconditions.operators.coreos.com;
      kubectl delete crd operatorgroups.operators.coreos.com;
      kubectl delete crd operators.operators.coreos.com;
      kubectl delete crd subscriptions.operators.coreos.com;

      kubectl delete namespace operators --force;
      kubectl delete namespace olm --force;

  changed_when: false
  failed_when: false
  register: install_olm
  when: >
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    'olm_uninstall' in ansible_run_tags


- name: Enable Cluster Encryption
  ansible.builtin.command:
    cmd: kubectl patch felixconfiguration default --type='merge' -p '{"spec":{"wireguardEnabled":true,"wireguardEnabledV6":true}}'
  changed_when: false
  failed_when: false    # New cluster will fail
  when: >
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    kubernetes_config.cluster.networking.encrypt | default(false) | bool
      and
    (
      'calico_manifest' in ansible_run_tags
        or
      (
        'operator_migrate_calico' not in ansible_run_tags
          or
        'operator_calico' not in ansible_run_tags
      )
    )


- name: Fetch Join Token
  ansible.builtin.slurp:
    src: /var/lib/rancher/k3s/server/token
  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
  run_once: true
  register: k3s_join_token
  no_log: true # Value is sensitive


- name: Create Token fact
  ansible.builtin.set_fact:
    k3s_join_token: "{{ k3s_join_token.content | b64decode | replace('\n', '') }}"
  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
  run_once: true
  no_log: true # Value is sensitive


- name: Install K3s (master nodes)
  ansible.builtin.shell:
    cmd: |
      INSTALL_K3S_EXEC="server" \
      INSTALL_K3S_SKIP_DOWNLOAD=true \
      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
      K3S_TOKEN="{{ k3s_join_token }}" \
      /tmp/install.sh
    executable: /bin/bash
  changed_when: false
  when: >
    nfc_role_kubernetes_master | default(false) | bool
      and
    not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    k3s_installed.rc == 1


- name: Install K3s (worker nodes)
  ansible.builtin.shell:
    cmd: |
      set -o pipefail
      INSTALL_K3S_EXEC="agent" \
      INSTALL_K3S_SKIP_DOWNLOAD=true \
      INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
      K3S_TOKEN="{{ k3s_join_token }}" \
      K3S_URL="https://{{ hostvars[kubernetes_config.cluster.prime.name | default(inventory_hostname)].ansible_host }}:6443" \
      /tmp/install.sh -
    executable: /bin/bash
  changed_when: false
  when: >
    not nfc_role_kubernetes_master | default(false) | bool
      and
    not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    k3s_installed.rc == 1


- name: Set Kubernetes Final Install Fact
  ansible.builtin.set_fact:
    kubernetes_installed: true
    # Clear Token as no llonger required and due to being a sensitive value
    k3s_join_token: null