kubernetes/roles/nfc_kubernetes/tasks/k3s/install.yaml

---

- name: Install required python modules
  ansible.builtin.pip:
    name: "{{ item }}"
    state: present
  loop: "{{ pip_packages }}"
  vars:
    pip_packages:
      - kubernetes>=12.0.0
      - PyYAML>=3.11


- name: Check for calico deployment manifest
  ansible.builtin.stat:
    name: /var/lib/rancher/k3s/server/manifests/calico.yaml
  register: file_calico_yaml_metadata


- name: Check for calico Operator deployment manifest
  ansible.builtin.stat:
    name: /var/lib/rancher/k3s/ansible/deployment-manifest-calico_operator.yaml
  register: file_calico_operator_yaml_metadata


- name: Install dependent packages
  ansible.builtin.apt:
    name: "{{ package }}"
    state: present
  loop: "{{ packages }}"
  loop_control:
    loop_var: package
  vars:
    packages:
      - wget
      - curl
      - iptables
      - jq
      - wireguard


- name: Remove swapfile from /etc/fstab
  ansible.posix.mount:
    name: "{{ item }}"
    fstype: swap
    state: absent
  with_items:
    - swap
    - none
  when:
    - ansible_os_family == 'Debian' # ansible_lsb.codename = bullseye, ansible_lsb.major_release = 11
  tags:
    - install


- name: Testing Environment try/catch
  block:


    - name: Disable swap
      ansible.builtin.command:
        cmd: swapoff -a
      changed_when: false
      when:
        - ansible_os_family == 'Debian'
      tags:
        - install


  rescue:

    - name: Check if inside Gitlab CI
      ansible.builtin.assert:
        that:
          - lookup('ansible.builtin.env', 'CI_COMMIT_SHA') | default('') != ''
        success_msg: "Inside testing enviroment, 'Disable swap' error OK"
        fail_msg: "You should figure out what went wrong"


- name: Check an armbian os system
  ansible.builtin.stat:
    path: /etc/default/armbian-zram-config
  register: armbian_stat_result


- name: Armbian Disable Swap
  ansible.builtin.shell:
    cmd: |
      sed -i 's/\# SWAP=false/SWAP=false/g' /etc/default/armbian-zram-config;
      sed -i 's/ENABLED=true/ENABLED=false/g' /etc/default/armbian-zram-config;
  args:
    executable: bash
  changed_when: false
  when: armbian_stat_result.stat.exists


- name: Create Required directories
  ansible.builtin.file:
    name: "{{ item.name }}"
    state: "{{ item.state }}"
    mode: "{{ item.mode }}"
  loop: "{{ dirs }}"
  vars:
    dirs:
      - name: /etc/rancher/k3s
        state: directory
        mode: 700
      - name: /var/lib/rancher/k3s/server/logs
        state: directory
        mode: 700
      - name: /var/lib/rancher/k3s/server/manifests
        state: directory
        mode: 700
      - name: /var/lib/rancher/k3s/ansible
        state: directory
        mode: 700


- name: Add sysctl net.ipv4.ip_forward
  ansible.posix.sysctl:
    name: "{{ item.name }}"
    value: "{{ item.value }}"
    sysctl_set: true
    state: present
    reload: true
  loop: "{{ settings }}"
  notify: reboot_host    # On change reboot
  vars:
    settings:
      - name: net.ipv4.ip_forward
        value: '1'
      - name: fs.inotify.max_user_watches
        value: '524288'
      - name: fs.inotify.max_user_instances
        value: '512'
      - name: net.ipv6.conf.all.disable_ipv6
        value: '1'
  when:
    - ansible_os_family == 'Debian'


- name: Check for Network Manager Directory
  ansible.builtin.stat:
    name: /etc/NetworkManager/conf.d
  register: directory_network_manager_metadata


- name: Network Manager Setup
  ansible.builtin.copy:
    content: |-
      #
      # K3s Configuration for Network Manager
      #
      # Managed By ansible/role/nfc_kubernetes
      #
      # Dont edit this file directly as it will be overwritten.
      #
      [keyfile]
      unmanaged-devices=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:vxlan-v6.calico;interface-name:wireguard.cali;interface-name:wg-v6.cali
    dest: /etc/NetworkManager/conf.d/calico.conf
    mode: '770'
    owner: root
    group: root
  diff: true
  when: directory_network_manager_metadata.stat.exists


- name: File Metadata - k3s binary
  ansible.builtin.stat:
    checksum_algorithm: sha256
    name: /usr/local/bin/k3s
  register: metadata_file_k3s_existing_binary


- name: File Metadata - k3s[-agent].service
  ansible.builtin.stat:
    checksum_algorithm: sha256
    name: |-
      /etc/systemd/system/k3s
      {%- if not nfc_role_kubernetes_master | default(false) | bool -%}
        -agent
      {%- endif -%}
      .service
  register: metadata_file_k3s_service


- name: Directory Metadata - /etc/rancher/k3s/k3s.yaml
  ansible.builtin.stat:
    name: /etc/rancher/k3s/k3s.yaml
  register: metadata_dir_etc_k3s


- name: File Metadata - /var/lib/rancher/k3s/server/token
  ansible.builtin.stat:
    checksum_algorithm: sha256
    name: /var/lib/rancher/k3s/server/token
  register: metadata_file_var_k3s_token


- name: Config Link
  ansible.builtin.shell:
    cmd: >
      ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config
    executable: bash
    creates: ~/.kube/config
  when: >
    nfc_role_kubernetes_master | default(false) | bool
      and
    metadata_dir_etc_k3s.stat.exists | default(false) | bool


- name: Fetch Kubernetes Node Object
  kubernetes.core.k8s_info:
    kind: Node
    name: "{{ inventory_hostname }}"
  register: kubernetes_node
  when: >
    metadata_file_k3s_existing_binary.stat.exists | default(false) | bool
      and
    metadata_file_k3s_service.stat.exists | default(false) | bool
      and
    metadata_dir_etc_k3s.stat.exists | default(false) | bool
      and
    metadata_file_var_k3s_token.stat.exists | default(false) | bool


- name: Fetch Installed K3s Metadata
  ansible.builtin.shell:
    cmd: |
      export installed_version=$(k3s --version | grep k3s | awk '{print $3}');
      export installed="
      {%- if
        metadata_file_k3s_existing_binary.stat.exists | default(false) | bool
          and
        metadata_file_k3s_service.stat.exists | default(false) | bool
          and
        metadata_dir_etc_k3s.stat.exists | default(false) | bool
          and
        metadata_file_var_k3s_token.stat.exists | default(false) | bool
      -%}
        true
      {%- else -%}
        false
      {%- endif -%}";

      if ! service k3s status > /dev/null; then

        export installed='false';

      fi

      export running_version="{{ kubernetes_node.resources[0].status.nodeInfo.kubeletVersion | default('0') }}";

      export correct_hash=$(wget -q https://github.com/k3s-io/k3s/releases/download/v
        {{-KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode -}}
          /sha256sum-
          {%- if ansible_architecture | lower == 'x86_64' -%}
            amd64
          {%- elif ansible_architecture | lower == 'aarch64' -%}
            arm64
          {%- endif %}.txt -O - | grep -v 'images' | awk '{print $1}');

      cat <<EOF
      {
        "current_hash": "{{ metadata_file_k3s_existing_binary.stat.checksum | default('') }}",
        "current_version": "${installed_version}",
        "desired_hash": "${correct_hash}",
        "desired_version": "v{{ KubernetesVersion + KubernetesVersion_k3s_prefix | default('') }}",
        "installed": ${installed},
        "running_version": "${running_version}"
      }

      EOF

    executable: /bin/bash
  changed_when: false
  check_mode: false
  failed_when: false
  register: k3s_metadata


- name: K3s Metadata Fact
  ansible.builtin.set_fact:
    node_k3s: "{{ k3s_metadata.stdout | from_yaml }}"


- name: Cached K3s Binary Details
  ansible.builtin.stat:
    path: "/tmp/k3s.{{ ansible_architecture }}"
    checksum_algorithm: sha256
  delegate_to: localhost
  register: file_cached_k3s_binary
  vars:
    ansible_connection: local


- name: Remove Cached K3s Binaries
  ansible.builtin.file:
    path: "/tmp/k3s.{{ ansible_architecture }}"
    state: absent
  delegate_to: localhost
  vars:
    ansible_connection: local
  when: >
    file_cached_k3s_binary.stat.checksum | default('0') != node_k3s.desired_hash

  # Workaround. See: https://github.com/ansible/awx/issues/15161
- name: Build K3s Download URL
  ansible.builtin.set_fact:
    cacheable: false
    url_download_k3s: |-
      [
        {%- for key, value in nfc_kubernetes_install_architectures | dict2items -%}
        "https://github.com/k3s-io/k3s/releases/download/
          {{- node_k3s.desired_version | urlencode -}}
        /k3s
        {%- if key == 'aarch64' -%}
        -arm64
        {%- endif %}",
        {%- endfor -%}
      ]
  changed_when: false
  check_mode: false
  delegate_to: localhost
  loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
  loop_control:
    loop_var: cpu_arch
  vars:
    ansible_connection: local


- name: Download K3s Binary
  ansible.builtin.uri:
    url: "{{ url | string }}"
    method: GET
    return_content: false
    status_code:
      - 200
      - 304
    dest: "/tmp/k3s.{{ ansible_architecture }}"
    mode: "744"
  changed_when: not ansible_check_mode
  check_mode: false
  delay: 10
  retries: 3
  register: k3s_download_files
  delegate_to: localhost
  failed_when: >
    (lookup('ansible.builtin.file', '/tmp/k3s.' + ansible_architecture) | hash('sha256') | string) != node_k3s.desired_hash
      and
    (
      k3s_download_files.status | int != 200
        or
      k3s_download_files.status | int != 304
    )
  run_once: true
  when: ansible_os_family == 'Debian'
  loop: "{{ url_download_k3s | from_yaml }}"
  loop_control:
    loop_var: url
  vars:
    ansible_connection: local


- name: Copy K3s binary to Host
  ansible.builtin.copy:
    src: "/tmp/k3s.{{ ansible_architecture }}"
    dest: "/usr/local/bin/k3s"
    mode: '741'
    owner: root
    group: root
  register: k3s_binary_copy
  when: >
    node_k3s.current_hash != node_k3s.desired_hash


- name: K3s Binary Upgrade
  ansible.builtin.service:
    name: |-
      {%- if nfc_role_kubernetes_master | default(false) | bool -%}
        k3s
      {%- else -%}
        k3s-agent
      {%- endif %}
    state: restarted
  register: k3s_upgrade_service_restart
  when: >
    (
      k3s_binary_copy.changed | default(false) | bool
        and
      node_k3s.installed | default(false) | bool
    )
      or
    (
      node_k3s.running_version != node_k3s.desired_version
        and
      node_k3s.installed | default(false) | bool
    )


- name: Create Fact - cluster_upgraded
  ansible.builtin.set_fact:
    nfc_role_kubernetes_cluster_upgraded: true
    cacheable: true
  when: >
    k3s_upgrade_service_restart.changed | default(false) | bool


- name: Download Install Scripts
  ansible.builtin.uri:
    url: "{{ item.url }}"
    method: GET
    return_content: true
    status_code:
      - 200
      - 304
    dest: "{{ item.dest }}"
    mode: "744"
  check_mode: false
  changed_when: false
  delay: 10
  retries: 3
  register: k3s_download_script
  delegate_to: localhost
  run_once: true
  # no_log: true
  when: >
    ansible_os_family == 'Debian'
      and
    item.when | default(true) | bool
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
  loop: "{{ download_files }}"
  vars:
    ansible_connection: local
    download_files:
      - dest: /tmp/install.sh
        url: https://get.k3s.io
      - dest: /tmp/install_olm.sh
        url: https://raw.githubusercontent.com/operator-framework/operator-lifecycle-manager/v{{ kubernetes_version_olm }}/scripts/install.sh
        when: "{{ nfc_role_kubernetes_install_olm }}"


- name: Copy install scripts to Host
  ansible.builtin.copy:
    src: "{{ item.path }}"
    dest: "{{ item.path }}"
    mode: '755'
    owner: root
    group: root
  changed_when: false
  loop: "{{ install_scripts }}"
  vars:
    install_scripts:
      - path: "/tmp/install.sh"
      - path: "/tmp/install_olm.sh"
        when: "{{ nfc_role_kubernetes_install_olm }}"
  when: >
    item.when | default(true) | bool
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Required Initial config files
  ansible.builtin.copy:
    content: |
      {{  item.content }}
    dest: "{{  item.path }}/{{  item.name }}"
    mode: '740'
    owner: root
    group: root
  loop: "{{ k3s.files }}"
  when: >
    item.when | default(true) | bool
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Copy Intial required templates
  ansible.builtin.template:
    src: "{{ item.src }}"
    dest: "{{ item.dest }}"
    owner: root
    mode: '700'
    force: true
  notify: "{{ item.notify | default(omit) }}"
  loop: "{{ templates_to_apply }}"
  diff: true
  when: >
    item.when | default(true) | bool
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
  vars:
    templates_to_apply:
      - src: k3s-config.yaml.j2
        dest: /etc/rancher/k3s/config.yaml
        notify: kubernetes_restart
      - src: "calico.yaml.j2"
        dest: /var/lib/rancher/k3s/server/manifests/calico.yaml
        when: >
          {{
            kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
              and
            (
              (
                not file_calico_operator_yaml_metadata.stat.exists
                  and
                file_calico_yaml_metadata.stat.exists
                  and
                not node_k3s.installed | bool
              )
                or
              'calico_manifest' in ansible_run_tags
            )
          }}
      - src: k3s-registries.yaml.j2
        dest: /etc/rancher/k3s/registries.yaml
        notify: kubernetes_restart
        when: "{{ (kubernetes_private_container_registry | default([])) | from_yaml | list | length > 0 }}"


# - name: Templates IPv6
#   ansible.builtin.template:
#     src: iptables-kubernetes.rules.j2
#     dest: "/etc/ip6tables.rules.d/ip6tables-kubernetes.rules"
#     owner: root
#     mode: '700'
#     force: true
#   vars:
#     ipv6: true


- name: Set IPTables to legacy mode
  ansible.builtin.command:
    cmd: update-alternatives --set iptables /usr/sbin/iptables-legacy
  changed_when: false
  when: >
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Install K3s (prime master)
  ansible.builtin.shell:
    cmd: |
      INSTALL_K3S_SKIP_DOWNLOAD=true \
      INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \
      /tmp/install.sh {% if nfc_role_kubernetes_etcd_enabled %}--cluster-init{% endif %}
  changed_when: false
  when: >
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    not node_k3s.installed | bool
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Install Calico Operator
  ansible.builtin.include_tasks:
    file: migrate_to_operator.yaml
    apply:
      tags:
        - always
  when: >-
    (
      (
        'operator_migrate_calico' in ansible_run_tags
          or
        'operator_calico' in ansible_run_tags
      )
        or
      not file_calico_yaml_metadata.stat.exists
    )
      and
    'calico_manifest' not in ansible_run_tags
      and
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Install MetalLB Operator
  ansible.builtin.include_tasks:
    file: manifest_apply.yaml
    apply:
      tags:
        - always
  loop: "{{ manifests }}"
  loop_control:
    loop_var: manifest
  vars:
    manifests:
      - name: MetalLB Operator
        template: Deployment-manifest-MetalLB_Operator.yaml
  when: >-
    nfc_kubernetes_enable_metallb | default(false) | bool
      and
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Wait for kubernetes prime to be ready
  ansible.builtin.shell:
    cmd: |
      set -o pipefail
      if [ `which jq` ]; then
        echo $(kubectl get no $(hostname) -o json | jq .status.conditions[4].status | tr -d '"');
      else
        echo jq command not found;
        exit 127;
      fi
    executable: /bin/bash
  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
  run_once: true
  register: kubernetes_ready_check
  retries: 30
  delay: 10
  until: >
    kubernetes_ready_check.stdout | default(false) | bool
      or
    kubernetes_ready_check.rc != 0
  changed_when: false
  failed_when: kubernetes_ready_check.rc != 0
  when: >
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
      and
    not ansible_check_mode


- name: Config Link
  ansible.builtin.shell:
    cmd: >
      ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config
    executable: bash
    creates: ~/.kube/config
  when: >
    nfc_role_kubernetes_master | default(false) | bool
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Install olm
  ansible.builtin.shell:
    cmd: |
      /tmp/install_olm.sh v{{ kubernetes_version_olm }}
  changed_when: false
  failed_when: >
    'already installed' not in install_olm.stdout
      and
    install_olm.rc == 1
  register: install_olm
  when: >
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    nfc_role_kubernetes_install_olm | default(false) | bool
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Uninstall OLM
  ansible.builtin.shell:
    cmd: |
      kubectl delete -n olm deployment packageserver;
      kubectl delete -n olm deployment catalog-operator;
      kubectl delete -n olm deployment olm-operator;

      kubectl delete crd catalogsources.operators.coreos.com;
      kubectl delete crd clusterserviceversions.operators.coreos.com;
      kubectl delete crd installplans.operators.coreos.com;
      kubectl delete crd olmconfigs.operators.coreos.com;
      kubectl delete crd operatorconditions.operators.coreos.com;
      kubectl delete crd operatorgroups.operators.coreos.com;
      kubectl delete crd operators.operators.coreos.com;
      kubectl delete crd subscriptions.operators.coreos.com;

      kubectl delete namespace operators --force;
      kubectl delete namespace olm --force;

  changed_when: false
  failed_when: false
  register: install_olm
  when: >
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    'olm_uninstall' in ansible_run_tags
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Enable Cluster Encryption
  ansible.builtin.command:
    cmd: kubectl patch felixconfiguration default --type='merge' -p '{"spec":{"wireguardEnabled":true,"wireguardEnabledV6":true}}'
  changed_when: false
  failed_when: false    # New cluster will fail
  when: >
    kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    kubernetes_config.cluster.networking.encrypt | default(false) | bool
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool
      and
    (
      'calico_manifest' in ansible_run_tags
        or
      (
        'operator_migrate_calico' not in ansible_run_tags
          or
        'operator_calico' not in ansible_run_tags
      )
    )


- name: Fetch Join Token
  ansible.builtin.slurp:
    src: /var/lib/rancher/k3s/server/token
  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
  run_once: true
  register: k3s_join_token
  no_log: true # Value is sensitive
  when: >
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Create Token fact
  ansible.builtin.set_fact:
    k3s_join_token: "{{ k3s_join_token.content | b64decode | replace('\n', '') }}"
  delegate_to: "{{ kubernetes_config.cluster.prime.name | default(inventory_hostname) }}"
  run_once: true
  no_log: true # Value is sensitive
  when: >
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Install K3s (master nodes)
  ansible.builtin.shell:
    cmd: |
      INSTALL_K3S_EXEC="server" \
      INSTALL_K3S_SKIP_DOWNLOAD=true \
      INSTALL_K3S_VERSION="{{ node_k3s.desired_version }}" \
      K3S_TOKEN="{{ k3s_join_token }}" \
      /tmp/install.sh
    executable: /bin/bash
  changed_when: false
  when: >
    nfc_role_kubernetes_master | default(false) | bool
      and
    not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    not node_k3s.installed | bool
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Install K3s (worker nodes)
  ansible.builtin.shell:
    cmd: |
      set -o pipefail
      INSTALL_K3S_EXEC="agent" \
      INSTALL_K3S_SKIP_DOWNLOAD=true \
      INSTALL_K3S_VERSION="v{{ node_k3s.desired_version }}" \
      K3S_TOKEN="{{ k3s_join_token }}" \
      K3S_URL="https://{{ hostvars[kubernetes_config.cluster.prime.name | default(inventory_hostname)].ansible_host }}:6443" \
      /tmp/install.sh -
    executable: /bin/bash
  changed_when: false
  when: >
    not nfc_role_kubernetes_master | default(false) | bool
      and
    not kubernetes_config.cluster.prime.name | default(inventory_hostname) == inventory_hostname
      and
    not node_k3s.installed | bool
      and
    not nfc_role_kubernetes_cluster_upgraded | default(false) | bool


- name: Set Kubernetes Final Install Fact
  ansible.builtin.set_fact:
    kubernetes_installed: true
    # Clear Token as no llonger required and due to being a sensitive value
    k3s_join_token: null