feat(calico_operator): deploy the operator to configure calico cni

!17 fixes #3

defaults/main.yml

@@ -9,6 +9,21 @@ ContainerDioVersion: 1.6.20-1
 KubernetesVersion: '1.26.2'                                # must match the repository release version
 kubernetes_version_olm: '0.26.0'
 
+
+# Depreciated:
+#      Calico is being migrated to use the calico operator.
+#      in a near future release, this method of deploying calico
+#      will be removed. use tag `operator_migrate_calico` to migrate
+calico_image_tag: v3.25.0 # Depreciated
+# EoF Depreciated
+# SoF New Variables
+nfc_kubernetes_calico_version: v3.27.0
+nfc_kubernetes_tigera_operator_registry: quay.io
+nfc_kubernetes_tigera_operator_image: tigera/operator
+nfc_kubernetes_tigera_operator_tag: v1.32.3               # Calico v3.27.0
+# EoF New Variables, EEoF Depreciated
+
+
 KubernetesVersion_k8s_prefix: '-00'
 KubernetesVersion_k3s_prefix: '+k3s1'
 

tasks/k3s.yaml

@@ -4,16 +4,31 @@
 - name: K3s Install
   ansible.builtin.include_tasks:
     file: k3s/install.yaml
+    apply:
+      tags:
+        - always
   when: >
     install_kubernetes | default(true) | bool
       and
     not kubernetes_installed | default(false) | bool
+  tags:
+    - never
+    - install
+    - operator_calico
+    - operator_migrate_calico
 
 
 - name: K3s Configure
   ansible.builtin.include_tasks:
     file: k3s/configure.yaml
+    apply:
+      tags:
+        - always
   when: >
     install_kubernetes | default(true) | bool
       and
     kubernetes_installed | default(false) | bool
+  tags:
+    - never
+    - install
+    - operator_migrate_calico

tasks/k3s/install.yaml

@@ -243,7 +243,14 @@
         notify: kubernetes_restart
       - src: "calico.yaml.j2"
         dest: /var/lib/rancher/k3s/server/manifests/calico.yaml
-        when: "{{ kubernetes_config.cluster.prime.name == inventory_hostname }}"
+        when: >
+          {{ kubernetes_config.cluster.prime.name == inventory_hostname
+            and
+          (
+            'operator_migrate_calico' not in ansible_run_tags
+              and
+            'operator_calico' not in ansible_run_tags
+          ) }}
       - src: k3s-registries.yaml.j2
         dest: /etc/rancher/k3s/registries.yaml
         notify: kubernetes_restart
@@ -316,6 +323,23 @@
       and
     kubernetes_olm_install | default(false) | bool
 
+
+- name: Install Calico Operator
+  ansible.builtin.include_tasks:
+    file: migrate_to_operator.yaml
+    apply:
+      tags:
+        - always
+  when: >-
+    (
+      'operator_migrate_calico' in ansible_run_tags
+        or
+      'operator_calico' in ansible_run_tags
+    )
+      and
+    kubernetes_config.cluster.prime.name == inventory_hostname
+
+
 - name: Enable Cluster Encryption
   ansible.builtin.command:
     cmd: kubectl patch felixconfiguration default --type='merge' -p '{"spec":{"wireguardEnabled":true,"wireguardEnabledV6":true}}'
@@ -324,6 +348,12 @@
     kubernetes_config.cluster.prime.name == inventory_hostname
       and
     kubernetes_config.cluster.networking.encrypt | default(false) | bool
+      and
+    (
+      'operator_migrate_calico' not in ansible_run_tags
+        or
+      'operator_calico' not in ansible_run_tags
+    )
 
 
 - name: Fetch Join Token

tasks/k3s/migrate_to_operator.yaml

@@ -0,0 +1,215 @@
+---
+
+# Reference https://docs.tigera.io/calico/3.25/operations/operator-migration
+
+# Script creation of imageset: https://docs.tigera.io/calico/latest/operations/image-options/imageset#create-an-imageset
+# above may pull sha for arch of machine who ran the script
+
+- name: Try / Catch
+  vars:
+    operator_manifests:
+      - Deployment-manifest-Calico_Operator.yaml.j2
+      - Installation-manifest-Calico_Cluster.yaml.j2
+      - FeilixConfiguration-manifest-Calico_Cluster.yaml
+      - IPPool-manifest-Calico_Cluster.yaml.j2
+      - APIServer-manifest-Calico_Cluster.yaml
+      # - calico.yaml.j2
+  block:
+
+
+    - name: Check for calico deployment manifest
+      ansible.builtin.stat:
+        name: /var/lib/rancher/k3s/server/manifests/calico.yaml
+      become: true
+      register: file_calico_yaml_metadata
+
+
+    - name: Move Calico Manifest from addons directory
+      ansible.builtin.command:
+        cmd: mv /var/lib/rancher/k3s/server/manifests/calico.yaml /tmp/
+      become: true
+      changed_when: false
+      when: file_calico_yaml_metadata.stat.exists
+
+
+    - name: Remove addon from Kubernetes
+      ansible.builtin.command:
+        cmd: kubectl delete addon -n kube-system calico
+      become: true
+      changed_when: false
+      when: file_calico_yaml_metadata.stat.exists
+
+
+    - name: Uninstall Calico
+      ansible.builtin.command:
+        cmd: kubectl delete -f /tmp/calico.yaml
+      become: true
+      changed_when: false
+      when: file_calico_yaml_metadata.stat.exists
+
+
+    - name: Copy Manifest for addition
+      ansible.builtin.template:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item | lower | replace('.j2', '') }}"
+        mode: '744'
+      become: true
+      loop: "{{ operator_manifests }}"
+
+
+    - name: Try / Catch
+      block:
+
+
+        - name: Apply Operator Manifests
+          ansible.builtin.command:
+            cmd: "kubectl create -f /tmp/{{ item | lower | replace('.j2', '') }}"
+          become: true
+          changed_when: false
+          diff: true
+          failed_when: >
+            {{ 'Error from server' in operator_manifest_stdout.stderr }}
+          loop: "{{ operator_manifests }}"
+          register: operator_manifest_stdout
+
+
+      rescue:
+
+
+        - name: TRACE - Operator manifest apply
+          ansible.builtin.debug:
+            msg: "{{ operator_manifest_stdout }}"
+
+
+        - name: Apply Operator Manifests - "Rescue"
+          ansible.builtin.command:
+            cmd: "kubectl replace -f /tmp/{{ item | lower | replace('.j2', '') }}"
+          become: true
+          changed_when: false
+          diff: true
+          failed_when: >
+            {{
+              'Error from server' in operator_manifest_stdout.stderr
+                and
+              'ensure CRDs are installed first' in operator_manifest_stdout.stderr
+            }}
+          loop: "{{ operator_manifests }}"
+          register: operator_manifest_stdout
+
+
+        - name: TRACE - Operator manifest apply. Rescued
+          ansible.builtin.debug:
+            msg: "{{ operator_manifest_stdout }}"
+
+
+    - name: Fetch Calico Kubectl Plugin
+      ansible.builtin.uri:
+        url: |-
+          https://github.com/projectcalico/calico/releases/download/{{ nfc_kubernetes_calico_version }}/calicoctl-linux-
+          {%- if cpu_arch.key == 'aarch64' -%}
+            arm64
+          {%- else -%}
+            amd64
+          {%- endif %}
+        status_code:
+          - 200
+          - 304
+        dest: "/tmp/kubectl-calico.{{ cpu_arch.key }}"
+        mode: '777'
+        owner: root
+        group: 'root'
+      become: true
+      delegate_to: localhost
+      loop: "{{ nfc_kubernetes_install_architectures | dict2items }}"
+      loop_control:
+        loop_var: cpu_arch
+      vars:
+        ansible_connection: local
+
+
+    - name: Add calico Plugin
+      ansible.builtin.copy:
+        src: "/tmp/kubectl-calico.{{ ansible_architecture }}"
+        dest: /usr/local/bin/kubectl-calico
+        mode: '770'
+        owner: root
+        group: 'root'
+      become: true
+      when: inventory_hostname in groups['kubernetes_master']
+
+
+    - name: Setup Automagic Host Endpoints
+      ansible.builtin.shell:
+        cmd: |-
+          kubectl calico \
+            patch kubecontrollersconfiguration \
+            default --patch='{"spec": {"controllers": {"node": {"hostEndpoint": {"autoCreate": "Enabled"}}}}}'
+        executable: bash
+      become: true
+      changed_when: false
+      failed_when: false # fixme
+
+
+    - name: Remove calico migration label
+      ansible.builtin.shell:
+        cmd: |-
+          kubectl label \
+            {{ inventory_hostname }} \
+            projectcalico.org/operator-node-migration-
+        executable: bash
+      become: true
+      delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
+      changed_when: false
+      failed_when: false
+      loop: "{{ groups[kubernetes_config.cluster.group_name] }}"
+
+    # kubectl label node ip-10-229-92-202.eu-west-1.compute.internal projectcalico.org/operator-node-migration-
+    # migration started
+
+  rescue:
+
+
+    - name: Remove Operator Manifests
+      ansible.builtin.command:
+        cmd: "kubectl delete -f /tmp/{{ item | lower | replace('.j2', '') }}"
+      become: true
+      changed_when: false
+      failed_when: false
+      loop: "{{ operator_manifests }}"
+      when: file_calico_yaml_metadata.stat.exists # Only rescue if it was a migration
+
+
+    - name: Move Calico Manifest from addons directory
+      ansible.builtin.command:
+        cmd: mv /tmp/calico.yaml /var/lib/rancher/k3s/server/manifests/
+      become: true
+      changed_when: false
+      when: file_calico_yaml_metadata.stat.exists
+
+
+    - name: Re-install Calico
+      ansible.builtin.command:
+        cmd: kubectl apply -f /var/lib/rancher/k3s/server/manifests/calico.yaml
+      become: true
+      changed_when: false
+      when: file_calico_yaml_metadata.stat.exists
+
+
+  always:
+
+
+    - name: Remove Operator Manifests files
+      ansible.builtin.file:
+        name: "/tmp/{{ item | lower | replace('.j2', '') }}"
+        state: absent
+      become: true
+      changed_when: false
+      loop: "{{ operator_manifests }}"
+
+
+    - name: Clean-up Temp File
+      ansible.builtin.file:
+        name: /tmp/calico.yaml
+        state: absent
+      become: true
+      when: file_calico_yaml_metadata.stat.exists

tasks/main.yml

@@ -1,14 +1,38 @@
 ---
+
 - name: Firewall Rules
   ansible.builtin.include_role:
     name: nfc_firewall
   vars:
     nfc_firewall_enabled_kubernetes: "{{ nfc_kubernetes.enable_firewall | default(false) | bool }}"
+  tags:
+    - never
+    - install
+
+
+# fix, reload firewall `iptables-reloader`
+- name: Reload iptables
+  ansible.builtin.command:
+    cmd: bash -c /usr/bin/iptables-reloader
+  changed_when: false
+  tags:
+    - never
+    - install
+
 
 - name: K8s Cluster
   ansible.builtin.include_tasks: k8s.yaml
   when: kubernetes_type == 'k8s'
+  tags:
+    - never
+    - install
+
 
 - name: K3s Cluster
   ansible.builtin.include_tasks: k3s.yaml
   when: kubernetes_type == 'k3s'
+  tags:
+    - never
+    - install
+    - operator_calico
+    - operator_migrate_calico

templates/APIServer-manifest-Calico_Cluster.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: operator.tigera.io/v1
+kind: APIServer
+metadata:
+  name: default
+spec: {}

templates/FeilixConfiguration-manifest-Calico_Cluster.yaml

@@ -0,0 +1,14 @@
+---
+
+apiVersion: crd.projectcalico.org/v1
+kind: FelixConfiguration
+metadata:
+  name: default
+spec:
+  bpfLogLevel: ""
+  floatingIPs: Disabled
+  healthPort: 9099
+  logSeverityScreen: Info
+  reportingInterval: 0s
+  wireguardEnabled: true
+  wireguardEnabledV6: true

templates/IPPool-manifest-Calico_Cluster.yaml.j2

@@ -0,0 +1,16 @@
+---
+
+apiVersion: crd.projectcalico.org/v1
+kind: IPPool
+metadata:
+  name: default-ipv4-ippool
+spec:
+  allowedUses:
+    - Workload
+    - Tunnel
+  blockSize: 26
+  cidr: {{ kubernetes_config.cluster.networking.podSubnet }}
+  ipipMode: Never
+  natOutgoing: true
+  nodeSelector: all()
+  vxlanMode: Always

templates/Installation-manifest-Calico_Cluster.yaml.j2

@@ -0,0 +1,44 @@
+---
+
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  calicoNetwork:
+    bgp: Disabled
+    containerIPForwarding: Enabled
+    hostPorts: Enabled
+    ipPools:
+    - blockSize: 26
+      cidr: {{ kubernetes_config.cluster.networking.podSubnet }}
+      disableBGPExport: false
+      encapsulation: VXLAN
+      natOutgoing: Enabled
+      nodeSelector: all()
+    linuxDataplane: Iptables
+    mtu: 0
+    multiInterfaceMode: None
+    nodeAddressAutodetectionV4:
+      kubernetes: NodeInternalIP
+  cni:
+    ipam:
+      type: Calico
+    type: Calico
+  componentResources:
+  - componentName: Node
+    resourceRequirements:
+      requests:
+        cpu: 250m
+  controlPlaneReplicas: 3
+  flexVolumePath: None
+  kubeletVolumePluginPath: None
+  nodeUpdateStrategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
+  nonPrivileged: Disabled
+  serviceCIDRs:
+    - {{ kubernetes_config.cluster.networking.ServiceSubnet }}
+  variant: Calico

templates/calico.yaml.j2

@@ -1,4 +1,11 @@
 ---
+# Depreciated:
+#      Calico is being migrated to use the calico operator.
+#      in a near future release, this method of deploying calico
+#      will be removed. use tag `operator_migrate_calico` to migrate
+#      and tag `operator_calico` to keep.
+#
+#
 # URL: https://github.com/projectcalico/calico/blob/8f2548a71ddc4fbe2497a0c20a3b24fc7a165851/manifests/calico.yaml
 # Source: calico/templates/calico-kube-controllers.yaml
 # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict