feat(k8s): remove k8s cluster install

!35
This commit is contained in:
2024-03-12 18:53:57 +09:30
parent 9dad960208
commit 21d0dbefa9
10 changed files with 25 additions and 903 deletions

View File

@ -36,7 +36,6 @@ kubernetes_version_olm: '0.26.0'
KubernetesVersion_k8s_prefix: '-00'
KubernetesVersion_k3s_prefix: '+k3s1' KubernetesVersion_k3s_prefix: '+k3s1'
kubernetes_private_container_registry: [] # Optional, Array. if none use `[]` kubernetes_private_container_registry: [] # Optional, Array. if none use `[]`

View File

@ -1,303 +0,0 @@
---
- name: "{{ role_name }} Install Software"
include_role:
name: nfc_common
vars:
common_gather_facts: false
aptSigningKeys:
- name: docker
url: https://download.docker.com/linux/debian/gpg
save_directory: /usr/share/keyrings
file_extension: asc
- name: kubernetes
url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
save_directory: /usr/share/keyrings
file_extension: asc
aptRepositories:
- name: docker
repo: deb [arch={{ dynamic_processor_architecture }} signed-by=/usr/share/keyrings/docker.asc] http://download.docker.com/linux/{{ ansible_os_family | lower }} {{ ansible_lsb.codename | lower }} stable
- name: kubernetes
repo: deb [signed-by=/usr/share/keyrings/kubernetes.asc] http://apt.kubernetes.io/ kubernetes-xenial main
aptInstall:
- name: gnupg2
- name: apt-transport-https
- name: software-properties-common
- name: ca-certificates
- name: iptables
- name: python3-pip
- name: python3-virtualenv
- name: containerd.io
version: "{{ ContainerDioVersion }}"
- name: kubectl
version: "{{ KubernetesVersion }}"
- name: kubelet
version: "{{ KubernetesVersion }}"
- name: kubeadm
version: "{{ KubernetesVersion }}"
tags:
- install
# containerd.io=1.6.22-1 kubectl=1.26.9-00 kubelet=1.26.9-00 kubeadm=1.26.9-00
- name: Remove swapfile from /etc/fstab
mount:
name: "{{ item }}"
fstype: swap
state: absent
with_items:
- swap
- none
when:
- ansible_os_family == 'Debian' # ansible_lsb.codename = bullseye, ansible_lsb.major_release = 11
tags:
- install
- name: Disable swap
command: swapoff -a
changed_when: true == false
when:
#- ansible_swaptotal_mb > 0
- ansible_os_family == 'Debian'
tags:
- install
- name: Check an armbian os system
stat:
path: /etc/default/armbian-zram-config
register: armbian_stat_result
- name: Armbian Disable Swap
ansible.builtin.shell:
cmd: |
sed -i 's/\# SWAP=false/SWAP=false/g' /etc/default/armbian-zram-config;
sed -i 's/ENABLED=true/ENABLED=false/g' /etc/default/armbian-zram-config;
args:
executable: bash
changed_when: false
# failed_when: false
#notify: RebootHost # doesnt need to reboot as swapoff -a covers the deployment
when: armbian_stat_result.stat.exists
- name: Add the overlay module
community.general.modprobe:
name: overlay
state: present
when:
- ansible_os_family == 'Debian'
tags:
- install
- name: Add the br_netfilter module
community.general.modprobe:
name: br_netfilter
state: present
when:
- ansible_os_family == 'Debian'
tags:
- install
- name: check if containerd installed
ansible.builtin.shell:
cmd: which containerd
failed_when: false
changed_when: false
register: containerd_installed
- name: "Containerd.io Started?"
service:
name: containerd
state: started
tags:
- configure
- install
when: >
ansible_os_family == 'Debian'
and
containerd_installed.rc | default(1) | int == 0
- name: containerd load modules config
template:
src: "etc_module_containerd.conf"
dest: /etc/modules-load.d/containerd.conf
owner: root
mode: 0700
notify: "restart ContainerD"
when:
- ansible_os_family == 'Debian'
tags:
- install
- name: Create containerD host directories.
become_method: sudo
become: yes
file:
path: /etc/containerd/certs.d/{{ item.name }}
state: directory
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0700
with_items: "{{ containerd.repositories }}"
tags:
- install
- containerRegistry
- name: containerD registry host
template:
src: "containerd-registry-hosts.toml.j2"
dest: /etc/containerd/certs.d/{{ item.name }}/hosts.toml
owner: root
mode: 0700
notify: "restart ContainerD"
with_items: "{{ containerd.repositories }}"
when:
- ansible_os_family == 'Debian'
tags:
- install
- containerRegistry
- name: containerD default config
template:
src: "etc_containerd_containerd.toml"
dest: /etc/containerd/config.toml
owner: root
mode: 0700
notify: "restart ContainerD"
register: containerd_config
when:
- ansible_os_family == 'Debian'
tags:
- install
- containerRegistry
- name: Install required python modules
ansible.builtin.pip:
name: kubernetes
state: forcereinstall
#virtualenv: /tmp/venv_ansible
when: inventory_hostname != 'op1'
tags:
- install
- name: sysctl net.bridge.bridge-nf-call-ip6tables
sysctl:
name: net.bridge.bridge-nf-call-ip6tables
value: '1'
sysctl_set: yes
state: present
reload: yes
when:
- ansible_os_family == 'Debian'
tags:
- install
- name: sysctl net.bridge.bridge-nf-call-iptables
sysctl:
name: net.bridge.bridge-nf-call-iptables
value: '1'
sysctl_set: yes
state: present
reload: yes
when:
- ansible_os_family == 'Debian'
tags:
- install
- name: sysctl net.ipv4.ip_forward
sysctl:
name: net.ipv4.ip_forward
value: '1'
sysctl_set: yes
state: present
reload: yes
when:
- ansible_os_family == 'Debian'
tags:
- install
# - name: Check if kubernetes has been Initialized
# stat:
# path: /etc/kubernetes/admin.conf
# register: KubernetesInit
# when:
# - kubernetes_config.cluster.prime.name == inventory_hostname
- name: check if iptables is installed
ansible.builtin.shell: |-
dpkg -s iptables &> /dev/null
changed_when: true == false
register: iptables_installed
when:
- ansible_os_family == 'Debian'
tags:
- install
- iptables
- firewall
- name: Add kubernetes Firewall Rules - '/etc/iptables-kubernetes.rules'
template:
src: iptables-kubernetes.rules.j2
dest: "/etc/iptables-kubernetes.rules"
owner: root
mode: 0700
force: yes
notify: "Apply Firewall Rules"
when:
- ansible_os_family == 'Debian'
- iptables_installed.rc == 0
tags:
- install
- iptables
- firewall
- name: File - '/etc/network/if-pre-up.d/firewall-kubernetes'
template:
src: firewall-kubernetes.j2
dest: "/etc/network/if-pre-up.d/firewall-kubernetes"
owner: root
mode: 0700
force: yes
when:
- ansible_os_family == 'Debian'
- iptables_installed.rc == 0
tags:
- install
- iptables
- firewall
- name: Create local workdir
file:
path: "{{ item }}"
state: directory
mode: 0700
delegate_to: localhost
connection: local
run_once: true
changed_when: true == false
with_items:
- /tmp/ansible/
tags:
- always

View File

@ -1,29 +0,0 @@
---
# kubernetes_installed
- name: K3s Install
ansible.builtin.include_tasks:
file: k3s/install.yaml
apply:
tags:
- always
when: >
install_kubernetes | default(true) | bool
and
not kubernetes_installed | default(false) | bool
tags:
- always
- name: K3s Configure
ansible.builtin.include_tasks:
file: k3s/configure.yaml
apply:
tags:
- always
when: >
install_kubernetes | default(true) | bool
and
kubernetes_installed | default(false) | bool
tags:
- always

View File

@ -1,103 +0,0 @@
---
- name: Common Tasks
include_tasks: common.yaml
# tags:
# - install
- name: Check if kubernetes has been Initialized
stat:
path: /etc/kubernetes/admin.conf
register: KubernetesInitialized
tags:
- always
- name: kubernetes prime
include_tasks: prime.yaml
when: kubernetes_config.cluster.prime.name == inventory_hostname
- name: kubernetes workers
include_tasks: workers.yaml
when: kubernetes_config.cluster.prime.name != inventory_hostname
- name: Add Kubernetes Node Labels
kubernetes.core.k8s:
definition:
apiVersion: v1
kind: Node
metadata:
name: "{{ inventory_hostname }}"
labels:
"{{ item | from_yaml_all }}"
delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
with_items:
- "{{ kubernetes_config.hosts[inventory_hostname].labels }}"
when:
- ( kubernetes_config.hosts[inventory_hostname].labels is defined and
kubernetes_config.hosts[inventory_hostname].labels|default('')|length > 0 )
tags:
- install
- nodelabels
- name: Add Node Taints
kubernetes.core.k8s_taint:
state: "present"
name: "{{ inventory_hostname }}"
taints:
- "{{ item | from_yaml_all }}"
delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
with_items:
- "{{ kubernetes_config.hosts[inventory_hostname].taints.present }}"
when:
- (kubernetes_config.hosts[inventory_hostname].taints.present is defined and
kubernetes_config.hosts[inventory_hostname].taints.present|default('')|length > 0 )
tags:
- install
- taints
- name: Remove Node Taints
kubernetes.core.k8s_taint:
state: "absent"
name: "{{ inventory_hostname }}"
taints:
- "{{ item | from_yaml_all }}"
delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
with_items:
- "{{ kubernetes_config.hosts[inventory_hostname].taints.absent }}"
when:
- ( kubernetes_config.hosts[inventory_hostname].taints.absent is defined and
kubernetes_config.hosts[inventory_hostname].taints.absent|default('')|length > 0 )
tags:
- install
- taints
- name: Create Cluster Namespaces
kubernetes.core.k8s:
state: present
definition:
apiVersion: v1
kind: Namespace
metadata:
name: "{{ item.name }}"
labels:
#app.kubernetes.io/version: # App version
#app.kubernetes.io/component:
#app.kubernetes.io/part-of:
app.kubernetes.io/managed-by: Ansible
#meta.kubernetes.io/description: "{{ item.description | default('') }}"
meta.kubernetes.io/version: "{{ deployment_git_current_short_hash | default('') }}"
with_items:
- "{{ kubernetes_config.namespaces }}"
when:
( kubernetes_config.namespaces is defined and
kubernetes_config.namespaces | default('') | length > 0 and
kubernetes_config.cluster.prime.name == inventory_hostname )
tags:
- install
- namespaces

View File

@ -20,19 +20,31 @@
- install - install
- name: K8s Cluster # kubernetes_installed
ansible.builtin.include_tasks: k8s.yaml
when: kubernetes_type == 'k8s' - name: K3s Install
ansible.builtin.include_tasks:
file: k3s/install.yaml
apply:
tags: tags:
- never - always
- install when: >
install_kubernetes | default(true) | bool
and
not kubernetes_installed | default(false) | bool
tags:
- always
- name: K3s Cluster - name: K3s Configure
ansible.builtin.include_tasks: k3s.yaml ansible.builtin.include_tasks:
when: kubernetes_type == 'k3s' file: k3s/configure.yaml
apply:
tags: tags:
- never - always
- install when: >
- operator_calico install_kubernetes | default(true) | bool
- operator_migrate_calico and
kubernetes_installed | default(false) | bool
tags:
- always

View File

@ -1,146 +0,0 @@
---
- name: initialize Kubernetes cluster
block:
- name: Intilizing Kubernetes Cluster
#command: kubeadm init --pod-network-cidr "{{ KubernetesPodSubnet }}" --apiserver-advertise-address "{{ ansible_default_ipv4.address }}" --ignore-preflight-errors Mem --cri-socket=unix:///var/run/crio/crio.sock
command: kubeadm init --pod-network-cidr "{{ KubernetesPodSubnet }}" --service-cidr "{{ KubernetesServiceSubnet }}" --apiserver-advertise-address "0.0.0.0" --ignore-preflight-errors Mem #--cri-socket=unix:///var/run/containerd/containerd.sock
when:
- not KubernetesInitialized.stat.exists
rescue:
- name: Reset Kubeadmn
ansible.builtin.shell: "{{ item }}"
#register: kube_reset
failed_when: item.rc != 0
with_items:
- kubeadm reset --force
- rm -Rf /etc/cni/net.d
- name: Check if kubernetes has been Initialized
stat:
path: /etc/kubernetes/admin.conf
register: KubernetesInitialized
tags:
- always
- name: fetch kubernetes health
ansible.builtin.shell: " wget http://localhost:10248/healthz -q -O - || true"
register: KubernetesHealth
changed_when: true == false
when: KubernetesInitialized.stat.exists
tags:
- always
- name: set kubeernetes health fact
set_fact:
kube_health: "{{ KubernetesHealth.stdout | default(false) == 'ok' }}"
changed_when: true == false
tags:
- always
- name: Create directory for kube config.
become_method: sudo
become: yes
file:
#path: /home/{{ ansible_user }}/.kube
path: ~/.kube
state: directory
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0700
# when: Kubernetes_Master
tags:
- always
- name: Copy Kube config for local user
copy:
remote_src: yes
src: /etc/kubernetes/admin.conf
#dest: /home/{{ ansible_user }}/.kube/config
dest: ~/.kube/config
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0700
tags:
- always
- name: Add calico networking.
template:
src: "calico.yaml.j2"
dest: /etc/kubernetes/manifests/calico.yaml
owner: root
mode: 0744
- name: apply calico manifest
command: kubectl apply -f /etc/kubernetes/manifests/calico.yaml
tags:
- install
- manifest
- name: create remote workdir
file:
path: "{{ item }}"
state: directory
mode: 0700
with_items:
- /tmp/ansible/
tags:
- always
- name: Create local workdir
file:
path: "{{ item }}"
state: directory
mode: 0700
delegate_to: localhost
connection: local
with_items:
- /tmp/ansible/
tags:
- always
- name: get join command
ansible.builtin.shell: kubeadm token create --print-join-command > /tmp/ansible/join_kubernetes.sh
changed_when: true == false
tags:
- always
- name: download join command
fetch:
src: /tmp/ansible/join_kubernetes.sh
dest: /tmp/ansible/
flat: yes
changed_when: true == false
tags:
- always
# always:
# - name: remove remote workdir
# file:
# path: "{{ item }}"
# state: absent
# with_items:
# - /tmp/ansible/join_kubernetes.sh
# changed_when: true == false
# when:
# #- Kubernetes_Prime
# #- KubernetesInit.stat.exists
# - kubernetes_config.cluster.prime.name == inventory_hostname

View File

@ -1,46 +0,0 @@
---
# - name: configure non-prime nodes - check node health
# shell: "curl http://localhost:10248/healthz || true"
# register: health
# changed_when: true == false
# - set_fact:
# kube_joined: "{{ health.stdout == 'ok' }}"
# changed_when: true == false
# # when:
# # - not Kubernetes_Prime
- name: configure non-prime nodes - create remote workdir
file:
path: "{{ item }}"
state: directory
mode: 0700
changed_when: true == false
with_items:
- /tmp/ansible/
- ansible.builtin.shell: " wget http://localhost:10248/healthz -q -O - || true"
register: health
changed_when: true == false
- set_fact:
kube_joined: "{{ health.stdout == 'ok' }}"
changed_when: true == false
- name: get join command from ansible controller
copy:
src: /tmp/ansible/join_kubernetes.sh
dest: /tmp/ansible/join_kubernetes.sh
mode: 0700
changed_when: true == false
when:
- not kube_joined
- name: configure non-prime nodes - join node to kubernetes cluster
command: sh /tmp/ansible/join_kubernetes.sh
when:
- not kube_joined

View File

@ -1,10 +0,0 @@
#
# {{ item.name }} Container Registry Configuration
# Managed by: Ansible
#
server = "{{ item.server }}"
[host."{{ item.url }}"]
capabilities = {{ item.capabilities | from_yaml_all }}
skip_verify = {{ item.skip_verify | default(false) | lower }}

View File

@ -1,250 +0,0 @@
disabled_plugins = []
imports = []
oom_score = 0
plugin_dir = ""
required_plugins = []
root = "/var/lib/containerd"
state = "/run/containerd"
temp = ""
version = 2
[cgroup]
path = ""
[debug]
address = ""
format = ""
gid = 0
level = ""
uid = 0
[grpc]
address = "/run/containerd/containerd.sock"
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
tcp_address = ""
tcp_tls_ca = ""
tcp_tls_cert = ""
tcp_tls_key = ""
uid = 0
[metrics]
address = ""
grpc_histogram = false
[plugins]
[plugins."io.containerd.gc.v1.scheduler"]
deletion_threshold = 0
mutation_threshold = 100
pause_threshold = 0.02
schedule_delay = "0s"
startup_delay = "100ms"
[plugins."io.containerd.grpc.v1.cri"]
device_ownership_from_security_context = false
disable_apparmor = false
disable_cgroup = false
disable_hugetlb_controller = true
disable_proc_mount = false
disable_tcp_service = true
enable_selinux = false
enable_tls_streaming = false
enable_unprivileged_icmp = false
enable_unprivileged_ports = false
ignore_image_defined_volumes = false
max_concurrent_downloads = 3
max_container_log_line_size = 16384
netns_mounts_under_state_dir = false
restrict_oom_score_adj = false
sandbox_image = "registry.k8s.io/pause:3.6"
selinux_category_range = 1024
stats_collect_period = 10
stream_idle_timeout = "4h0m0s"
stream_server_address = "127.0.0.1"
stream_server_port = "0"
systemd_cgroup = false
tolerate_missing_hugetlb_controller = true
unset_seccomp_profile = ""
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
conf_template = ""
ip_pref = ""
max_conf_num = 1
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
disable_snapshot_annotations = true
discard_unpacked_layers = false
ignore_rdt_not_enabled_errors = false
no_pivot = false
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = ""
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
BinaryName = ""
CriuImagePath = ""
CriuPath = ""
CriuWorkPath = ""
IoGid = 0
IoUid = 0
NoNewKeyring = false
NoPivotRoot = false
Root = ""
ShimCgroup = ""
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = ""
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]
[plugins."io.containerd.grpc.v1.cri".image_decryption]
key_model = "node"
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
[plugins."io.containerd.grpc.v1.cri".registry.auths]
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.headers]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""
[plugins."io.containerd.internal.v1.opt"]
path = "/opt/containerd"
[plugins."io.containerd.internal.v1.restart"]
interval = "10s"
[plugins."io.containerd.internal.v1.tracing"]
sampling_ratio = 1.0
service_name = "containerd"
[plugins."io.containerd.metadata.v1.bolt"]
content_sharing_policy = "shared"
[plugins."io.containerd.monitor.v1.cgroups"]
no_prometheus = false
[plugins."io.containerd.runtime.v1.linux"]
no_shim = false
runtime = "runc"
runtime_root = ""
shim = "containerd-shim"
shim_debug = false
[plugins."io.containerd.runtime.v2.task"]
platforms = ["linux/amd64"]
sched_core = false
[plugins."io.containerd.service.v1.diff-service"]
default = ["walking"]
[plugins."io.containerd.service.v1.tasks-service"]
rdt_config_file = ""
[plugins."io.containerd.snapshotter.v1.aufs"]
root_path = ""
[plugins."io.containerd.snapshotter.v1.btrfs"]
root_path = ""
[plugins."io.containerd.snapshotter.v1.devmapper"]
async_remove = false
base_image_size = ""
discard_blocks = false
fs_options = ""
fs_type = ""
pool_name = ""
root_path = ""
[plugins."io.containerd.snapshotter.v1.native"]
root_path = ""
[plugins."io.containerd.snapshotter.v1.overlayfs"]
root_path = ""
upperdir_label = false
[plugins."io.containerd.snapshotter.v1.zfs"]
root_path = ""
[plugins."io.containerd.tracing.processor.v1.otlp"]
endpoint = ""
insecure = false
protocol = ""
[proxy_plugins]
[stream_processors]
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar"
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar+gzip"
[timeouts]
"io.containerd.timeout.bolt.open" = "0s"
"io.containerd.timeout.shim.cleanup" = "5s"
"io.containerd.timeout.shim.load" = "5s"
"io.containerd.timeout.shim.shutdown" = "3s"
"io.containerd.timeout.task.state" = "2s"
[ttrpc]
address = ""
gid = 0
uid = 0

View File

@ -1,2 +0,0 @@
overlay
br_netfilter