chore: migrated from internal repo

!1 nofusscomputing/infrastructure/config!28
This commit is contained in:
2023-10-27 21:47:03 +09:30
parent e45190fab4
commit 93b63308ef
30 changed files with 7326 additions and 0 deletions

318
tasks/common.yaml Normal file
View File

@ -0,0 +1,318 @@
---
- name: "{{ role_name }} Install Software"
include_role:
name: nfc_common
vars:
common_gather_facts: false
aptSigningKeys:
- name: docker
url: https://download.docker.com/linux/debian/gpg
save_directory: /usr/share/keyrings
file_extension: asc
- name: kubernetes
url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
save_directory: /usr/share/keyrings
file_extension: asc
aptRepositories:
- name: docker
repo: deb [arch={{ dynamic_processor_architecture }} signed-by=/usr/share/keyrings/docker.asc] http://download.docker.com/linux/{{ ansible_os_family | lower }} {{ ansible_lsb.codename | lower }} stable
- name: kubernetes
repo: deb [signed-by=/usr/share/keyrings/kubernetes.asc] http://apt.kubernetes.io/ kubernetes-xenial main
aptInstall:
- name: gnupg2
- name: apt-transport-https
- name: software-properties-common
- name: ca-certificates
- name: iptables
- name: python3-pip
- name: python3-virtualenv
- name: containerd.io
version: "{{ ContainerDioVersion }}"
- name: kubectl
version: "{{ KubernetesVersion }}"
- name: kubelet
version: "{{ KubernetesVersion }}"
- name: kubeadm
version: "{{ KubernetesVersion }}"
tags:
- install
# containerd.io=1.6.22-1 kubectl=1.26.9-00 kubelet=1.26.9-00 kubeadm=1.26.9-00
- name: Remove swapfile from /etc/fstab
mount:
name: "{{ item }}"
fstype: swap
state: absent
with_items:
- swap
- none
when:
- ansible_os_family == 'Debian' # ansible_lsb.codename = bullseye, ansible_lsb.major_release = 11
tags:
- install
- name: Disable swap
command: swapoff -a
changed_when: true == false
when:
#- ansible_swaptotal_mb > 0
- ansible_os_family == 'Debian'
tags:
- install
- name: Check an armbian os system
stat:
path: /etc/default/armbian-zram-config
register: armbian_stat_result
- name: Armbian Disable Swap
ansible.builtin.shell:
cmd: |
sed -i 's/\# SWAP=false/SWAP=false/g' /etc/default/armbian-zram-config;
sed -i 's/ENABLED=true/ENABLED=false/g' /etc/default/armbian-zram-config;
args:
executable: bash
changed_when: false
# failed_when: false
#notify: RebootHost # doesnt need to reboot as swapoff -a covers the deployment
when: armbian_stat_result.stat.exists
- name: Add the overlay module
community.general.modprobe:
name: overlay
state: present
when:
- ansible_os_family == 'Debian'
tags:
- install
- name: Add the br_netfilter module
community.general.modprobe:
name: br_netfilter
state: present
when:
- ansible_os_family == 'Debian'
tags:
- install
- name: check if containerd installed
ansible.builtin.shell:
cmd: which containerd
failed_when: false
changed_when: false
register: containerd_installed
- name: "Containerd.io Started?"
service:
name: containerd
state: started
tags:
- configure
- install
when: >
ansible_os_family == 'Debian'
and
containerd_installed.rc | default(1) | int == 0
- name: containerd load modules config
template:
src: "etc_module_containerd.conf"
dest: /etc/modules-load.d/containerd.conf
owner: root
mode: 0700
notify: "restart ContainerD"
when:
- ansible_os_family == 'Debian'
tags:
- install
- name: Create containerD host directories.
become_method: sudo
become: yes
file:
path: /etc/containerd/certs.d/{{ item.name }}
state: directory
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0700
with_items: "{{ containerd.repositories }}"
tags:
- install
- containerRegistry
- name: containerD registry host
template:
src: "containerd-registry-hosts.toml.j2"
dest: /etc/containerd/certs.d/{{ item.name }}/hosts.toml
owner: root
mode: 0700
notify: "restart ContainerD"
with_items: "{{ containerd.repositories }}"
when:
- ansible_os_family == 'Debian'
tags:
- install
- containerRegistry
- name: containerD default config
template:
src: "etc_containerd_containerd.toml"
dest: /etc/containerd/config.toml
owner: root
mode: 0700
notify: "restart ContainerD"
register: containerd_config
when:
- ansible_os_family == 'Debian'
tags:
- install
- containerRegistry
- name: Restart ContainerD if required
meta: flush_handlers
tags:
- install
- containerRegistry
- name: Install required python modules
ansible.builtin.pip:
name: kubernetes
state: forcereinstall
#virtualenv: /tmp/venv_ansible
when: inventory_hostname != 'op1'
tags:
- install
- name: sysctl net.bridge.bridge-nf-call-ip6tables
sysctl:
name: net.bridge.bridge-nf-call-ip6tables
value: '1'
sysctl_set: yes
state: present
reload: yes
when:
- ansible_os_family == 'Debian'
tags:
- install
- name: sysctl net.bridge.bridge-nf-call-iptables
sysctl:
name: net.bridge.bridge-nf-call-iptables
value: '1'
sysctl_set: yes
state: present
reload: yes
when:
- ansible_os_family == 'Debian'
tags:
- install
- name: sysctl net.ipv4.ip_forward
sysctl:
name: net.ipv4.ip_forward
value: '1'
sysctl_set: yes
state: present
reload: yes
when:
- ansible_os_family == 'Debian'
tags:
- install
# - name: Check if kubernetes has been Initialized
# stat:
# path: /etc/kubernetes/admin.conf
# register: KubernetesInit
# when:
# - kubernetes_config.cluster.prime.name == inventory_hostname
- name: check if iptables is installed
ansible.builtin.shell: |-
dpkg -s iptables &> /dev/null
changed_when: true == false
register: iptables_installed
when:
- ansible_os_family == 'Debian'
tags:
- install
- iptables
- firewall
- name: Add kubernetes Firewall Rules - '/etc/iptables-kubernetes.rules'
template:
src: iptables-kubernetes.rules.j2
dest: "/etc/iptables-kubernetes.rules"
owner: root
mode: 0700
force: yes
notify: "Apply Firewall Rules"
when:
- ansible_os_family == 'Debian'
- iptables_installed.rc == 0
tags:
- install
- iptables
- firewall
- name: File - '/etc/network/if-pre-up.d/firewall-kubernetes'
template:
src: firewall-kubernetes.j2
dest: "/etc/network/if-pre-up.d/firewall-kubernetes"
owner: root
mode: 0700
force: yes
when:
- ansible_os_family == 'Debian'
- iptables_installed.rc == 0
tags:
- install
- iptables
- firewall
- name: Apply new firewall rules, if required
meta: flush_handlers
tags:
- install
- iptables
- firewall
- name: Create local workdir
file:
path: "{{ item }}"
state: directory
mode: 0700
delegate_to: localhost
connection: local
run_once: true
changed_when: true == false
with_items:
- /tmp/ansible/
tags:
- always

241
tasks/k3s.yaml Normal file
View File

@ -0,0 +1,241 @@
---
- name: Install Software
ansible.builtin.include_role:
name: nfc_common
vars:
common_gather_facts: false
aptInstall:
- name: curl
- name: iptables
- name: Create Required directories
ansible.builtin.file:
name: "{{ item.name }}"
state: "{{ item.state }}"
mode: "{{ item.mode }}"
loop: "{{ dirs }}"
vars:
dirs:
- name: /etc/rancher/k3s
state: directory
mode: 700
- name: /var/lib/rancher/k3s/server/logs
state: directory
mode: 700
- name: /var/lib/rancher/k3s/server/manifests
state: directory
mode: 700
# - name: Local Container Registry
# ansible.builtin.copy:
# content: |
# #
# # Private Container Registries for Kubernetes
# #
# # Managed By ansible/role/nfc_kubernetes
# #
# # Dont edit this file directly as it will be overwritten.
# #
# {% set registries = kubernetes_private_container_registry | default([]) -%}
# {% if registries | length > 0 %}mirrors:
# {% for entry in registries %}
# {{ entry.name }}:
# endpoint:
# - "{{ entry.url }}"
# {%- endfor %}
# {% endif %}
# dest: /etc/rancher/k3s/registries.yaml
# owner: root
# mode: '700'
# # notify: "restart ContainerD"
# # with_items: "{{ containerd.repositories }}"
# # when:
# # ansible_os_family == 'Debian'
# # and
# # Kubernetes_private_container_registry | default([]) | length > 0
- name: Add sysctl net.ipv4.ip_forward
sysctl:
name: net.ipv4.ip_forward
value: '1'
sysctl_set: true
state: present
reload: true
notify: reboot_host
when:
- ansible_os_family == 'Debian'
# On change reboot
- name: Check if K3s Installed
ansible.builtin.shell:
cmd: |
if [[ $(service k3s status) ]]; then exit 0; else exit 1; fi
executable: /bin/bash
changed_when: false
failed_when: false
register: k3s_installed
# - name: Download K3s Binary
# ansible.builtin.uri:
# url: "{{ item.url }}"
# method: GET
# return_content: true
# # body: ""
# status_code:
# - 200
# - 304
# # headers:
# # App-Token: "{{ glpi.app_token }}"
# # Authorization: "user_token {{ glpi.user_token }}"
# #body_format: json
# # validate_certs: false
# dest: "{{ item.dest }}"
# mode: "744"
# register: k3s_download_files
# delegate_to: localhost
# # no_log: true
# when: ansible_os_family == 'Debian'
# loop: "{{ download_files }}"
# vars:
# ansible_connection: local
# download_files:
# - dest: /tmp/install.sh
# url: https://get.k3s.io
# - dest: "/tmp/k3s"
# url: "https://github.com/k3s-io/k3s/releases/download/v{{ KubernetesVersion + KubernetesVersion_k3s_prefix | urlencode }}/k3s"
# - name: "[TRACE] Downloaded File SHA256"
# ansible.builtin.set_fact:
# hash_sha256_k3s_downloaded_binary: "{{ lookup('ansible.builtin.file', '/tmp/k3s') | hash('sha256') | string }}"
# delegate_to: localhost
# - name: Existing k3s File hash
# ansible.builtin.stat:
# checksum_algorithm: sha256
# name: /usr/local/bin/k3s
# register: hash_sha256_k3s_existing_binary
# - name: Copy K3s binary to Host
# ansible.builtin.copy:
# src: "/tmp/k3s"
# dest: "/usr/local/bin/k3s"
# mode: '740'
# owner: root
# group: root
# when: hash_sha256_k3s_existing_binary.stat.checksum | default('0') != hash_sha256_k3s_downloaded_binary
# - name: Copy install script to Host
# ansible.builtin.copy:
# src: "/tmp/install.sh"
# dest: "/tmp/install.sh"
# mode: '755'
# owner: root
# group: root
# # when: hash_sha256_k3s_existing_binary.stat.checksum | default('0') != hash_sha256_k3s_downloaded_binary
- name: Additional config files
ansible.builtin.copy:
content: |
{{ item.content }}
dest: "{{ item.path }}/{{ item.name }}"
mode: '740'
owner: root
group: root
loop: "{{ k3s.files }}"
- name: Copy Templates
ansible.builtin.template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: root
mode: '700'
force: true
notify: "{{ item.notify | default(omit) }}"
loop: "{{ templates_to_apply }}"
vars:
templates_to_apply:
- src: "calico.yaml.j2"
dest: /var/lib/rancher/k3s/server/manifests/calico.yaml
- src: kubernetes-manifest-rbac.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/rbac-authorization-common.yaml
# - src: firewall-kubernetes.j2
# dest: "/etc/network/if-up.d/firewall-kubernetes"
- src: iptables-kubernetes.rules.j2
dest: "/etc/iptables.rules.d/iptables-kubernetes.rules"
notify: firewall_reloader
- src: k3s-registries.yaml.j2
dest: /etc/rancher/k3s/registries.yaml
notify: kubernetes_restart
- src: k3s-config.yaml.j2
dest: /etc/rancher/k3s/config.yaml
notify: kubernetes_restart
# - name: Templates IPv6
# ansible.builtin.template:
# src: iptables-kubernetes.rules.j2
# dest: "/etc/ip6tables.rules.d/ip6tables-kubernetes.rules"
# owner: root
# mode: '700'
# force: true
# vars:
# ipv6: true
# # - name: Set IPTables to legacy mode
# # ansible.builtin.command:
# # cmd: update-alternatives --set iptables /usr/sbin/iptables-legacy
# # changed_when: false
# - name: Server install K3s
# ansible.builtin.shell:
# cmd: |
# # INSTALL_K3S_SKIP_DOWNLOAD=true \
# # INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
# # /tmp/install.sh
# curl -sfL https://get.k3s.io | \
# INSTALL_K3S_VERSION="v1.26.9+k3s1" \
# sh -
# failed_when: false
# # when: >
# # k3s_installed.rc | int == 1
# # and
# # Kubernetes_Master | default(false)
# when: Kubernetes_Master | default(false)
# - name: Agent install K3s
# ansible.builtin.shell:
# cmd: |
# INSTALL_K3S_SKIP_DOWNLOAD=true \
# INSTALL_K3S_VERSION="v{{ KubernetesVersion }}{{ KubernetesVersion_k3s_prefix }}" \
# K3S_URL=https://{{ hostvars[kubernetes_config.cluster.prime.name].ansible_host }}:6443 \
# K3S_TOKEN={{ node_token }} \
# /tmp/install.sh
# when: >
# k3s_installed.rc | int == 1
# and
# not Kubernetes_Master | default(false)
# # - name: Look up AAAA (IPv4) records for example.org
# # ansible.builtin.debug:
# # msg: "{{ query('community.dns.lookup', 'nww-au1.networkedweb.com.', type='A') }}"
# # - name: Look up AAAA (IPv6) records for example.org
# # ansible.builtin.debug:
# # msg: "{{ query('community.dns.lookup', 'nww-au1.networkedweb.com.', type='AAAA') }}"

103
tasks/k8s.yaml Normal file
View File

@ -0,0 +1,103 @@
---
- name: Common Tasks
include_tasks: common.yaml
# tags:
# - install
- name: Check if kubernetes has been Initialized
stat:
path: /etc/kubernetes/admin.conf
register: KubernetesInitialized
tags:
- always
- name: kubernetes prime
include_tasks: prime.yaml
when: kubernetes_config.cluster.prime.name == inventory_hostname
- name: kubernetes workers
include_tasks: workers.yaml
when: kubernetes_config.cluster.prime.name != inventory_hostname
- name: Add Kubernetes Node Labels
kubernetes.core.k8s:
definition:
apiVersion: v1
kind: Node
metadata:
name: "{{ inventory_hostname }}"
labels:
"{{ item | from_yaml_all }}"
delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
with_items:
- "{{ kubernetes_config.hosts[inventory_hostname].labels }}"
when:
- ( kubernetes_config.hosts[inventory_hostname].labels is defined and
kubernetes_config.hosts[inventory_hostname].labels|default('')|length > 0 )
tags:
- install
- nodelabels
- name: Add Node Taints
kubernetes.core.k8s_taint:
state: "present"
name: "{{ inventory_hostname }}"
taints:
- "{{ item | from_yaml_all }}"
delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
with_items:
- "{{ kubernetes_config.hosts[inventory_hostname].taints.present }}"
when:
- (kubernetes_config.hosts[inventory_hostname].taints.present is defined and
kubernetes_config.hosts[inventory_hostname].taints.present|default('')|length > 0 )
tags:
- install
- taints
- name: Remove Node Taints
kubernetes.core.k8s_taint:
state: "absent"
name: "{{ inventory_hostname }}"
taints:
- "{{ item | from_yaml_all }}"
delegate_to: "{{ kubernetes_config.cluster.prime.name }}"
with_items:
- "{{ kubernetes_config.hosts[inventory_hostname].taints.absent }}"
when:
- ( kubernetes_config.hosts[inventory_hostname].taints.absent is defined and
kubernetes_config.hosts[inventory_hostname].taints.absent|default('')|length > 0 )
tags:
- install
- taints
- name: Create Cluster Namespaces
kubernetes.core.k8s:
state: present
definition:
apiVersion: v1
kind: Namespace
metadata:
name: "{{ item.name }}"
labels:
#app.kubernetes.io/version: # App version
#app.kubernetes.io/component:
#app.kubernetes.io/part-of:
app.kubernetes.io/managed-by: Ansible
#meta.kubernetes.io/description: "{{ item.description | default('') }}"
meta.kubernetes.io/version: "{{ deployment_git_current_short_hash | default('') }}"
with_items:
- "{{ kubernetes_config.namespaces }}"
when:
( kubernetes_config.namespaces is defined and
kubernetes_config.namespaces | default('') | length > 0 and
kubernetes_config.cluster.prime.name == inventory_hostname )
tags:
- install
- namespaces

14
tasks/main.yml Normal file
View File

@ -0,0 +1,14 @@
---
- name: Firewall Rules
ansible.builtin.include_role:
name: nfc_firewall
vars:
nfc_firewall_enabled_kubernetes: "{{ nfc_kubernetes.enable_firewall | default(false) | bool }}"
- name: K8s Cluster
ansible.builtin.include_tasks: k8s.yaml
when: kubernetes_type == 'k8s'
- name: K3s Cluster
ansible.builtin.include_tasks: k3s.yaml
when: kubernetes_type == 'k3s'

146
tasks/prime.yaml Normal file
View File

@ -0,0 +1,146 @@
---
- name: initialize Kubernetes cluster
block:
- name: Intilizing Kubernetes Cluster
#command: kubeadm init --pod-network-cidr "{{ KubernetesPodSubnet }}" --apiserver-advertise-address "{{ ansible_default_ipv4.address }}" --ignore-preflight-errors Mem --cri-socket=unix:///var/run/crio/crio.sock
command: kubeadm init --pod-network-cidr "{{ KubernetesPodSubnet }}" --service-cidr "{{ KubernetesServiceSubnet }}" --apiserver-advertise-address "0.0.0.0" --ignore-preflight-errors Mem #--cri-socket=unix:///var/run/containerd/containerd.sock
when:
- not KubernetesInitialized.stat.exists
rescue:
- name: Reset Kubeadmn
ansible.builtin.shell: "{{ item }}"
#register: kube_reset
failed_when: item.rc != 0
with_items:
- kubeadm reset --force
- rm -Rf /etc/cni/net.d
- name: Check if kubernetes has been Initialized
stat:
path: /etc/kubernetes/admin.conf
register: KubernetesInitialized
tags:
- always
- name: fetch kubernetes health
ansible.builtin.shell: " wget http://localhost:10248/healthz -q -O - || true"
register: KubernetesHealth
changed_when: true == false
when: KubernetesInitialized.stat.exists
tags:
- always
- name: set kubeernetes health fact
set_fact:
kube_health: "{{ KubernetesHealth.stdout | default(false) == 'ok' }}"
changed_when: true == false
tags:
- always
- name: Create directory for kube config.
become_method: sudo
become: yes
file:
#path: /home/{{ ansible_user }}/.kube
path: ~/.kube
state: directory
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0700
# when: Kubernetes_Master
tags:
- always
- name: Copy Kube config for local user
copy:
remote_src: yes
src: /etc/kubernetes/admin.conf
#dest: /home/{{ ansible_user }}/.kube/config
dest: ~/.kube/config
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0700
tags:
- always
- name: Add calico networking.
template:
src: "calico.yaml.j2"
dest: /etc/kubernetes/manifests/calico.yaml
owner: root
mode: 0744
- name: apply calico manifest
command: kubectl apply -f /etc/kubernetes/manifests/calico.yaml
tags:
- install
- manifest
- name: create remote workdir
file:
path: "{{ item }}"
state: directory
mode: 0700
with_items:
- /tmp/ansible/
tags:
- always
- name: Create local workdir
file:
path: "{{ item }}"
state: directory
mode: 0700
delegate_to: localhost
connection: local
with_items:
- /tmp/ansible/
tags:
- always
- name: get join command
ansible.builtin.shell: kubeadm token create --print-join-command > /tmp/ansible/join_kubernetes.sh
changed_when: true == false
tags:
- always
- name: download join command
fetch:
src: /tmp/ansible/join_kubernetes.sh
dest: /tmp/ansible/
flat: yes
changed_when: true == false
tags:
- always
# always:
# - name: remove remote workdir
# file:
# path: "{{ item }}"
# state: absent
# with_items:
# - /tmp/ansible/join_kubernetes.sh
# changed_when: true == false
# when:
# #- Kubernetes_Prime
# #- KubernetesInit.stat.exists
# - kubernetes_config.cluster.prime.name == inventory_hostname

46
tasks/workers.yaml Normal file
View File

@ -0,0 +1,46 @@
---
# - name: configure non-prime nodes - check node health
# shell: "curl http://localhost:10248/healthz || true"
# register: health
# changed_when: true == false
# - set_fact:
# kube_joined: "{{ health.stdout == 'ok' }}"
# changed_when: true == false
# # when:
# # - not Kubernetes_Prime
- name: configure non-prime nodes - create remote workdir
file:
path: "{{ item }}"
state: directory
mode: 0700
changed_when: true == false
with_items:
- /tmp/ansible/
- ansible.builtin.shell: " wget http://localhost:10248/healthz -q -O - || true"
register: health
changed_when: true == false
- set_fact:
kube_joined: "{{ health.stdout == 'ok' }}"
changed_when: true == false
- name: get join command from ansible controller
copy:
src: /tmp/ansible/join_kubernetes.sh
dest: /tmp/ansible/join_kubernetes.sh
mode: 0700
changed_when: true == false
when:
- not kube_joined
- name: configure non-prime nodes - join node to kubernetes cluster
command: sh /tmp/ansible/join_kubernetes.sh
when:
- not kube_joined