From 6b4e94c236306d6f1e98904bec7e7b754cbcf03f Mon Sep 17 00:00:00 2001
From: Sam Schmit-Van Werweke <sam@schm.it>
Date: Sun, 2 Apr 2023 16:26:15 +0200
Subject: [PATCH] Transition to k3s installation and uninstall scripts

closes #108
---
 roles/download/tasks/main.yml                 | 36 -------
 roles/k3s/master/defaults/main.yml            |  1 -
 ...h_k3s_init_logs.yml => fetch_k3s_logs.yml} | 12 +--
 roles/k3s/master/tasks/main.yml               | 98 +++++--------------
 roles/k3s/master/templates/k3s.service.j2     | 24 -----
 roles/k3s/node/tasks/main.yml                 | 35 ++++---
 roles/k3s/node/templates/k3s.service.j2       | 24 -----
 roles/reset/tasks/main.yml                    | 57 +++--------
 roles/reset/tasks/umount_with_children.yml    | 16 ---
 site.yml                                      |  2 -
 10 files changed, 64 insertions(+), 241 deletions(-)
 delete mode 100644 roles/download/tasks/main.yml
 rename roles/k3s/master/tasks/{fetch_k3s_init_logs.yml => fetch_k3s_logs.yml} (60%)
 delete mode 100644 roles/k3s/master/templates/k3s.service.j2
 delete mode 100644 roles/k3s/node/templates/k3s.service.j2
 delete mode 100644 roles/reset/tasks/umount_with_children.yml

diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml
deleted file mode 100644
index 1450fd86e..000000000
--- a/roles/download/tasks/main.yml
+++ /dev/null
@@ -1,36 +0,0 @@
----
-
-- name: Download k3s binary x64
-  get_url:
-    url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s
-    checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt
-    dest: /usr/local/bin/k3s
-    owner: root
-    group: root
-    mode: 0755
-  when: ansible_facts.architecture == "x86_64"
-
-- name: Download k3s binary arm64
-  get_url:
-    url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-arm64
-    checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm64.txt
-    dest: /usr/local/bin/k3s
-    owner: root
-    group: root
-    mode: 0755
-  when:
-    - ( ansible_facts.architecture is search("arm") and
-        ansible_facts.userspace_bits == "64" ) or
-      ansible_facts.architecture is search("aarch64")
-
-- name: Download k3s binary armhf
-  get_url:
-    url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-armhf
-    checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm.txt
-    dest: /usr/local/bin/k3s
-    owner: root
-    group: root
-    mode: 0755
-  when:
-    - ansible_facts.architecture is search("arm")
-    - ansible_facts.userspace_bits == "32"
diff --git a/roles/k3s/master/defaults/main.yml b/roles/k3s/master/defaults/main.yml
index 9e2fe633e..8cc13abf1 100644
--- a/roles/k3s/master/defaults/main.yml
+++ b/roles/k3s/master/defaults/main.yml
@@ -11,6 +11,5 @@ server_init_args: >-
     {% else %}
       --server https://{{ hostvars[groups['master'][0]].k3s_node_ip | split(",") | first | ansible.utils.ipwrap }}:6443
     {% endif %}
-    --token {{ k3s_token }}
   {% endif %}
   {{ extra_server_args | default('') }}
diff --git a/roles/k3s/master/tasks/fetch_k3s_init_logs.yml b/roles/k3s/master/tasks/fetch_k3s_logs.yml
similarity index 60%
rename from roles/k3s/master/tasks/fetch_k3s_init_logs.yml
rename to roles/k3s/master/tasks/fetch_k3s_logs.yml
index 060f26a89..b8476cad4 100644
--- a/roles/k3s/master/tasks/fetch_k3s_init_logs.yml
+++ b/roles/k3s/master/tasks/fetch_k3s_logs.yml
@@ -1,12 +1,12 @@
 ---
-# Download logs of k3s-init.service from the nodes to localhost.
+# Download logs of k3s.service from the nodes to localhost.
 # Note that log_destination must be set.
 
-- name: Fetch k3s-init.service logs
+- name: Fetch k3s.service logs
   ansible.builtin.command:
-    cmd: journalctl --all --unit=k3s-init.service
+    cmd: journalctl --all --unit=k3s.service
   changed_when: false
-  register: k3s_init_log
+  register: k3s_log
 
 - name: Create {{ log_destination }}
   delegate_to: localhost
@@ -22,7 +22,7 @@
   become: false
   ansible.builtin.template:
     src: content.j2
-    dest: "{{ log_destination }}/k3s-init@{{ ansible_hostname }}.log"
+    dest: "{{ log_destination }}/k3s@{{ ansible_hostname }}.log"
     mode: 0644
   vars:
-    content: "{{ k3s_init_log.stdout }}"
+    content: "{{ k3s_log.stdout }}"
diff --git a/roles/k3s/master/tasks/main.yml b/roles/k3s/master/tasks/main.yml
index 76b910c39..18ee6ad79 100644
--- a/roles/k3s/master/tasks/main.yml
+++ b/roles/k3s/master/tasks/main.yml
@@ -1,35 +1,42 @@
 ---
 
-- name: Clean previous runs of k3s-init
+- name: Stop previous runs of k3s
   systemd:
-    name: k3s-init
+    name: k3s
     state: stopped
   failed_when: false
 
-- name: Clean previous runs of k3s-init
-  command: systemctl reset-failed k3s-init
+- name: Clean previous runs of k3s
+  command: systemctl reset-failed k3s
   failed_when: false
   changed_when: false
   args:
     warn: false  # The ansible systemd module does not support reset-failed
 
+- name: Download install script
+  ansible.builtin.get_url:
+    url: https://get.k3s.io
+    dest: "{{ ansible_user_dir }}/install_k3s.sh"
+    mode: 744
+
+- name: Install k3s
+  command:
+    cmd: "./install_k3s.sh {{ server_init_args }}"
+    creates: "{{ systemd_dir }}/k3s.service"
+  environment:
+    INSTALL_K3S_VERSION: "{{ k3s_version }}"
+    K3S_TOKEN: "{{ k3s_token }}"
+  failed_when: false
+
 - name: Deploy vip manifest
   include_tasks: vip.yml
 
 - name: Deploy metallb manifest
   include_tasks: metallb.yml
 
-- name: Init cluster inside the transient k3s-init service
-  command:
-    cmd: "systemd-run -p RestartSec=2 \
-                      -p Restart=on-failure \
-                      --unit=k3s-init \
-                      k3s server {{ server_init_args }}"
-    creates: "{{ systemd_dir }}/k3s.service"
-
 - name: Verification
   block:
-    - name: Verify that all nodes actually joined (check k3s-init.service if this fails)
+    - name: Verify that all nodes actually joined (check k3s.service if this fails)
       command:
         cmd: k3s kubectl get nodes -l "node-role.kubernetes.io/master=true" -o=jsonpath="{.items[*].metadata.name}"
       register: nodes
@@ -38,63 +45,14 @@
       delay: 10
       changed_when: false
   always:
-    - name: Save logs of k3s-init.service
-      include_tasks: fetch_k3s_init_logs.yml
+    - name: Save logs of k3s.service
+      include_tasks: fetch_k3s_logs.yml
       when: log_destination
       vars:
         log_destination: >-
           {{ lookup('ansible.builtin.env', 'ANSIBLE_K3S_LOG_DIR', default=False) }}
-    - name: Kill the temporary service used for initialization
-      systemd:
-        name: k3s-init
-        state: stopped
-      failed_when: false
   when: not ansible_check_mode
 
-- name: Copy K3s service file
-  register: k3s_service
-  template:
-    src: "k3s.service.j2"
-    dest: "{{ systemd_dir }}/k3s.service"
-    owner: root
-    group: root
-    mode: 0644
-
-- name: Enable and check K3s service
-  systemd:
-    name: k3s
-    daemon_reload: yes
-    state: restarted
-    enabled: yes
-
-- name: Wait for node-token
-  wait_for:
-    path: /var/lib/rancher/k3s/server/node-token
-
-- name: Register node-token file access mode
-  stat:
-    path: /var/lib/rancher/k3s/server
-  register: p
-
-- name: Change file access node-token
-  file:
-    path: /var/lib/rancher/k3s/server
-    mode: "g+rx,o+rx"
-
-- name: Read node-token from master
-  slurp:
-    src: /var/lib/rancher/k3s/server/node-token
-  register: node_token
-
-- name: Store Master node-token
-  set_fact:
-    token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}"
-
-- name: Restore node-token file access
-  file:
-    path: /var/lib/rancher/k3s/server
-    mode: "{{ p.stat.mode }}"
-
 - name: Create directory .kube
   file:
     path: "{{ ansible_user_dir }}/.kube"
@@ -124,18 +82,6 @@
   #                     would be undefined. This will not be the case during playbook execution.
   # noqa jinja[invalid]
 
-- name: Create kubectl symlink
-  file:
-    src: /usr/local/bin/k3s
-    dest: /usr/local/bin/kubectl
-    state: link
-
-- name: Create crictl symlink
-  file:
-    src: /usr/local/bin/k3s
-    dest: /usr/local/bin/crictl
-    state: link
-
 - name: Get contents of manifests folder
   find:
     paths: /var/lib/rancher/k3s/server/manifests
diff --git a/roles/k3s/master/templates/k3s.service.j2 b/roles/k3s/master/templates/k3s.service.j2
deleted file mode 100644
index ae5cb484e..000000000
--- a/roles/k3s/master/templates/k3s.service.j2
+++ /dev/null
@@ -1,24 +0,0 @@
-[Unit]
-Description=Lightweight Kubernetes
-Documentation=https://k3s.io
-After=network-online.target
-
-[Service]
-Type=notify
-ExecStartPre=-/sbin/modprobe br_netfilter
-ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s server {{ extra_server_args | default("") }}
-KillMode=process
-Delegate=yes
-# Having non-zero Limit*s causes performance problems due to accounting overhead
-# in the kernel. We recommend using cgroups to do container-local accounting.
-LimitNOFILE=1048576
-LimitNPROC=infinity
-LimitCORE=infinity
-TasksMax=infinity
-TimeoutStartSec=0
-Restart=always
-RestartSec=5s
-
-[Install]
-WantedBy=multi-user.target
diff --git a/roles/k3s/node/tasks/main.yml b/roles/k3s/node/tasks/main.yml
index 0ce8e08d0..291a83792 100644
--- a/roles/k3s/node/tasks/main.yml
+++ b/roles/k3s/node/tasks/main.yml
@@ -1,16 +1,25 @@
 ---
 
-- name: Copy K3s service file
-  template:
-    src: "k3s.service.j2"
-    dest: "{{ systemd_dir }}/k3s-node.service"
-    owner: root
-    group: root
-    mode: 0755
+- name: Download install script
+  ansible.builtin.get_url:
+    url: https://get.k3s.io
+    dest: "{{ ansible_user_dir }}/install_k3s.sh"
+    mode: 744
 
-- name: Enable and check K3s service
-  systemd:
-    name: k3s-node
-    daemon_reload: yes
-    state: restarted
-    enabled: yes
+- name: Install k3s
+  command:
+    cmd: "./install_k3s.sh {{ extra_agent_args | default('') }}"
+    creates: "{{ systemd_dir }}/k3s.service"
+  environment:
+    INSTALL_K3S_VERSION: "{{ k3s_version }}"
+    K3S_TOKEN: "{{ k3s_token }}"
+    K3S_URL: https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443
+
+- name: Ensure k3s is in a running state
+  service:
+    name: k3s-agent
+    state: started
+  register: k3sAgentServiceDetails
+  until: k3sAgentServiceDetails.status.ActiveState == "active"
+  retries: 20
+  delay: 10
\ No newline at end of file
diff --git a/roles/k3s/node/templates/k3s.service.j2 b/roles/k3s/node/templates/k3s.service.j2
deleted file mode 100644
index 67abadbb5..000000000
--- a/roles/k3s/node/templates/k3s.service.j2
+++ /dev/null
@@ -1,24 +0,0 @@
-[Unit]
-Description=Lightweight Kubernetes
-Documentation=https://k3s.io
-After=network-online.target
-
-[Service]
-Type=notify
-ExecStartPre=-/sbin/modprobe br_netfilter
-ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s agent --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 --token {{ hostvars[groups['master'][0]]['token'] | default(k3s_token) }} {{ extra_agent_args | default("") }}
-KillMode=process
-Delegate=yes
-# Having non-zero Limit*s causes performance problems due to accounting overhead
-# in the kernel. We recommend using cgroups to do container-local accounting.
-LimitNOFILE=1048576
-LimitNPROC=infinity
-LimitCORE=infinity
-TasksMax=infinity
-TimeoutStartSec=0
-Restart=always
-RestartSec=5s
-
-[Install]
-WantedBy=multi-user.target
diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml
index cd3bf720c..d75c625a8 100644
--- a/roles/reset/tasks/main.yml
+++ b/roles/reset/tasks/main.yml
@@ -1,50 +1,21 @@
 ---
-- name: Disable services
-  systemd:
-    name: "{{ item }}"
-    state: stopped
-    enabled: no
-  failed_when: false
-  with_items:
-    - k3s
-    - k3s-node
-    - k3s-init
+- name: Check if uninstall script exists
+  stat:
+    path: /usr/local/bin/k3s-uninstall.sh
+  register: uninstalscript
 
-- name: RUN pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc"
-  register: pkill_containerd_shim_runc
-  command: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc"
-  changed_when: "pkill_containerd_shim_runc.rc == 0"
-  failed_when: false
+- name: Uninstall k3s (server)
+  command: /usr/local/bin/k3s-uninstall.sh
+  when: uninstalscript.stat.exists
 
-- name: Umount k3s filesystems
-  include_tasks: umount_with_children.yml
-  with_items:
-    - /run/k3s
-    - /var/lib/kubelet
-    - /run/netns
-    - /var/lib/rancher/k3s
-    - /var/lib/kubelet/pods
-    - /var/lib/kubelet/plugins
-    - /run/netns/cni-
-  loop_control:
-    loop_var: mounted_fs
+- name: Check if agent uninstall script exists
+  stat:
+    path: /usr/local/bin/k3s-agent-uninstall.sh
+  register: agentuninstalscript
 
-- name: Remove service files, binaries and data
-  file:
-    name: "{{ item }}"
-    state: absent
-  with_items:
-    - /usr/local/bin/k3s
-    - "{{ systemd_dir }}/k3s.service"
-    - "{{ systemd_dir }}/k3s-node.service"
-    - /etc/rancher/k3s
-    - /run/k3s
-    - /run/flannel
-    - /etc/rancher/
-    - /var/lib/kubelet
-    - /var/lib/rancher/k3s
-    - /var/lib/rancher/
-    - /var/lib/cni/
+- name: Uninstall k3s (agent)
+  command: /usr/local/bin/k3s-agent-uninstall.sh
+  when: agentuninstalscript.stat.exists
 
 - name: Reload daemon_reload
   systemd:
diff --git a/roles/reset/tasks/umount_with_children.yml b/roles/reset/tasks/umount_with_children.yml
deleted file mode 100644
index 5883b70a6..000000000
--- a/roles/reset/tasks/umount_with_children.yml
+++ /dev/null
@@ -1,16 +0,0 @@
----
-- name: Get the list of mounted filesystems
-  shell: set -o pipefail && cat /proc/mounts | awk '{ print $2}' | grep -E "^{{ mounted_fs }}"
-  register: get_mounted_filesystems
-  args:
-    executable: /bin/bash
-  failed_when: false
-  changed_when: get_mounted_filesystems.stdout | length > 0
-  check_mode: false
-
-- name: Umount filesystem
-  mount:
-    path: "{{ item }}"
-    state: unmounted
-  with_items:
-    "{{ get_mounted_filesystems.stdout_lines | reverse | list }}"
diff --git a/site.yml b/site.yml
index 5104cce28..82f9485a9 100644
--- a/site.yml
+++ b/site.yml
@@ -15,8 +15,6 @@
       when: proxmox_lxc_configure
     - role: prereq
       become: true
-    - role: download
-      become: true
     - role: raspberrypi
       become: true