Use bootstrap tokens provisioned by ansible for K3s instead of persistent tokens in cloud-init metadata

ansible/bootstrap.yml

@@ -312,4 +312,4 @@
   tasks:
     - ansible.builtin.include_role:
         name: k3s
-        tasks_from: install.yml
+        tasks_from: "{{ 'install.yml' if 'builder' in group_names else 'runtime.yml' }}"

ansible/roles/k3s/defaults/main.yml

@@ -3,3 +3,4 @@ k3s_version: "v1.31.0+k3s1"
 k3s_selinux_release: v1.6.latest.1
 k3s_selinux_rpm_version: 1.6-1
 k3s_helm_version: v3.11.0
+k3s_bootstrap_token_expiry: 20m

ansible/roles/k3s/tasks/install.yml

@@ -71,8 +71,3 @@
   ansible.builtin.lineinfile:
     path: /etc/environment
     line: "KUBECONFIG=/etc/rancher/k3s/k3s.yaml"
-
-- name: Install ansible-init playbook for k3s agent or server activation
-  copy:
-    src: start_k3s.yml
-    dest: /etc/ansible-init/playbooks/0-start-k3s.yml

ansible/roles/k3s/tasks/runtime.yml

@@ -0,0 +1,64 @@
+---
+- name: Check if k3s agents are already connected
+  service_facts:
+  register: services_state
+
+- name: Initialise and authenticate k3s server and agents
+  vars:
+    k3s_server_name: "{{ hostvars[groups['k3s_server'].0].ansible_host }}"
+    access_ip: "{{ ansible_host }}"
+    services_states: > # getting list of all unique agent service states
+      groups['k3s_agent']
+      | map('extract', hostvars, ['services', 'k3s-agent.service', 'state'])
+      | unique
+  when: not (services_state | length == 1 and services_state[0] == 'running') 
+  block:
+  - name: Initialise server and generate bootstrap tokens
+    when: inventory_hostname in groups['k3s_server']
+    block:
+    - name: Template k3s env file
+      ansible.builtin.template:
+        dest: /etc/systemd/system/k3s.service.env
+        src: k3s.service.env.j2
+
+    - name: Start k3s server
+      ansible.builtin.systemd:
+        name: k3s
+        daemon_reload: true
+        state: started
+        enabled: true
+
+    - name: Generate bootstrap token
+      no_log: true
+      shell:
+        cmd: "k3s token create --ttl {{ k3s_bootstrap_token_expiry }}"
+      register: _token_output
+
+  - name: Initialise agents
+    when: inventory_hostname in groups['k3s_agent']
+    block:
+    - name: Template k3s agent env file
+      ansible.builtin.template:
+        dest: /etc/systemd/system/k3s-agent.service.env
+        src: k3s-agent.service.env.j2
+
+    - name: Ensure password directory exists
+      ansible.builtin.file: 
+        path: "/etc/rancher/node"
+        state: directory
+
+    - name: Write node password
+      ansible.builtin.copy:
+        dest: /etc/rancher/node/password
+        content: "{{ vault_k3s_node_password }}"
+        owner: root
+        group: root
+        mode: 640 # normal k3s install is 644 but that doesn't feel right
+
+    - name: Start k3s agent
+      ansible.builtin.systemd:
+        name: k3s-agent
+        daemon_reload: true
+        state: started
+        enabled: true
+

ansible/roles/k3s/templates/k3s-agent.service.env.j2

@@ -0,0 +1,3 @@
+K3S_NODE_IP={{ access_ip }}
+K3S_TOKEN={{ hostvars[groups['control'] | first]._token_output.stdout }}
+K3S_URL=https://{{ k3s_server_name }}:6443

ansible/roles/k3s/templates/k3s.service.env.j2

@@ -0,0 +1 @@
+K3S_NODE_IP={{ access_ip }}

ansible/roles/passwords/defaults/main.yml

@@ -8,7 +8,7 @@ slurm_appliance_secrets:
   vault_openhpc_mungekey: "{{ secrets_openhpc_mungekey | default(vault_openhpc_mungekey | default(secrets_openhpc_mungekey_default)) }}"
   vault_freeipa_ds_password: "{{ vault_freeipa_ds_password | default(lookup('password', '/dev/null')) }}"
   vault_freeipa_admin_password: "{{ vault_freeipa_admin_password | default(lookup('password', '/dev/null')) }}"
-  vault_k3s_token: "{{ vault_k3s_token | default(lookup('ansible.builtin.password', '/dev/null', length=64)) }}"
+  vault_k3s_node_password: "{{ vault_k3s_node_password | default(lookup('ansible.builtin.password', '/dev/null', length=64)) }}"
   vault_pulp_admin_password: "{{ vault_pulp_admin_password | default(lookup('password', '/dev/null', chars=['ascii_letters', 'digits'])) }}"
   vault_demo_user_password: "{{ vault_demo_user_password | default(lookup('password', '/dev/null')) }}"
 

environments/.stackhpc/tofu/cluster_image.auto.tfvars.json

@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-250211-1540-a0b4a57e",
-        "RL9": "openhpc-RL9-250211-1540-a0b4a57e"
+        "RL8": "openhpc-RL8-250219-1007-bfe9cafd",
+        "RL9": "openhpc-RL9-250219-1008-bfe9cafd"
     }
 }

environments/common/inventory/groups

@@ -145,8 +145,16 @@ freeipa_client
 [compute_init]
 # EXPERIMENTAL: Compute hosts to enable joining cluster on boot on
 
-[k3s]
+[k3s:children]
 # Hosts to run k3s server/agent
+k3s_server
+k3s_agent
+
+[k3s_server]
+# Hosts to run k3s server (should only be single node i.e control node)
+
+[k3s_agent]
+# Hosts to run k3s agent
 
 [k9s]
 # Hosts to install k9s on

environments/common/layouts/everything

@@ -97,9 +97,14 @@ cluster
 # EXPERIMENTAL: Compute hosts to enable joining cluster on boot on
 compute
 
-[k3s:children]
-# Hosts to run k3s server/agent
-openhpc
+[k3s_server:children]
+# Hosts to run k3s server (should only be single node i.e control node)
+control
+
+[k3s_agent:children]
+# Hosts to run k3s agent
+compute
+login
 
 [k9s:children]
 # Hosts to install k9s on

environments/skeleton/{{cookiecutter.environment}}/tofu/compute.tf

@@ -29,7 +29,6 @@ module "compute" {
   availability_zone = lookup(each.value, "availability_zone", "nova")
 
   # computed
-  k3s_token = local.k3s_token
   # not using openstack_compute_instance_v2.control.access_ip_v4 to avoid
   # updates to node metadata on deletion/recreation of the control node:
   control_address = openstack_networking_port_v2.control[var.cluster_networks[0].network].all_fixed_ips[0]

environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf

@@ -59,7 +59,6 @@ resource "openstack_compute_instance_v2" "control" {
 
   metadata = {
     environment_root = var.environment_root
-    k3s_token = local.k3s_token
     access_ip = openstack_networking_port_v2.control[var.cluster_networks[0].network].all_fixed_ips[0]
   }
 

environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf

@@ -33,7 +33,6 @@ module "login" {
   ignore_image_changes = false
 
   # computed
-  k3s_token = local.k3s_token
   # not using openstack_compute_instance_v2.control.access_ip_v4 to avoid
   # updates to node metadata on deletion/recreation of the control node:
   control_address = openstack_networking_port_v2.control[var.cluster_networks[0].network].all_fixed_ips[0]

environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/nodes.tf

@@ -85,7 +85,6 @@ resource "openstack_compute_instance_v2" "compute_fixed_image" {
   metadata = merge(
     {
         environment_root = var.environment_root
-        k3s_token          = var.k3s_token
         control_address    = var.control_address
         access_ip = openstack_networking_port_v2.compute["${each.key}-${var.networks[0].network}"].all_fixed_ips[0]
     },
@@ -139,7 +138,6 @@ resource "openstack_compute_instance_v2" "compute" {
   metadata = merge(
     {
         environment_root = var.environment_root
-        k3s_token          = var.k3s_token
         control_address    = var.control_address
         access_ip = openstack_networking_port_v2.compute["${each.key}-${var.networks[0].network}"].all_fixed_ips[0]
     },

environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf

@@ -70,10 +70,6 @@ variable "security_group_ids" {
     type = list
 }
 
-variable "k3s_token" {
-    type = string
-}
-
 variable "control_address" {
     description = "Name/address of control node"
     type = string

environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf

@@ -189,7 +189,3 @@ variable "inventory_secrets_path" {
   type = string
   default = ""
 }
-
-locals {
-    k3s_token = data.external.inventory_secrets.result["vault_k3s_token"]
-}