feat: handle node drain failure without manual intervention by uncordoning node (#35)

Author: marshallford

README.md

@@ -110,7 +110,5 @@ provider "kubernetes" {
 - [ ] Explore Keepalived `max_auto_priority` option
 - [ ] Assert podman version
 - [ ] Token rotation
-- [ ] Stop Zicanati at start of playbook and start at the end
+- [ ] Stop Zincati at start of playbook and start at the end
 - [ ] Knownhost management
-- [ ] Variables for fine-grained node draining (`--disable-eviction`, timeout, etc)
-- [ ] Handle node drain failure without manual intervention

ansible/playbook.yaml

@@ -136,6 +136,7 @@
       __k3s_files: "{{ k3s_files }}"
       __k3s_kubelet_configs: "{{ k3s_kubelet_configs }}"
       __k3s_registries_config: "{{ k3s_registries_config }}"
+      __k3s_drain_options: "{{ k3s_drain_options }}"
 - name: Configure existing k3s servers
   hosts: servers:!new_servers
   gather_facts: false
@@ -159,6 +160,7 @@
       __k3s_files: "{{ k3s_files }}"
       __k3s_kubelet_configs: "{{ k3s_kubelet_configs }}"
       __k3s_registries_config: "{{ k3s_registries_config }}"
+      __k3s_drain_options: "{{ k3s_drain_options }}"
 - name: Configure new k3s agents
   hosts: new_agents
   gather_facts: false
@@ -181,6 +183,7 @@
       __k3s_files: "{{ k3s_files }}"
       __k3s_kubelet_configs: "{{ k3s_kubelet_configs }}"
       __k3s_registries_config: "{{ k3s_registries_config }}"
+      __k3s_drain_options: "{{ k3s_drain_options }}"
 - name: Configure existing k3s agents
   hosts: agents:!new_agents
   gather_facts: false
@@ -204,6 +207,7 @@
       __k3s_files: "{{ k3s_files }}"
       __k3s_kubelet_configs: "{{ k3s_kubelet_configs }}"
       __k3s_registries_config: "{{ k3s_registries_config }}"
+      __k3s_drain_options: "{{ k3s_drain_options }}"
 - name: Cleanup machines
   hosts: servers:agents
   gather_facts: false

ansible/roles/k3s/defaults/main.yaml

@@ -26,5 +26,9 @@ __k3s_selinux_download_baseurl: "https://github.com/k3s-io/k3s-selinux/releases/
 __k3s_selinux_download_package: "k3s-selinux-{{ __k3s_selinux_major_minor }}-{{ __k3s_selinux_patch }}.coreos.noarch.rpm"
 __k3s_selinux_download_checksum: sha256sum-coreos-noarch.txt
 __k3s_selinux_rpm_ostree_package: "k3s-selinux-{{ __k3s_selinux_major_minor }}-{{ __k3s_selinux_patch }}.coreos.noarch"
-__k3s_drain_opts: --ignore-daemonsets --delete-emptydir-data --force --timeout 10m
+__k3s_drain_options:
+  deletion_fallback: false
+  eviction_timeout: "30m"
+  deletion_timeout: "10m"
+__k3s_drain_cli_options: --ignore-daemonsets --delete-emptydir-data --force
 __k3s_https_listen_port: 8443

ansible/roles/k3s/tasks/drain.yaml

@@ -4,14 +4,23 @@
   changed_when: true
 
 - name: Drain node
-  ansible.builtin.command:
-    cmd: "{{ __k3s_binary_dir }}/k3s kubectl drain {{ __k3s_drain_opts }} {{ __k3s_node_config['node-name'] }}"
-  register: __k3s_drain
-  failed_when: __k3s_drain.rc not in [0, 1]
-  changed_when: true
-
-- name: Force drain node
-  ansible.builtin.command:
-    cmd: "{{ __k3s_binary_dir }}/k3s kubectl drain {{ __k3s_drain_opts }} --disable-eviction {{ __k3s_node_config['node-name'] }}"
-  when: __k3s_drain.rc == 1
-  changed_when: true
+  block:
+  - name: Drain node (evict)
+    ansible.builtin.command:
+      cmd: "{{ __k3s_binary_dir }}/k3s kubectl drain {{ __k3s_drain_cli_options }} --timeout {{ __k3s_drain_options.eviction_timeout }} {{ __k3s_node_config['node-name'] }}"
+    register: __k3s_drain
+    failed_when: __k3s_drain.rc != 0 and not __k3s_drain_options.deletion_fallback
+    changed_when: true
+  - name: Drain node (delete)
+    ansible.builtin.command:
+      cmd: "{{ __k3s_binary_dir }}/k3s kubectl drain {{ __k3s_drain_cli_options }} --timeout {{ __k3s_drain_options.deletion_timeout }} --disable-eviction {{ __k3s_node_config['node-name'] }}"
+    when: __k3s_drain.rc != 0 and __k3s_drain_options.deletion_fallback
+    changed_when: true
+  rescue:
+  - name: Uncordon node
+    ansible.builtin.command:
+      cmd: "{{ __k3s_binary_dir }}/k3s kubectl uncordon {{ __k3s_node_config['node-name'] }}"
+    changed_when: true
+  - name: Drain failed
+    ansible.builtin.fail:
+      msg: "Draining node {{ __k3s_node_config['node-name'] }} failed."

main.tf

@@ -22,6 +22,7 @@ locals {
         k3s_kubelet_configs         = var.kubelet_configs
         k3s_registries_config       = var.registries_config
         k3s_cleanup                 = var.cleanup
+        k3s_drain_options           = var.drain_options
         system_upgrade_trigger      = var.system_upgrade_trigger
         haproxy_container_image     = var.haproxy_container_image
         haproxy_container_image_tag = var.haproxy_container_image_tag
@@ -117,9 +118,9 @@ resource "ansible_navigator_run" "this" {
     }
   }
   timeouts = {
-    create = "60m"
-    update = "60m"
-    delete = "60m"
+    create = "2h"
+    update = "4h"
+    delete = "1h"
   }
   run_on_destroy   = var.reset_on_destroy
   destroy_playbook = file("${path.module}/ansible/destroy_playbook.yaml")

variables.tf

@@ -65,7 +65,6 @@ variable "server_machines" {
     config = optional(object({
       cluster_init     = optional(bool, false)
       node_name        = optional(string)
-      with_node_id     = optional(bool, false)
       node_ip          = optional(string)
       node_external_ip = optional(string)
       node_label       = optional(map(string), {})
@@ -102,7 +101,6 @@ variable "agent_machine_groups" {
     })
     config = optional(object({
       node_name        = optional(string)
-      with_node_id     = optional(bool, false)
       node_ip          = optional(string)
       node_external_ip = optional(string)
       node_label       = optional(map(string), {})
@@ -248,6 +246,25 @@ variable "registries_config" {
   description = "Registry configuration to be used by k3s when generating the containerd configuration."
 }
 
+variable "drain_options" {
+  type = object({
+    deletion_fallback = optional(bool, false)
+    eviction_timeout  = optional(string, "30m")
+    deletion_timeout  = optional(string, "10m")
+  })
+  nullable    = false
+  default     = {}
+  description = "Node drain options."
+  validation {
+    condition     = can(timeadd(timestamp(), var.drain_options.eviction_timeout))
+    error_message = "The eviction_timeout must be a valid duration (e.g., '10m', '30s', '1h30m')."
+  }
+  validation {
+    condition     = can(timeadd(timestamp(), var.drain_options.deletion_timeout))
+    error_message = "The deletion_timeout must be a valid duration (e.g., '10m', '30s', '1h30m')."
+  }
+}
+
 variable "system_upgrade_trigger" {
   type        = string
   nullable    = false