[cleanup_openstack] Force-drain stuck OpenStack namespace

docs/dictionary/en-custom.txt

@@ -238,6 +238,7 @@ fedoraproject
 fil
 Fernet
 filesystem
+finalizers
 fips
 firewalld
 flbxutz

roles/cleanup_openstack/README.md

@@ -9,3 +9,4 @@ None
 As this role is for cleanup it utilizes default vars from other roles which can be referenced at their role readme page: kustomize_deploy, deploy_bmh
 
 * `cifmw_cleanup_openstack_detach_bmh`: (Boolean) Detach BMH when cleaning flag, this is used to avoid deprovision when is not required. Default: `true`
+* `cifmw_cleanup_openstack_operators_wait_timeout`: (Integer) Seconds to wait for the operator CRs and the OpenStack namespace to be removed before giving up and force-draining a namespace stuck in `Terminating`. Kept short so the recovery (strip leftover finalizers + retry) kicks in quickly instead of burning the full default deletion timeout. Default: `300`

roles/cleanup_openstack/defaults/main.yaml

@@ -1 +1,6 @@
 cifmw_cleanup_openstack_detach_bmh: true
+# Seconds to wait for the operator CRs and the OpenStack namespace to be removed
+# before giving up and force-draining a namespace stuck in "Terminating". Kept
+# short so the recovery (strip leftover finalizers + retry) kicks in quickly
+# instead of burning the full default deletion timeout.
+cifmw_cleanup_openstack_operators_wait_timeout: 300

roles/cleanup_openstack/tasks/cleanup_crs.yaml

@@ -13,12 +13,12 @@
     state: absent
     src: "{{ item.stat.path }}"
     wait: true
-    wait_timeout: 600
+    wait_timeout: "{{ cifmw_cleanup_crs_wait_timeout | default(600) }}"
   loop: "{{ _crs_to_delete_files.results }}"
   register: _cleanup_results
   until: "_cleanup_results is success"
-  retries: 3
-  delay: 120
+  retries: "{{ cifmw_cleanup_crs_retries | default(3) }}"
+  delay: "{{ cifmw_cleanup_crs_delay | default(120) }}"
   when:
     - item.stat.exists
 

roles/cleanup_openstack/tasks/force_drain_namespace.yaml

@@ -0,0 +1,59 @@
+---
+# Last-resort teardown helper: an OpenStack namespace can stay stuck in
+# "Terminating" forever when CRs still hold finalizers and the operators that
+# would clear them are already (being) removed. The per-kind finalizer removal
+# in `wait_and_cleanup_resource.yaml` only covers the top-level CRs
+# (OpenStackControlPlane, dataplane, RabbitmqCluster); their children
+# (keystone*, nova*, galera, mariadbaccount, keystoneendpoint, ...) and core
+# resources that carry OpenStack finalizers (secrets, configmaps, services) can
+# remain orphaned. Discover every namespaced OpenStack CR kind and strip the
+# finalizers from whatever is left so the namespace can finish terminating.
+- name: Discover namespaced OpenStack CRDs
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit) }}"
+    context: "{{ cifmw_openshift_context | default(omit) }}"
+    kind: CustomResourceDefinition
+    api_version: apiextensions.k8s.io/v1
+  register: _cifmw_cleanup_crds
+
+- name: Build the list of namespaced OpenStack CR kinds to drain
+  ansible.builtin.set_fact:
+    _cifmw_cleanup_drain_kinds: >-
+      {{
+        _cifmw_cleanup_drain_kinds | default([]) + [{
+          'kind': item.spec.names.kind,
+          'api_version': item.spec.group ~ '/' ~ (
+            item.spec.versions
+            | selectattr('storage', 'defined') | selectattr('storage')
+            | map(attribute='name') | first
+          )
+        }]
+      }}
+  loop: >-
+    {{
+      _cifmw_cleanup_crds.resources
+      | selectattr('spec.scope', 'equalto', 'Namespaced')
+      | selectattr('spec.group', 'search', '(\.openstack\.org$|^rabbitmq\.com$)')
+      | list
+    }}
+  loop_control:
+    label: "{{ item.spec.names.kind }}"
+
+- name: Add core kinds that may carry OpenStack finalizers
+  ansible.builtin.set_fact:
+    _cifmw_cleanup_drain_kinds: >-
+      {{
+        (_cifmw_cleanup_drain_kinds | default([])) + [
+          {'kind': 'Secret', 'api_version': 'v1'},
+          {'kind': 'ConfigMap', 'api_version': 'v1'},
+          {'kind': 'Service', 'api_version': 'v1'}
+        ]
+      }}
+
+- name: Strip finalizers from every remaining resource in the namespace
+  ansible.builtin.include_tasks: strip_finalizers.yaml
+  loop: "{{ _cifmw_cleanup_drain_kinds }}"
+  loop_control:
+    loop_var: cifmw_cleanup_resource
+    label: "{{ cifmw_cleanup_resource.kind }}"

roles/cleanup_openstack/tasks/main.yaml

@@ -106,9 +106,6 @@
       api_version: rabbitmq.com/v1beta1
   loop_control:
     loop_var: cifmw_cleanup_resource
-  vars:
-    cifmw_cleanup_resource_kind: "{{ cifmw_cleanup_resource.kind }}"
-    cifmw_cleanup_resource_api_version: "{{ cifmw_cleanup_resource.api_version }}"
 
 - name: Delete deployment CRs (Phase 2 - Operators)
   vars:
@@ -118,7 +115,25 @@
       - "{{ cifmw_kustomize_deploy_kustomizations_dest_dir }}/openstack.yaml"
       - "{{ cifmw_kustomize_deploy_olm_dest_file }}"
     _crs_to_delete: "{{ _operators_crs }}"
-  ansible.builtin.import_tasks: cleanup_crs.yaml
+    # Fail fast so the force-drain safety net kicks in quickly instead of
+    # burning the full default deletion timeout waiting on a namespace that is
+    # already wedged in "Terminating".
+    cifmw_cleanup_crs_wait_timeout: "{{ cifmw_cleanup_openstack_operators_wait_timeout }}"
+    cifmw_cleanup_crs_retries: 1
+  block:
+    - name: Remove operator CRs and the OpenStack namespace
+      ansible.builtin.include_tasks: cleanup_crs.yaml
+  rescue:
+    # Deleting the OpenStack namespace issues a deletionTimestamp on everything
+    # inside it; if some CRs are stuck because the operators that own their
+    # finalizers are already gone, the namespace hangs in "Terminating". Strip
+    # the leftover finalizers and retry the removal. Only reachable on a failed
+    # graceful teardown, so it adds zero cost to the happy path.
+    - name: Force-drain the stuck OpenStack namespace
+      ansible.builtin.include_tasks: force_drain_namespace.yaml
+
+    - name: Retry removing operator CRs and the OpenStack namespace
+      ansible.builtin.include_tasks: cleanup_crs.yaml
 
 - name: Get artifacts scripts
   ansible.builtin.find:

roles/cleanup_openstack/tasks/strip_finalizers.yaml

@@ -0,0 +1,39 @@
+---
+# Strip finalizers from every remaining resource of a single kind in the
+# OpenStack namespace. Used as a last resort when a resource is stuck because
+# the operator that owns its finalizer is already gone, which would otherwise
+# keep the namespace in "Terminating" forever.
+- name: "List remaining resources of kind - {{ cifmw_cleanup_resource.kind }}"
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit) }}"
+    context: "{{ cifmw_openshift_context | default(omit) }}"
+    namespace: "{{ cifmw_openstack_namespace }}"
+    kind: "{{ cifmw_cleanup_resource.kind }}"
+    api_version: "{{ cifmw_cleanup_resource.api_version }}"
+  register: _cifmw_cleanup_remaining
+  failed_when: false
+
+- name: "Force-remove finalizers from stuck resources of kind - {{ cifmw_cleanup_resource.kind }}"
+  kubernetes.core.k8s_json_patch:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit) }}"
+    context: "{{ cifmw_openshift_context | default(omit) }}"
+    namespace: "{{ cifmw_openstack_namespace }}"
+    kind: "{{ cifmw_cleanup_resource.kind }}"
+    api_version: "{{ cifmw_cleanup_resource.api_version }}"
+    name: "{{ item.metadata.name }}"
+    patch:
+      - op: replace
+        path: /metadata/finalizers
+        value: []
+  loop: >-
+    {{
+      _cifmw_cleanup_remaining.resources | default([])
+      | selectattr('metadata.finalizers', 'defined')
+      | selectattr('metadata.finalizers', 'truthy')
+      | list
+    }}
+  loop_control:
+    label: "{{ item.metadata.name }}"
+  failed_when: false

roles/cleanup_openstack/tasks/wait_and_cleanup_resource.yaml

@@ -1,33 +1,18 @@
 ---
-- name: "Ensure resources are removed - {{ cifmw_cleanup_resource_kind }}"
+- name: "Ensure resources are removed - {{ cifmw_cleanup_resource.kind }}"
   block:
-    - name: "Wait for resource to be deleted - {{ cifmw_cleanup_resource_kind }}"
+    - name: "Wait for resource to be deleted - {{ cifmw_cleanup_resource.kind }}"
       kubernetes.core.k8s_info:
         kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
         api_key: "{{ cifmw_openshift_token | default(omit) }}"
         context: "{{ cifmw_openshift_context | default(omit) }}"
         namespace: "{{ cifmw_openstack_namespace }}"
-        kind: "{{ cifmw_cleanup_resource_kind }}"
-        api_version: "{{ cifmw_cleanup_resource_api_version }}"
+        kind: "{{ cifmw_cleanup_resource.kind }}"
+        api_version: "{{ cifmw_cleanup_resource.api_version }}"
       register: cifmw_cleanup_resource_result
       until: cifmw_cleanup_resource_result.resources | default([]) | length == 0
       retries: 60
       delay: 10
   rescue:
-    - name: "Force-remove finalizers from stuck resource - {{ cifmw_cleanup_resource_kind }}"
-      kubernetes.core.k8s_json_patch:
-        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
-        api_key: "{{ cifmw_openshift_token | default(omit) }}"
-        context: "{{ cifmw_openshift_context | default(omit) }}"
-        namespace: "{{ cifmw_openstack_namespace }}"
-        kind: "{{ cifmw_cleanup_resource_kind }}"
-        api_version: "{{ cifmw_cleanup_resource_api_version }}"
-        name: "{{ item.metadata.name }}"
-        patch:
-          - op: replace
-            path: /metadata/finalizers
-            value: []
-      loop: "{{ cifmw_cleanup_resource_result.resources | default([]) }}"
-      loop_control:
-        label: "{{ item.metadata.name }}"
-      failed_when: false
+    - name: "Force-remove finalizers from stuck resource - {{ cifmw_cleanup_resource.kind }}"
+      ansible.builtin.include_tasks: strip_finalizers.yaml