[update] Serialize VM evacuation during compute reboot

roles/update/README.md

@@ -18,6 +18,8 @@ Role to run update
 * `cifmw_update_ctl_plane_max_tries`: (Integer) For continuous control plane testing, number of retries allowed to stop and destroy the last vm created. Each retry is 5 seconds apart. Default to 84, so 7 minutes.
 * `cifmw_update_openstackclient_pod_timeout`: (Integer) Maximum number of seconds to wait for the openstackclient Pod to be available during control plane testing, as it is being restarted during update.  Default to `10` seconds.
 * `cifmw_update_reboot_test`: (Boolean) Activate the reboot test after update. Default to `False`.
+* `cifmw_update_evacuate_vms_synchronously`: (Boolean) Live-migrate VMs from a hypervisor one at a time, then return them in the same creation order after reboot. Default to `False`.
+* `cifmw_update_vm_migration_wait_retries`: (Integer) Maximum number of checks for each synchronous live migration, five seconds apart. Default to `720`, allowing about one hour per VM.
 * `cifmw_update_ansible_ssh_private_key_file`: (String) Define the path to the private key file used for the compute nodes.
 * `cifmw_update_wait_retries_reboot`: (Integer) Number of retries to wait for a compute node reboot. One retry is done every five seconds. Default to 60, so five minutes.
 * `cifmw_update_resources_monitoring_interval`: (Integer) Interval, in seconds, between two resources monitor during update. Default to 10 seconds.

roles/update/defaults/main.yml

@@ -52,6 +52,8 @@ cifmw_update_cleanup: true
 cifmw_update_openstack_update_kpatch: false
 
 cifmw_update_reboot_test: false
+cifmw_update_evacuate_vms_synchronously: false
+cifmw_update_vm_migration_wait_retries: 720
 cifmw_update_ansible_ssh_private_key_file: >-
   "{{ ansible_ssh_private_key_file | default(ansible_user_dir ~ '/.ssh/id_cifw') }}"
 cifmw_update_wait_retries_reboot: 60

roles/update/tasks/migrate_instance_from_hypervisor.yml

@@ -0,0 +1,30 @@
+---
+- name: Start live migration of {{ cifmw_update_instance_id }}
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    {{ cifmw_update_bash_cmd }} ". cloudrc &&
+    nova live-migration
+    {% if 'ceph' not in storage_backend.stdout %}
+    --block-migrate
+    {% endif %}
+    {{ cifmw_update_instance_id }}"
+  changed_when: true
+
+- name: Wait for live migration of {{ cifmw_update_instance_id }}
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ cifmw_update_openstack_cmd }} server show
+    {{ cifmw_update_instance_id }} -f json
+    | jq -r -c --arg source_host "{{ cifmw_update_target_hypervisor }}"
+    '((.status == "ACTIVE") or (.status == "PAUSED")) and
+    (.["OS-EXT-SRV-ATTR:host"] != $source_host)'
+  register: cifmw_update_instance_migration
+  changed_when: false
+  until: cifmw_update_instance_migration.stdout == 'true'
+  retries: "{{ cifmw_update_vm_migration_wait_retries | int }}"
+  delay: 5

roles/update/tasks/reboot_computes.yml

@@ -50,7 +50,7 @@
   ansible.builtin.include_tasks: reboot_hypervisor.yml
   loop: "{{ hypervisor_hostnames }}"
   loop_control:
-    loop_var: hypervisor
+    loop_var: cifmw_update_target_hypervisor
 
 - name: Stop the monitor servers script if running
   ansible.builtin.shell: |

roles/update/tasks/reboot_hypervisor.yml

@@ -1,7 +1,7 @@
 ---
 - name: Extract short hostname from FQDN
   ansible.builtin.set_fact:
-    cifmw_update_hypervisor_short_name: "{{ hypervisor.split('.')[0] }}"
+    cifmw_update_hypervisor_short_name: "{{ cifmw_update_target_hypervisor.split('.')[0] }}"
 
 - name: Display current stage
   ansible.builtin.debug:
@@ -18,12 +18,12 @@
     PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
   ansible.builtin.shell: >-
     set -o pipefail;
-    {{ cifmw_update_openstack_cmd }} server list --all --host {{ hypervisor }} -f json
+    {{ cifmw_update_openstack_cmd }} server list --all --host {{ cifmw_update_target_hypervisor }} -f json
     | jq -r -c '.[] | select(.Status | contains("ACTIVE") or contains("PAUSED")) | .ID'
   register: active_vms
   changed_when: false
 
-- name: Evacuate VMs if they are running
+- name: Evacuate VMs asynchronously if they are running
   environment:
     KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
     PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
@@ -33,24 +33,53 @@
     {% if 'ceph' not in storage_backend.stdout %}
     --block-migrate
     {% endif %}
-    {{ hypervisor }}"
+    {{ cifmw_update_target_hypervisor }}"
   when:
     - active_vms.stdout != ''
     - hypervisor_hostnames | length > 1
+    - not cifmw_update_evacuate_vms_synchronously | bool
   changed_when: true
 
+- name: Order active VMs for synchronous evacuation
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ cifmw_update_openstack_cmd }} server list --all --host {{ cifmw_update_target_hypervisor }}
+    -f json -c ID -c Status -c "Created At"
+    | jq -r -c '[.[] | select(.Status |
+    contains("ACTIVE") or contains("PAUSED"))]
+    | sort_by(.["Created At"])[] | .ID'
+  register: cifmw_update_ordered_active_vms
+  changed_when: false
+  when:
+    - active_vms.stdout != ''
+    - hypervisor_hostnames | length > 1
+    - cifmw_update_evacuate_vms_synchronously | bool
+
+- name: Evacuate VMs synchronously if they are running
+  ansible.builtin.include_tasks: migrate_instance_from_hypervisor.yml
+  loop: "{{ cifmw_update_ordered_active_vms.stdout_lines }}"
+  loop_control:
+    loop_var: cifmw_update_instance_id
+  when:
+    - active_vms.stdout != ''
+    - hypervisor_hostnames | length > 1
+    - cifmw_update_evacuate_vms_synchronously | bool
+
 - name: Wait for compute node to get quiesced
   environment:
     KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
     PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
   ansible.builtin.shell: >-
     set -o pipefail;
-    {{ cifmw_update_openstack_cmd }} server list --all --host {{ hypervisor }} -f json
+    {{ cifmw_update_openstack_cmd }} server list --all --host {{ cifmw_update_target_hypervisor }} -f json
     | jq -r -c '[.[] | select(.Status |
     contains("ACTIVE") or contains("PAUSED") or contains("MIGRATING"))]
     | length'
   register: compute_node_instances
-  until: compute_node_instances.stdout.find("0") > -1
+  until: compute_node_instances.rc == 0 and compute_node_instances.stdout == '0'
   retries: 30
   delay: 5
   when:
@@ -63,12 +92,15 @@
 - name: Perform sanity checks post-reboot
   ansible.builtin.include_tasks: reboot_hypervisor_sanity_checks.yml
   vars:
-    current_hypervisor: "{{ hypervisor }}"
+    current_hypervisor: "{{ cifmw_update_target_hypervisor }}"
 
 - name: Display current stage
   ansible.builtin.debug:
     msg: "Migrate back {{ item }} to {{ cifmw_update_hypervisor_short_name }}."
-  with_items: "{{ active_vms.stdout_lines }}"
+  with_items: >-
+    {{ (cifmw_update_ordered_active_vms.stdout_lines | default([]))
+       if cifmw_update_evacuate_vms_synchronously | bool
+       else active_vms.stdout_lines }}
 
 - name: Migrate back VMs post-reboot
   environment:
@@ -81,14 +113,17 @@
     {% if 'ceph' not in storage_backend.stdout %}
     --block-migrate
     {% endif %}
-    {{ item }} {{ hypervisor }}";
+    {{ item }} {{ cifmw_update_target_hypervisor }}";
     {{ cifmw_update_openstack_cmd }} server show {{ item }} -f json |
     jq -r -c '. | .["OS-EXT-SRV-ATTR:host"]'
   register: instance_migration_result
-  until: instance_migration_result.stdout.find(hypervisor) > -1
+  until: instance_migration_result.stdout.find(cifmw_update_target_hypervisor) > -1
   retries: 30
   delay: 5
-  with_items: "{{ active_vms.stdout_lines }}"
+  with_items: >-
+    {{ (cifmw_update_ordered_active_vms.stdout_lines | default([]))
+       if cifmw_update_evacuate_vms_synchronously | bool
+       else active_vms.stdout_lines }}
   when:
     - active_vms.stdout != ''
     - hypervisor_hostnames | length > 1