From 7518db12416d10f4a11060856931a91c7415406c Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 4 Aug 2026 12:08:44 -0400 Subject: [PATCH 1/9] [Test] In test_patching_cluster, reduce the risk of ICEs by using c5.xlarge for hea dnode and login nodes rather than GPU instances. --- .../tests/patching/test_patching.py | 13 ++++++++++++- .../test_patching_cluster/pcluster.config.yaml | 9 ++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/integration-tests/tests/patching/test_patching.py b/tests/integration-tests/tests/patching/test_patching.py index f07b9765d9..326172eb5b 100644 --- a/tests/integration-tests/tests/patching/test_patching.py +++ b/tests/integration-tests/tests/patching/test_patching.py @@ -42,6 +42,9 @@ # so it is defined in a single place. FSX_LUSTRE_MOUNT_DIR = "/shared-fsxlustre" +HEAD_NODE_INSTANCE = "c5.xlarge" +LOGIN_NODE_INSTANCE = "c5.xlarge" + def test_patching_cluster( region, @@ -83,7 +86,11 @@ def test_patching_cluster( # Start the cluster creation but do not block on it: the AMI patching below # runs concurrently while the cluster comes up. create_config = pcluster_config_reader( - output_file="pcluster.config.create.yaml", login_nodes_count=1, fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR + output_file="pcluster.config.create.yaml", + login_nodes_count=1, + head_node_instance=HEAD_NODE_INSTANCE, + login_node_instance=LOGIN_NODE_INSTANCE, + fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR, ) cluster = clusters_factory(create_config, wait=False) @@ -123,6 +130,8 @@ def test_patching_cluster( output_file="pcluster.config.stop-login.yaml", login_nodes_count=0, base_ami=base_ami_pin, + head_node_instance=HEAD_NODE_INSTANCE, + login_node_instance=LOGIN_NODE_INSTANCE, fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR, ) cluster.update(str(stop_login_config)) @@ -137,6 +146,8 @@ def test_patching_cluster( login_nodes_count=1, base_ami=base_ami_pin, patched_ami=patched_ami, + head_node_instance=HEAD_NODE_INSTANCE, + login_node_instance=LOGIN_NODE_INSTANCE, fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR, ) cluster.update(str(update_config)) diff --git a/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/pcluster.config.yaml b/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/pcluster.config.yaml index f780ca2f25..72887fe1d7 100644 --- a/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/pcluster.config.yaml +++ b/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/pcluster.config.yaml @@ -4,7 +4,7 @@ Image: CustomAmi: {{ base_ami }} {% endif %} HeadNode: - InstanceType: {{ instance }} + InstanceType: {{ head_node_instance }} Networking: SubnetId: {{ public_subnet_id }} ElasticIp: true @@ -15,7 +15,7 @@ HeadNode: LoginNodes: Pools: - Name: login1 - InstanceType: {{ instance }} + InstanceType: {{ login_node_instance }} Count: {{ login_nodes_count }} GracetimePeriod: 3 {% if patched_ami %} @@ -50,7 +50,10 @@ Scheduling: - {{ private_subnet_id }} ComputeResources: - Name: cr1 - InstanceType: {{ instance }} + Instances: + {% for instance_type in flexible_instance_types %} + - InstanceType: {{ instance_type }} + {% endfor %} MinCount: 1 MaxCount: 1 Efa: From 1472dae958dbdee19abed2b0326a4c9c7dc16e0a Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 4 Aug 2026 12:09:24 -0400 Subject: [PATCH 2/9] [Test] In test_patching_cluster, print the full ImageBuilder log when the patching fails to facilitate troubleshooting. --- tests/integration-tests/conftest.py | 60 ++++++++++------------------- 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/tests/integration-tests/conftest.py b/tests/integration-tests/conftest.py index a0785fdd64..8b9ebb70b0 100644 --- a/tests/integration-tests/conftest.py +++ b/tests/integration-tests/conftest.py @@ -1911,49 +1911,25 @@ def _copy_image(image_id, test_name): logging.error("Delete copied AMI snapshot failed due to %s", e) -def _log_ami_patching_build_output(region, stack_name): # noqa: C901 - """Fetch and log the EC2 Image Builder execution logs for the patched-AMI build. - - Image Builder streams the output of every recipe step (including the patch - script stdout from the ApplyPatches ExecuteBash step) to the CloudWatch log - group /aws/imagebuilder/pcluster-ami-patching-recipe-. Surfacing it - here mirrors how the head node patching output is logged by the test, so the AMI - patching output is available in the test logs too. Any failure to retrieve the - logs is swallowed so it never fails the test. +def _log_ami_patching_build_output(region, stack_name): + """Log the EC2 Image Builder execution logs for the patched-AMI build. + + Image Builder streams the output of every recipe step (including the patch script + output) to the CloudWatch log group + /aws/imagebuilder/pcluster-ami-patching-recipe-. Logging it surfaces the + patch output (and, on failure, the root cause) in the test logs. Any failure to + retrieve the logs is swallowed so it never fails the test. """ log_group_name = f"/aws/imagebuilder/pcluster-ami-patching-recipe-{stack_name}" logs_client = boto3.client("logs", region_name=region) try: - streams = [] - next_token = None - while True: - kwargs = {"logGroupName": log_group_name, "orderBy": "LogStreamName"} - if next_token: - kwargs["nextToken"] = next_token - response = logs_client.describe_log_streams(**kwargs) - streams.extend(response.get("logStreams", [])) - next_token = response.get("nextToken") - if not next_token: - break - if not streams: - logging.info("No AMI patching Image Builder log streams found in log group %s", log_group_name) - return - for stream in streams: - stream_name = stream["logStreamName"] - messages = [] - prev_token = None - event_token = None - while True: - event_kwargs = {"logGroupName": log_group_name, "logStreamName": stream_name, "startFromHead": True} - if event_token: - event_kwargs["nextToken"] = event_token - event_response = logs_client.get_log_events(**event_kwargs) - messages.extend(event["message"] for event in event_response.get("events", [])) - event_token = event_response.get("nextForwardToken") - if event_token == prev_token: - break - prev_token = event_token - logging.info("AMI patching Image Builder log (stream %s):\n%s", stream_name, "\n".join(messages)) + messages = [] + for page in logs_client.get_paginator("filter_log_events").paginate(logGroupName=log_group_name): + messages.extend(event["message"] for event in page.get("events", [])) + if messages: + logging.info("AMI patching Image Builder logs:\n%s", "\n".join(messages)) + else: + logging.info("No AMI patching Image Builder logs found in log group %s", log_group_name) except Exception as e: # noqa: BLE001 logging.warning("Could not retrieve AMI patching Image Builder logs from log group %s: %s", log_group_name, e) @@ -2019,7 +1995,11 @@ def _build(base_ami, builder_instance, flavour="minimal"): ) # create_stack blocks until CREATE_COMPLETE, i.e. until Image Builder has # finished building the patched AMI. - cfn_stacks_factory.create_stack(stack) + try: + cfn_stacks_factory.create_stack(stack) + except Exception: # noqa: BLE001 + _log_ami_patching_build_output(region, stack_name) + raise ami_id = stack.cfn_outputs["AmiId"] built.append((ami_id, stack_name)) logging.info("Patched AMI %s is available", ami_id) From c67b4a35afdbaf6668af09fde7aae77dbc540b74 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 4 Aug 2026 16:24:14 -0400 Subject: [PATCH 3/9] [Test] Add `UBUNTU_` OS group for test OS rotation. --- .../framework/tests_configuration/config_renderer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration-tests/framework/tests_configuration/config_renderer.py b/tests/integration-tests/framework/tests_configuration/config_renderer.py index f2fa7a52f0..04f5eacfd1 100644 --- a/tests/integration-tests/framework/tests_configuration/config_renderer.py +++ b/tests/integration-tests/framework/tests_configuration/config_renderer.py @@ -93,6 +93,9 @@ def _get_os_parameters(config=None, args=None): rhel_oss = [os for os in SUPPORTED_OSES if "rhel" in os] _propagate_os_jinja_variables("RHEL_", result, rotation_seed, rhel_oss) + + ubuntu_oss = [os for os in SUPPORTED_OSES if "ubuntu" in os] + _propagate_os_jinja_variables("UBUNTU_", result, rotation_seed, ubuntu_oss) return result From 97b7d076984eced903c48465237da1c9398e4900 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 4 Aug 2026 17:30:52 -0400 Subject: [PATCH 4/9] [Test] In test_patching_cluster, increase the timeout for patching the head node to 3600s (from 1800s) to support slower patching when small instance type is used. --- tests/integration-tests/tests/patching/test_patching.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration-tests/tests/patching/test_patching.py b/tests/integration-tests/tests/patching/test_patching.py index 326172eb5b..42c5fe0640 100644 --- a/tests/integration-tests/tests/patching/test_patching.py +++ b/tests/integration-tests/tests/patching/test_patching.py @@ -30,7 +30,7 @@ ) # Time budget (seconds) for the patching to complete on the head node. -PATCHING_TIMEOUT = 1800 +PATCHING_TIMEOUT = 3600 # Patching flavour passed to patch_node.sh: "minimal" applies only security patches, # "full" applies all available OS package updates. It is selected via the test's From 9b9a7911065f1df8efe4304ffff56c30cda4ea11 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 4 Aug 2026 17:35:44 -0400 Subject: [PATCH 5/9] [Test] In test_patching_cluster, add capped patching flavours to bound the kernel to Lustre support. --- cloudformation/patching/ami-patching.yaml | 6 +- .../test_patching_cluster/patch_node.sh | 101 +++++++++++++++--- 2 files changed, 91 insertions(+), 16 deletions(-) diff --git a/cloudformation/patching/ami-patching.yaml b/cloudformation/patching/ami-patching.yaml index 2987940fbf..443feceb8d 100644 --- a/cloudformation/patching/ami-patching.yaml +++ b/cloudformation/patching/ami-patching.yaml @@ -21,12 +21,16 @@ Parameters: PatchFlavour: Description: >- Patching flavour passed to the patch script. 'minimal' applies only security - errata (smallest set); 'full' applies all available package updates. + errata (smallest set); 'full' applies all available package updates. The + '-capped' variants additionally cap the kernel to the newest version the FSx + Lustre client supports. Type: String Default: minimal AllowedValues: - minimal - full + - minimal-capped + - full-capped Resources: diff --git a/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh b/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh index 9401417203..e520aa9d05 100644 --- a/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh +++ b/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh @@ -2,30 +2,43 @@ # # Patching script. # -# Applies OS package updates using the native package manager, in one of two +# Applies OS package updates using the native package manager, in one of four # flavours (mandatory first argument): # -# minimal: apply only the available *security* patches (smallest set). -# full: apply all available package updates (comprehensive upgrade). +# minimal: apply only the available *security* patches (smallest set). +# full: apply all available package updates (comprehensive upgrade). +# minimal-capped: like minimal, but cap the kernel (see below). +# full-capped: like full, but cap the kernel (see below). # -# Kernel packages are intentionally NOT excluded in either flavour: if an update -# requires a newer kernel, the bump is accepted. A reboot after this script runs -# is required to activate a new kernel. +# Kernel packages are NOT excluded: if an update requires a newer kernel, the bump is +# accepted. A reboot after this script runs is required to activate a new kernel. # -# Usage: patch_node.sh +# The "-capped" variants additionally constrain the kernel so it is never bumped past the +# newest version the FSx Lustre client supports, which prevents FSx mounts from breaking +# after reboot when the client lags the kernel. Capping applies on Ubuntu (per-kernel +# lustre-client-modules package) and RHEL/Rocky (minor-pinned FSx repo); AL2/AL2023 ship +# the Lustre module in-tree with the kernel, so there is nothing to cap there. +# +# Usage: patch_node.sh # # Supports dnf (AL2023/RHEL9/Rocky9), yum (AL2/RHEL8) and apt (Ubuntu). set -euo pipefail if [[ $# -lt 1 ]]; then - echo "ERROR: missing mandatory patching flavour argument (minimal|full)" >&2 + echo "ERROR: missing mandatory patching flavour argument (minimal|full|minimal-capped|full-capped)" >&2 exit 1 fi FLAVOUR="$1" -if [[ "${FLAVOUR}" != "minimal" && "${FLAVOUR}" != "full" ]]; then - echo "ERROR: invalid patching flavour '${FLAVOUR}', expected 'minimal' or 'full'" >&2 - exit 1 -fi +case "${FLAVOUR}" in + minimal | full | minimal-capped | full-capped) ;; + *) + echo "ERROR: invalid patching flavour '${FLAVOUR}', expected one of: minimal, full, minimal-capped, full-capped" >&2 + exit 1 + ;; +esac + +BASE_FLAVOUR="${FLAVOUR%-capped}" +[[ "${FLAVOUR}" == *-capped ]] && CAPPED=true || CAPPED=false # Recover the rpmdb in case a previously killed process left it corrupted (stale # Berkeley DB locks on EL8/AL2 cause "rpmdb open failed" on every rpm/dnf/yum call). @@ -77,6 +90,53 @@ refresh_lustre_client_debian() { || { echo "ERROR: no FSx Lustre client available for kernel ${new_kernel}" >&2; exit 1; } } +# Cap the kernel on Ubuntu (used only by the "-capped" flavours). The FSx Lustre client is a +# per-kernel package (lustre-client-modules-), so the newest such package is the +# highest kernel Lustre supports. Install exactly that kernel and hold the kernel +# meta-packages so neither the security nor the full upgrade can pull a newer one. Assumes +# the apt cache is already refreshed. No-op when FSx Lustre is not configured. +cap_kernel_debian() { + [[ -f /etc/apt/sources.list.d/fsxlustreclientrepo.list ]] || return 0 + local cap + cap=$(apt-cache pkgnames lustre-client-modules- \ + | sed -n 's/^lustre-client-modules-\([0-9].*-aws\)$/\1/p' | sort -V | tail -n1) + [[ -n "${cap}" ]] || { echo "ERROR: could not determine the max FSx Lustre-supported kernel" >&2; exit 1; } + echo "Capping kernel to the max FSx Lustre-supported version: ${cap}" + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + "linux-image-${cap}" "linux-headers-${cap}" "linux-modules-${cap}" "linux-modules-extra-${cap}" + sudo apt-mark hold linux-aws linux-image-aws linux-headers-aws +} + +# Cap the kernel on RHEL/Rocky (used only by the "-capped" flavours). FSx publishes the +# Lustre kmod per EL minor (/el/./), so a kernel is supported only if FSx ships +# a repo for that kernel's minor. Cap to the newest available kernel whose minor FSx already +# publishes and version-lock it so the upgrade stops there. AL2/AL2023 ship the Lustre module +# in-tree with the kernel, so there is nothing to cap. No-op when FSx Lustre is not configured. +cap_kernel_dnf() { + local id + id=$(. /etc/os-release && echo "${ID}") + if [[ ! "${id}" =~ ^(rhel|rocky)$ ]]; then + echo "Kernel cap not applicable on '${id}' (in-tree Lustre module); skipping cap" + return 0 + fi + [[ -f /etc/yum.repos.d/aws-fsx.repo ]] || return 0 + local maj arch cap + maj=$(. /etc/os-release && echo "${VERSION_ID%%.*}") + arch=$(uname -m) + cap=$(sudo dnf -q repoquery kernel --qf '%{version}-%{release}.%{arch}\n' 2>/dev/null | sort -rV | while read -r k; do + minor=$(printf '%s' "${k}" | sed -n "s/.*\.el${maj}_\([0-9]\+\)\..*/\1/p") + [[ -n "${minor}" ]] || continue + if curl -fsL -o /dev/null "https://fsx-lustre-client-repo.s3.amazonaws.com/el/${maj}.${minor}/${arch}/repodata/repomd.xml"; then + echo "${k}" + break + fi + done || true) + [[ -n "${cap}" ]] || { echo "ERROR: could not determine the max FSx Lustre-supported kernel" >&2; exit 1; } + echo "Capping kernel to the max FSx Lustre-supported version: ${cap}" + sudo dnf install -y python3-dnf-plugin-versionlock + sudo dnf versionlock add "kernel-${cap}" "kernel-core-${cap}" "kernel-modules-${cap}" +} + echo "===== Starting system ${FLAVOUR} patching on $(hostname) =====" # Report the running kernel before patching. The kernel after the reboot is # reported separately once the node has rebooted (the reboot is mandatory to @@ -88,7 +148,10 @@ if command -v dnf >/dev/null 2>&1; then fix_rpmdb sudo dnf clean all sudo dnf makecache --refresh -y || true - if [[ "${FLAVOUR}" == "minimal" ]]; then + if [[ "${CAPPED}" == "true" ]]; then + cap_kernel_dnf + fi + if [[ "${BASE_FLAVOUR}" == "minimal" ]]; then # Apply only security errata. Kernel packages are allowed to be upgraded. sudo dnf upgrade --security -y else @@ -100,7 +163,12 @@ elif command -v yum >/dev/null 2>&1; then fix_rpmdb sudo yum clean all sudo yum makecache -y || true - if [[ "${FLAVOUR}" == "minimal" ]]; then + if [[ "${CAPPED}" == "true" ]]; then + # The yum path is AL2, which ships the Lustre module in-tree/co-released with the + # kernel (RHEL/Rocky use the dnf path). Nothing to cap here. + echo "Kernel cap not applicable on this platform (in-tree/co-released Lustre); skipping cap" + fi + if [[ "${BASE_FLAVOUR}" == "minimal" ]]; then # update-minimal --security applies the smallest set of security errata. # Kernel bumps are allowed (no --exclude=kernel*). sudo yum update-minimal --security -y @@ -116,7 +184,10 @@ elif command -v apt-get >/dev/null 2>&1; then # so an exported var alone would not reach the root apt-get process. _envars=(DEBIAN_FRONTEND=noninteractive) sudo "${_envars[@]}" apt-get update -y - if [[ "${FLAVOUR}" == "minimal" ]]; then + if [[ "${CAPPED}" == "true" ]]; then + cap_kernel_debian + fi + if [[ "${BASE_FLAVOUR}" == "minimal" ]]; then # unattended-upgrades applies only the security pocket by default and will # upgrade linux-image-* (kernel) packages when needed. sudo "${_envars[@]}" apt-get install -y unattended-upgrades From 36c61ee57fd9a3fa82f87515e52e86aad517d2c5 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 4 Aug 2026 17:36:19 -0400 Subject: [PATCH 6/9] [Test] Run patching capped on ubuntu/rhel (rotated) and full on al2023, on both develop and released tests. --- tests/integration-tests/configs/develop.yaml | 15 ++++++++++----- tests/integration-tests/configs/released.yaml | 15 ++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/integration-tests/configs/develop.yaml b/tests/integration-tests/configs/develop.yaml index 68746f9232..b466cae5f1 100644 --- a/tests/integration-tests/configs/develop.yaml +++ b/tests/integration-tests/configs/develop.yaml @@ -176,16 +176,21 @@ test-suites: patching: test_patching.py::test_patching_cluster: dimensions: + - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] + instances: ["g4dn.8xlarge"] + oss: [{{ UBUNTU_OS_X86_0 }}] + schedulers: ["slurm"] + flags: ["patching:full-capped"] - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_rhel9 }}] instances: ["g4dn.8xlarge"] - oss: {{ RHEL_OS_X86 }} + oss: [{{ RHEL_OS_X86_0 }}] schedulers: ["slurm"] - flags: ["patching:minimal"] - - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] + flags: ["patching:full-capped"] + - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_alinux2023 }}] instances: ["g4dn.8xlarge"] - oss: {{ NO_RHEL_OS_X86 }} - flags: ["patching:minimal"] + oss: ["alinux2023"] schedulers: ["slurm"] + flags: ["patching:full"] custom_resource: test_cluster_custom_resource.py::test_cluster_create: dimensions: diff --git a/tests/integration-tests/configs/released.yaml b/tests/integration-tests/configs/released.yaml index ae02e04747..6a4cc4f4ff 100644 --- a/tests/integration-tests/configs/released.yaml +++ b/tests/integration-tests/configs/released.yaml @@ -118,16 +118,21 @@ test-suites: # patching: # test_patching.py::test_patching_cluster: # dimensions: + # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] + # instances: ["g4dn.8xlarge"] + # oss: [{{ UBUNTU_OS_X86_0 }}] + # schedulers: ["slurm"] + # flags: ["patching:full-capped"] # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_rhel9 }}] # instances: ["g4dn.8xlarge"] - # oss: {{ RHEL_OS_X86 }} + # oss: [{{ RHEL_OS_X86_0 }}] # schedulers: ["slurm"] - # flags: ["patching:minimal"] - # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] + # flags: ["patching:full-capped"] + # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_alinux2023 }}] # instances: ["g4dn.8xlarge"] - # oss: {{ NO_RHEL_OS_X86 }} + # oss: ["alinux2023"] # schedulers: ["slurm"] - # flags: ["patching:minimal"] + # flags: ["patching:full"] custom_resource: test_cluster_custom_resource.py::test_cluster_1_click: dimensions: From cf2fe902653b573eb00d98e073cbd4fa293fee37 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Tue, 4 Aug 2026 17:39:02 -0400 Subject: [PATCH 7/9] [Test] Reduce patching test capacity reservation to 2 instances --- tests/integration-tests/configs/develop.yaml | 6 +++--- tests/integration-tests/configs/released.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration-tests/configs/develop.yaml b/tests/integration-tests/configs/develop.yaml index b466cae5f1..099c4d9c63 100644 --- a/tests/integration-tests/configs/develop.yaml +++ b/tests/integration-tests/configs/develop.yaml @@ -176,17 +176,17 @@ test-suites: patching: test_patching.py::test_patching_cluster: dimensions: - - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] + - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] instances: ["g4dn.8xlarge"] oss: [{{ UBUNTU_OS_X86_0 }}] schedulers: ["slurm"] flags: ["patching:full-capped"] - - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_rhel9 }}] + - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_rhel9 }}] instances: ["g4dn.8xlarge"] oss: [{{ RHEL_OS_X86_0 }}] schedulers: ["slurm"] flags: ["patching:full-capped"] - - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_alinux2023 }}] + - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_alinux2023 }}] instances: ["g4dn.8xlarge"] oss: ["alinux2023"] schedulers: ["slurm"] diff --git a/tests/integration-tests/configs/released.yaml b/tests/integration-tests/configs/released.yaml index 6a4cc4f4ff..cb5d912b0a 100644 --- a/tests/integration-tests/configs/released.yaml +++ b/tests/integration-tests/configs/released.yaml @@ -118,17 +118,17 @@ test-suites: # patching: # test_patching.py::test_patching_cluster: # dimensions: - # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] + # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] # instances: ["g4dn.8xlarge"] # oss: [{{ UBUNTU_OS_X86_0 }}] # schedulers: ["slurm"] # flags: ["patching:full-capped"] - # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_rhel9 }}] + # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_rhel9 }}] # instances: ["g4dn.8xlarge"] # oss: [{{ RHEL_OS_X86_0 }}] # schedulers: ["slurm"] # flags: ["patching:full-capped"] - # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_alinux2023 }}] + # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_alinux2023 }}] # instances: ["g4dn.8xlarge"] # oss: ["alinux2023"] # schedulers: ["slurm"] From ad516c29e3c75634da786d6c4211b376bdccd6a2 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Thu, 6 Aug 2026 12:56:29 -0400 Subject: [PATCH 8/9] [Test] Use a c5.4xlarge head node in test_patching_cluster Patching the head node in place rebuilds the out-of-tree kernel modules against the new kernel. A known issue in the EFA compilation makes that rebuild memory hungry, so c5.xlarge is not enough. --- tests/integration-tests/tests/patching/test_patching.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration-tests/tests/patching/test_patching.py b/tests/integration-tests/tests/patching/test_patching.py index 42c5fe0640..9463566c22 100644 --- a/tests/integration-tests/tests/patching/test_patching.py +++ b/tests/integration-tests/tests/patching/test_patching.py @@ -42,7 +42,10 @@ # so it is defined in a single place. FSX_LUSTRE_MOUNT_DIR = "/shared-fsxlustre" -HEAD_NODE_INSTANCE = "c5.xlarge" +# The head node is patched in place, which recompiles EFA against the new kernel. Use at least +# the memory of a c5.4xlarge: a known issue in the EFA compilation can cause an OOM error on +# smaller instance types. +HEAD_NODE_INSTANCE = "c5.4xlarge" LOGIN_NODE_INSTANCE = "c5.xlarge" From 13828cdedb82b90ff1a5748ec832392dee59b5e0 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Thu, 6 Aug 2026 17:59:40 -0400 Subject: [PATCH 9/9] [Test] In test_patching_cluster, prevent false positive failures by mod-probing kernel modules that are lazily loaded. --- .../tests/patching/test_patching.py | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/tests/integration-tests/tests/patching/test_patching.py b/tests/integration-tests/tests/patching/test_patching.py index 9463566c22..17df4dff93 100644 --- a/tests/integration-tests/tests/patching/test_patching.py +++ b/tests/integration-tests/tests/patching/test_patching.py @@ -22,6 +22,7 @@ from tests.common.utils import ( COMPUTE_NODE, GPU_JOB_SCRIPT, + HEAD_NODE, LOGIN_NODE, NODE_TYPES, reboot_head_node, @@ -48,6 +49,48 @@ HEAD_NODE_INSTANCE = "c5.4xlarge" LOGIN_NODE_INSTANCE = "c5.xlarge" +# Kernel modules that are loaded lazily (e.g. by ss, systemd-networkd, or other daemons that +# may or may not have run by the time we snapshot). We force-load these after patching so that +# the before/after comparison does not flag them as missing. +# Maintained per OS and per node type. Keep module names alphabetically sorted. +LAZY_KERNEL_MODULES = { + "alinux2023": { + HEAD_NODE: ["tls"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "rhel8": { + HEAD_NODE: ["af_packet_diag", "inet_diag", "tcp_diag", "tls", "udp_diag"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "rhel9": { + HEAD_NODE: ["tls"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "rocky8": { + HEAD_NODE: ["af_packet_diag", "inet_diag", "tcp_diag", "tls", "udp_diag"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "rocky9": { + HEAD_NODE: ["tls"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "ubuntu2204": { + HEAD_NODE: ["tls"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "ubuntu2404": { + HEAD_NODE: ["tls"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, +} + def test_patching_cluster( region, @@ -187,7 +230,7 @@ def test_patching_cluster( _run_gpu_workload(cluster, scheduler_commands_factory, use_login_node=True) # Trigger lazily-loaded modules so that we can compare pre v/s post reboot kernel modules. - _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory) + _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory, os) kernel_modules_after = _collect_loaded_kernel_modules(cluster, scheduler_commands_factory) logging.info("Kernel modules loaded after patching: %s", kernel_modules_after) with soft_assertions(): @@ -258,7 +301,7 @@ def _run_gpu_workload(cluster, scheduler_commands_factory, use_login_node): logging.info("GPU validation job %s submitted from the %s succeeded", job_id, source) -def _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory): +def _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory, os): """Ensure on-demand kernel modules are loaded before the post-reboot snapshot. Some modules load lazily and are absent right after the reboot until something @@ -269,8 +312,9 @@ def _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory): executor = _node_executor(cluster, scheduler_commands_factory, node_type) # Read the FSx for Lustre mountpoint to trigger the loading of lustre kernel module. executor.run_remote_command(f"ls {FSX_LUSTRE_MOUNT_DIR}") - # Load the kernel TLS module directly - executor.run_remote_command("sudo modprobe tls") + # Force-load known lazily-loaded modules for this OS and node type. + for module in LAZY_KERNEL_MODULES.get(os, {}).get(node_type, []): + executor.run_remote_command(f"sudo modprobe {module}") def _collect_loaded_kernel_modules(cluster, scheduler_commands_factory):