aws · gmarciani · Aug 7, 2026 · Aug 4, 2026 · Aug 4, 2026 · Aug 4, 2026
@@ -21,12 +21,16 @@ Parameters:
   PatchFlavour:
     Description: >-
       Patching flavour passed to the patch script. 'minimal' applies only security
-      errata (smallest set); 'full' applies all available package updates.
+      errata (smallest set); 'full' applies all available package updates. The
+      '-capped' variants additionally cap the kernel to the newest version the FSx
+      Lustre client supports.
     Type: String
     Default: minimal
     AllowedValues:
       - minimal
       - full
+      - minimal-capped
+      - full-capped
 
 Resources:
 

@@ -176,16 +176,21 @@ test-suites:
   patching:
     test_patching.py::test_patching_cluster:
       dimensions:
-        - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_rhel9 }}]
+        - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}]
           instances: ["g4dn.8xlarge"]
-          oss: {{ RHEL_OS_X86 }}
+          oss: [{{ UBUNTU_OS_X86_0 }}]
           schedulers: ["slurm"]
-          flags: ["patching:minimal"]
-        - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}]
+          flags: ["patching:full-capped"]
+        - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_rhel9 }}]
           instances: ["g4dn.8xlarge"]
-          oss: {{ NO_RHEL_OS_X86 }}
-          flags: ["patching:minimal"]
+          oss: [{{ RHEL_OS_X86_0 }}]
+          schedulers: ["slurm"]
+          flags: ["patching:full-capped"]
+        - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_alinux2023 }}]
+          instances: ["g4dn.8xlarge"]
+          oss: ["alinux2023"]
           schedulers: ["slurm"]
+          flags: ["patching:full"]
   custom_resource:
     test_cluster_custom_resource.py::test_cluster_create:
       dimensions:

@@ -118,16 +118,21 @@ test-suites:
   # patching:
   #   test_patching.py::test_patching_cluster:
   #     dimensions:
-  #       - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_rhel9 }}]
+  #       - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}]
   #         instances: ["g4dn.8xlarge"]
-  #         oss: {{ RHEL_OS_X86 }}
+  #         oss: [{{ UBUNTU_OS_X86_0 }}]
   #         schedulers: ["slurm"]
-  #         flags: ["patching:minimal"]
-  #       - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}]
+  #         flags: ["patching:full-capped"]
+  #       - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_rhel9 }}]
   #         instances: ["g4dn.8xlarge"]
-  #         oss: {{ NO_RHEL_OS_X86 }}
+  #         oss: [{{ RHEL_OS_X86_0 }}]
   #         schedulers: ["slurm"]
-  #         flags: ["patching:minimal"]
+  #         flags: ["patching:full-capped"]
+  #       - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_alinux2023 }}]
+  #         instances: ["g4dn.8xlarge"]
+  #         oss: ["alinux2023"]
+  #         schedulers: ["slurm"]
+  #         flags: ["patching:full"]
   custom_resource:
     test_cluster_custom_resource.py::test_cluster_1_click:
       dimensions:

@@ -1911,49 +1911,25 @@ def _copy_image(image_id, test_name):
             logging.error("Delete copied AMI snapshot failed due to %s", e)
 
 
-def _log_ami_patching_build_output(region, stack_name):  # noqa: C901
-    """Fetch and log the EC2 Image Builder execution logs for the patched-AMI build.
-
-    Image Builder streams the output of every recipe step (including the patch
-    script stdout from the ApplyPatches ExecuteBash step) to the CloudWatch log
-    group /aws/imagebuilder/pcluster-ami-patching-recipe-<stack_name>. Surfacing it
-    here mirrors how the head node patching output is logged by the test, so the AMI
-    patching output is available in the test logs too. Any failure to retrieve the
-    logs is swallowed so it never fails the test.
+def _log_ami_patching_build_output(region, stack_name):
+    """Log the EC2 Image Builder execution logs for the patched-AMI build.
+
+    Image Builder streams the output of every recipe step (including the patch script
+    output) to the CloudWatch log group
+    /aws/imagebuilder/pcluster-ami-patching-recipe-<stack_name>. Logging it surfaces the
+    patch output (and, on failure, the root cause) in the test logs. Any failure to
+    retrieve the logs is swallowed so it never fails the test.
     """
     log_group_name = f"/aws/imagebuilder/pcluster-ami-patching-recipe-{stack_name}"
     logs_client = boto3.client("logs", region_name=region)
     try:
-        streams = []
-        next_token = None
-        while True:
-            kwargs = {"logGroupName": log_group_name, "orderBy": "LogStreamName"}
-            if next_token:
-                kwargs["nextToken"] = next_token
-            response = logs_client.describe_log_streams(**kwargs)
-            streams.extend(response.get("logStreams", []))
-            next_token = response.get("nextToken")
-            if not next_token:
-                break
-        if not streams:
-            logging.info("No AMI patching Image Builder log streams found in log group %s", log_group_name)
-            return
-        for stream in streams:
-            stream_name = stream["logStreamName"]
-            messages = []
-            prev_token = None
-            event_token = None
-            while True:
-                event_kwargs = {"logGroupName": log_group_name, "logStreamName": stream_name, "startFromHead": True}
-                if event_token:
-                    event_kwargs["nextToken"] = event_token
-                event_response = logs_client.get_log_events(**event_kwargs)
-                messages.extend(event["message"] for event in event_response.get("events", []))
-                event_token = event_response.get("nextForwardToken")
-                if event_token == prev_token:
-                    break
-                prev_token = event_token
-            logging.info("AMI patching Image Builder log (stream %s):\n%s", stream_name, "\n".join(messages))
+        messages = []
+        for page in logs_client.get_paginator("filter_log_events").paginate(logGroupName=log_group_name):
+            messages.extend(event["message"] for event in page.get("events", []))
+        if messages:
+            logging.info("AMI patching Image Builder logs:\n%s", "\n".join(messages))
+        else:
+            logging.info("No AMI patching Image Builder logs found in log group %s", log_group_name)
     except Exception as e:  # noqa: BLE001
         logging.warning("Could not retrieve AMI patching Image Builder logs from log group %s: %s", log_group_name, e)
 
@@ -2019,7 +1995,11 @@ def _build(base_ami, builder_instance, flavour="minimal"):
         )
         # create_stack blocks until CREATE_COMPLETE, i.e. until Image Builder has
         # finished building the patched AMI.
-        cfn_stacks_factory.create_stack(stack)
+        try:
+            cfn_stacks_factory.create_stack(stack)
+        except Exception:  # noqa: BLE001
+            _log_ami_patching_build_output(region, stack_name)
+            raise
         ami_id = stack.cfn_outputs["AmiId"]
         built.append((ami_id, stack_name))
         logging.info("Patched AMI %s is available", ami_id)

@@ -93,6 +93,9 @@ def _get_os_parameters(config=None, args=None):
 
     rhel_oss = [os for os in SUPPORTED_OSES if "rhel" in os]
     _propagate_os_jinja_variables("RHEL_", result, rotation_seed, rhel_oss)
+
+    ubuntu_oss = [os for os in SUPPORTED_OSES if "ubuntu" in os]
+    _propagate_os_jinja_variables("UBUNTU_", result, rotation_seed, ubuntu_oss)
     return result
 
 

@@ -30,7 +30,7 @@
 )
 
 # Time budget (seconds) for the patching to complete on the head node.
-PATCHING_TIMEOUT = 1800
+PATCHING_TIMEOUT = 3600
 
 # Patching flavour passed to patch_node.sh: "minimal" applies only security patches,
 # "full" applies all available OS package updates. It is selected via the test's
@@ -42,6 +42,12 @@
 # so it is defined in a single place.
 FSX_LUSTRE_MOUNT_DIR = "/shared-fsxlustre"
 
+# The head node is patched in place, which recompiles EFA against the new kernel. Use at least
+# the memory of a c5.4xlarge: a known issue in the EFA compilation can cause an OOM error on
+# smaller instance types.
+HEAD_NODE_INSTANCE = "c5.4xlarge"
+LOGIN_NODE_INSTANCE = "c5.xlarge"
+
 
 def test_patching_cluster(
     region,
@@ -83,7 +89,11 @@ def test_patching_cluster(
     # Start the cluster creation but do not block on it: the AMI patching below
     # runs concurrently while the cluster comes up.
     create_config = pcluster_config_reader(
-        output_file="pcluster.config.create.yaml", login_nodes_count=1, fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR
+        output_file="pcluster.config.create.yaml",
+        login_nodes_count=1,
+        head_node_instance=HEAD_NODE_INSTANCE,
+        login_node_instance=LOGIN_NODE_INSTANCE,
+        fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR,
     )
     cluster = clusters_factory(create_config, wait=False)
 
@@ -123,6 +133,8 @@ def test_patching_cluster(
         output_file="pcluster.config.stop-login.yaml",
         login_nodes_count=0,
         base_ami=base_ami_pin,
+        head_node_instance=HEAD_NODE_INSTANCE,
+        login_node_instance=LOGIN_NODE_INSTANCE,
         fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR,
     )
     cluster.update(str(stop_login_config))
@@ -137,6 +149,8 @@ def test_patching_cluster(
         login_nodes_count=1,
         base_ami=base_ami_pin,
         patched_ami=patched_ami,
+        head_node_instance=HEAD_NODE_INSTANCE,
+        login_node_instance=LOGIN_NODE_INSTANCE,
         fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR,
     )
     cluster.update(str(update_config))

@@ -2,30 +2,43 @@
 #
 # Patching script.
 #
-# Applies OS package updates using the native package manager, in one of two
+# Applies OS package updates using the native package manager, in one of four
 # flavours (mandatory first argument):
 #
-#   minimal: apply only the available *security* patches (smallest set).
-#   full:    apply all available package updates (comprehensive upgrade).
+#   minimal:        apply only the available *security* patches (smallest set).
+#   full:           apply all available package updates (comprehensive upgrade).
+#   minimal-capped: like minimal, but cap the kernel (see below).
+#   full-capped:    like full, but cap the kernel (see below).
 #
-# Kernel packages are intentionally NOT excluded in either flavour: if an update
-# requires a newer kernel, the bump is accepted. A reboot after this script runs
-# is required to activate a new kernel.
+# Kernel packages are NOT excluded: if an update requires a newer kernel, the bump is
+# accepted. A reboot after this script runs is required to activate a new kernel.
 #
-# Usage: patch_node.sh <minimal|full>
+# The "-capped" variants additionally constrain the kernel so it is never bumped past the
+# newest version the FSx Lustre client supports, which prevents FSx mounts from breaking
+# after reboot when the client lags the kernel. Capping applies on Ubuntu (per-kernel
+# lustre-client-modules package) and RHEL/Rocky (minor-pinned FSx repo); AL2/AL2023 ship
+# the Lustre module in-tree with the kernel, so there is nothing to cap there.
+#
+# Usage: patch_node.sh <minimal|full|minimal-capped|full-capped>
 #
 # Supports dnf (AL2023/RHEL9/Rocky9), yum (AL2/RHEL8) and apt (Ubuntu).
 set -euo pipefail
 
 if [[ $# -lt 1 ]]; then
-    echo "ERROR: missing mandatory patching flavour argument (minimal|full)" >&2
+    echo "ERROR: missing mandatory patching flavour argument (minimal|full|minimal-capped|full-capped)" >&2
     exit 1
 fi
 FLAVOUR="$1"
-if [[ "${FLAVOUR}" != "minimal" && "${FLAVOUR}" != "full" ]]; then
-    echo "ERROR: invalid patching flavour '${FLAVOUR}', expected 'minimal' or 'full'" >&2
-    exit 1
-fi
+case "${FLAVOUR}" in
+    minimal | full | minimal-capped | full-capped) ;;
+    *)
+        echo "ERROR: invalid patching flavour '${FLAVOUR}', expected one of: minimal, full, minimal-capped, full-capped" >&2
+        exit 1
+        ;;
+esac
+
+BASE_FLAVOUR="${FLAVOUR%-capped}"
+[[ "${FLAVOUR}" == *-capped ]] && CAPPED=true || CAPPED=false
 
 # Recover the rpmdb in case a previously killed process left it corrupted (stale
 # Berkeley DB locks on EL8/AL2 cause "rpmdb open failed" on every rpm/dnf/yum call).
@@ -77,6 +90,53 @@ refresh_lustre_client_debian() {
         || { echo "ERROR: no FSx Lustre client available for kernel ${new_kernel}" >&2; exit 1; }
 }
 
+# Cap the kernel on Ubuntu (used only by the "-capped" flavours). The FSx Lustre client is a
+# per-kernel package (lustre-client-modules-<uname-r>), so the newest such package is the
+# highest kernel Lustre supports. Install exactly that kernel and hold the kernel
+# meta-packages so neither the security nor the full upgrade can pull a newer one. Assumes
+# the apt cache is already refreshed. No-op when FSx Lustre is not configured.
+cap_kernel_debian() {
+    [[ -f /etc/apt/sources.list.d/fsxlustreclientrepo.list ]] || return 0
+    local cap
+    cap=$(apt-cache pkgnames lustre-client-modules- \
+        | sed -n 's/^lustre-client-modules-\([0-9].*-aws\)$/\1/p' | sort -V | tail -n1)
+    [[ -n "${cap}" ]] || { echo "ERROR: could not determine the max FSx Lustre-supported kernel" >&2; exit 1; }
+    echo "Capping kernel to the max FSx Lustre-supported version: ${cap}"
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        "linux-image-${cap}" "linux-headers-${cap}" "linux-modules-${cap}" "linux-modules-extra-${cap}"
+    sudo apt-mark hold linux-aws linux-image-aws linux-headers-aws
+}
+
+# Cap the kernel on RHEL/Rocky (used only by the "-capped" flavours). FSx publishes the
+# Lustre kmod per EL minor (/el/<major>.<minor>/), so a kernel is supported only if FSx ships
+# a repo for that kernel's minor. Cap to the newest available kernel whose minor FSx already
+# publishes and version-lock it so the upgrade stops there. AL2/AL2023 ship the Lustre module
+# in-tree with the kernel, so there is nothing to cap. No-op when FSx Lustre is not configured.
+cap_kernel_dnf() {
+    local id
+    id=$(. /etc/os-release && echo "${ID}")
+    if [[ ! "${id}" =~ ^(rhel|rocky)$ ]]; then
+        echo "Kernel cap not applicable on '${id}' (in-tree Lustre module); skipping cap"
+        return 0
+    fi
+    [[ -f /etc/yum.repos.d/aws-fsx.repo ]] || return 0
+    local maj arch cap
+    maj=$(. /etc/os-release && echo "${VERSION_ID%%.*}")
+    arch=$(uname -m)
+    cap=$(sudo dnf -q repoquery kernel --qf '%{version}-%{release}.%{arch}\n' 2>/dev/null | sort -rV | while read -r k; do
+        minor=$(printf '%s' "${k}" | sed -n "s/.*\.el${maj}_\([0-9]\+\)\..*/\1/p")
+        [[ -n "${minor}" ]] || continue
+        if curl -fsL -o /dev/null "https://fsx-lustre-client-repo.s3.amazonaws.com/el/${maj}.${minor}/${arch}/repodata/repomd.xml"; then
+            echo "${k}"
+            break
+        fi
+    done || true)
+    [[ -n "${cap}" ]] || { echo "ERROR: could not determine the max FSx Lustre-supported kernel" >&2; exit 1; }
+    echo "Capping kernel to the max FSx Lustre-supported version: ${cap}"
+    sudo dnf install -y python3-dnf-plugin-versionlock
+    sudo dnf versionlock add "kernel-${cap}" "kernel-core-${cap}" "kernel-modules-${cap}"
+}
+
 echo "===== Starting system ${FLAVOUR} patching on $(hostname) ====="
 # Report the running kernel before patching. The kernel after the reboot is
 # reported separately once the node has rebooted (the reboot is mandatory to
@@ -88,7 +148,10 @@ if command -v dnf >/dev/null 2>&1; then
     fix_rpmdb
     sudo dnf clean all
     sudo dnf makecache --refresh -y  || true
-    if [[ "${FLAVOUR}" == "minimal" ]]; then
+    if [[ "${CAPPED}" == "true" ]]; then
+        cap_kernel_dnf
+    fi
+    if [[ "${BASE_FLAVOUR}" == "minimal" ]]; then
         # Apply only security errata. Kernel packages are allowed to be upgraded.
         sudo dnf upgrade --security -y
     else
@@ -100,7 +163,12 @@ elif command -v yum >/dev/null 2>&1; then
     fix_rpmdb
     sudo yum clean all
     sudo yum makecache -y || true
-    if [[ "${FLAVOUR}" == "minimal" ]]; then
+    if [[ "${CAPPED}" == "true" ]]; then
+        # The yum path is AL2, which ships the Lustre module in-tree/co-released with the
+        # kernel (RHEL/Rocky use the dnf path). Nothing to cap here.
+        echo "Kernel cap not applicable on this platform (in-tree/co-released Lustre); skipping cap"
+    fi
+    if [[ "${BASE_FLAVOUR}" == "minimal" ]]; then
         # update-minimal --security applies the smallest set of security errata.
         # Kernel bumps are allowed (no --exclude=kernel*).
         sudo yum update-minimal --security -y
@@ -116,7 +184,10 @@ elif command -v apt-get >/dev/null 2>&1; then
     # so an exported var alone would not reach the root apt-get process.
     _envars=(DEBIAN_FRONTEND=noninteractive)
     sudo "${_envars[@]}" apt-get update -y
-    if [[ "${FLAVOUR}" == "minimal" ]]; then
+    if [[ "${CAPPED}" == "true" ]]; then
+        cap_kernel_debian
+    fi
+    if [[ "${BASE_FLAVOUR}" == "minimal" ]]; then
         # unattended-upgrades applies only the security pocket by default and will
         # upgrade linux-image-* (kernel) packages when needed.
         sudo "${_envars[@]}" apt-get install -y unattended-upgrades

@@ -4,7 +4,7 @@ Image:
   CustomAmi: {{ base_ami }}
   {% endif %}
 HeadNode:
-  InstanceType: {{ instance }}
+  InstanceType: {{ head_node_instance }}
   Networking:
     SubnetId: {{ public_subnet_id }}
     ElasticIp: true
@@ -15,7 +15,7 @@ HeadNode:
 LoginNodes:
   Pools:
     - Name: login1
-      InstanceType: {{ instance }}
+      InstanceType: {{ login_node_instance }}
       Count: {{ login_nodes_count }}
       GracetimePeriod: 3
       {% if patched_ami %}
@@ -50,7 +50,10 @@ Scheduling:
           - {{ private_subnet_id }}
       ComputeResources:
         - Name: cr1
-          InstanceType: {{ instance }}
+          Instances:
+            {% for instance_type in flexible_instance_types %}
+            - InstanceType: {{ instance_type }}
+            {% endfor %}
           MinCount: 1
           MaxCount: 1
           Efa: