diff --git a/cloudformation/patching/ami-patching.yaml b/cloudformation/patching/ami-patching.yaml index 2987940fbf..443feceb8d 100644 --- a/cloudformation/patching/ami-patching.yaml +++ b/cloudformation/patching/ami-patching.yaml @@ -21,12 +21,16 @@ Parameters: PatchFlavour: Description: >- Patching flavour passed to the patch script. 'minimal' applies only security - errata (smallest set); 'full' applies all available package updates. + errata (smallest set); 'full' applies all available package updates. The + '-capped' variants additionally cap the kernel to the newest version the FSx + Lustre client supports. Type: String Default: minimal AllowedValues: - minimal - full + - minimal-capped + - full-capped Resources: diff --git a/tests/integration-tests/configs/develop.yaml b/tests/integration-tests/configs/develop.yaml index c5456038a9..8f51def088 100644 --- a/tests/integration-tests/configs/develop.yaml +++ b/tests/integration-tests/configs/develop.yaml @@ -176,16 +176,21 @@ test-suites: patching: test_patching.py::test_patching_cluster: dimensions: - - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_rhel9 }}] + - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] instances: ["g4dn.8xlarge"] - oss: {{ RHEL_OS_X86 }} + oss: [{{ UBUNTU_OS_X86_0 }}] schedulers: ["slurm"] - flags: ["patching:minimal"] - - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] + flags: ["patching:full-capped"] + - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_rhel9 }}] instances: ["g4dn.8xlarge"] - oss: {{ NO_RHEL_OS_X86 }} - flags: ["patching:minimal"] + oss: [{{ RHEL_OS_X86_0 }}] + schedulers: ["slurm"] + flags: ["patching:full-capped"] + - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_alinux2023 }}] + instances: ["g4dn.8xlarge"] + oss: ["alinux2023"] schedulers: ["slurm"] + flags: ["patching:full"] custom_resource: test_cluster_custom_resource.py::test_cluster_create: dimensions: diff --git a/tests/integration-tests/configs/released.yaml b/tests/integration-tests/configs/released.yaml index ae02e04747..cb5d912b0a 100644 --- a/tests/integration-tests/configs/released.yaml +++ b/tests/integration-tests/configs/released.yaml @@ -118,16 +118,21 @@ test-suites: # patching: # test_patching.py::test_patching_cluster: # dimensions: - # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_rhel9 }}] + # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] # instances: ["g4dn.8xlarge"] - # oss: {{ RHEL_OS_X86 }} + # oss: [{{ UBUNTU_OS_X86_0 }}] # schedulers: ["slurm"] - # flags: ["patching:minimal"] - # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_3_INSTANCES_2_HOURS_NOPG_ubuntu2404 }}] + # flags: ["patching:full-capped"] + # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_rhel9 }}] # instances: ["g4dn.8xlarge"] - # oss: {{ NO_RHEL_OS_X86 }} + # oss: [{{ RHEL_OS_X86_0 }}] # schedulers: ["slurm"] - # flags: ["patching:minimal"] + # flags: ["patching:full-capped"] + # - regions: [{{ g4dn_8xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_NOPG_alinux2023 }}] + # instances: ["g4dn.8xlarge"] + # oss: ["alinux2023"] + # schedulers: ["slurm"] + # flags: ["patching:full"] custom_resource: test_cluster_custom_resource.py::test_cluster_1_click: dimensions: diff --git a/tests/integration-tests/conftest.py b/tests/integration-tests/conftest.py index a0785fdd64..8b9ebb70b0 100644 --- a/tests/integration-tests/conftest.py +++ b/tests/integration-tests/conftest.py @@ -1911,49 +1911,25 @@ def _copy_image(image_id, test_name): logging.error("Delete copied AMI snapshot failed due to %s", e) -def _log_ami_patching_build_output(region, stack_name): # noqa: C901 - """Fetch and log the EC2 Image Builder execution logs for the patched-AMI build. - - Image Builder streams the output of every recipe step (including the patch - script stdout from the ApplyPatches ExecuteBash step) to the CloudWatch log - group /aws/imagebuilder/pcluster-ami-patching-recipe-. Surfacing it - here mirrors how the head node patching output is logged by the test, so the AMI - patching output is available in the test logs too. Any failure to retrieve the - logs is swallowed so it never fails the test. +def _log_ami_patching_build_output(region, stack_name): + """Log the EC2 Image Builder execution logs for the patched-AMI build. + + Image Builder streams the output of every recipe step (including the patch script + output) to the CloudWatch log group + /aws/imagebuilder/pcluster-ami-patching-recipe-. Logging it surfaces the + patch output (and, on failure, the root cause) in the test logs. Any failure to + retrieve the logs is swallowed so it never fails the test. """ log_group_name = f"/aws/imagebuilder/pcluster-ami-patching-recipe-{stack_name}" logs_client = boto3.client("logs", region_name=region) try: - streams = [] - next_token = None - while True: - kwargs = {"logGroupName": log_group_name, "orderBy": "LogStreamName"} - if next_token: - kwargs["nextToken"] = next_token - response = logs_client.describe_log_streams(**kwargs) - streams.extend(response.get("logStreams", [])) - next_token = response.get("nextToken") - if not next_token: - break - if not streams: - logging.info("No AMI patching Image Builder log streams found in log group %s", log_group_name) - return - for stream in streams: - stream_name = stream["logStreamName"] - messages = [] - prev_token = None - event_token = None - while True: - event_kwargs = {"logGroupName": log_group_name, "logStreamName": stream_name, "startFromHead": True} - if event_token: - event_kwargs["nextToken"] = event_token - event_response = logs_client.get_log_events(**event_kwargs) - messages.extend(event["message"] for event in event_response.get("events", [])) - event_token = event_response.get("nextForwardToken") - if event_token == prev_token: - break - prev_token = event_token - logging.info("AMI patching Image Builder log (stream %s):\n%s", stream_name, "\n".join(messages)) + messages = [] + for page in logs_client.get_paginator("filter_log_events").paginate(logGroupName=log_group_name): + messages.extend(event["message"] for event in page.get("events", [])) + if messages: + logging.info("AMI patching Image Builder logs:\n%s", "\n".join(messages)) + else: + logging.info("No AMI patching Image Builder logs found in log group %s", log_group_name) except Exception as e: # noqa: BLE001 logging.warning("Could not retrieve AMI patching Image Builder logs from log group %s: %s", log_group_name, e) @@ -2019,7 +1995,11 @@ def _build(base_ami, builder_instance, flavour="minimal"): ) # create_stack blocks until CREATE_COMPLETE, i.e. until Image Builder has # finished building the patched AMI. - cfn_stacks_factory.create_stack(stack) + try: + cfn_stacks_factory.create_stack(stack) + except Exception: # noqa: BLE001 + _log_ami_patching_build_output(region, stack_name) + raise ami_id = stack.cfn_outputs["AmiId"] built.append((ami_id, stack_name)) logging.info("Patched AMI %s is available", ami_id) diff --git a/tests/integration-tests/framework/tests_configuration/config_renderer.py b/tests/integration-tests/framework/tests_configuration/config_renderer.py index f2fa7a52f0..04f5eacfd1 100644 --- a/tests/integration-tests/framework/tests_configuration/config_renderer.py +++ b/tests/integration-tests/framework/tests_configuration/config_renderer.py @@ -93,6 +93,9 @@ def _get_os_parameters(config=None, args=None): rhel_oss = [os for os in SUPPORTED_OSES if "rhel" in os] _propagate_os_jinja_variables("RHEL_", result, rotation_seed, rhel_oss) + + ubuntu_oss = [os for os in SUPPORTED_OSES if "ubuntu" in os] + _propagate_os_jinja_variables("UBUNTU_", result, rotation_seed, ubuntu_oss) return result diff --git a/tests/integration-tests/tests/patching/test_patching.py b/tests/integration-tests/tests/patching/test_patching.py index f07b9765d9..17df4dff93 100644 --- a/tests/integration-tests/tests/patching/test_patching.py +++ b/tests/integration-tests/tests/patching/test_patching.py @@ -22,6 +22,7 @@ from tests.common.utils import ( COMPUTE_NODE, GPU_JOB_SCRIPT, + HEAD_NODE, LOGIN_NODE, NODE_TYPES, reboot_head_node, @@ -30,7 +31,7 @@ ) # Time budget (seconds) for the patching to complete on the head node. -PATCHING_TIMEOUT = 1800 +PATCHING_TIMEOUT = 3600 # Patching flavour passed to patch_node.sh: "minimal" applies only security patches, # "full" applies all available OS package updates. It is selected via the test's @@ -42,6 +43,54 @@ # so it is defined in a single place. FSX_LUSTRE_MOUNT_DIR = "/shared-fsxlustre" +# The head node is patched in place, which recompiles EFA against the new kernel. Use at least +# the memory of a c5.4xlarge: a known issue in the EFA compilation can cause an OOM error on +# smaller instance types. +HEAD_NODE_INSTANCE = "c5.4xlarge" +LOGIN_NODE_INSTANCE = "c5.xlarge" + +# Kernel modules that are loaded lazily (e.g. by ss, systemd-networkd, or other daemons that +# may or may not have run by the time we snapshot). We force-load these after patching so that +# the before/after comparison does not flag them as missing. +# Maintained per OS and per node type. Keep module names alphabetically sorted. +LAZY_KERNEL_MODULES = { + "alinux2023": { + HEAD_NODE: ["tls"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "rhel8": { + HEAD_NODE: ["af_packet_diag", "inet_diag", "tcp_diag", "tls", "udp_diag"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "rhel9": { + HEAD_NODE: ["tls"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "rocky8": { + HEAD_NODE: ["af_packet_diag", "inet_diag", "tcp_diag", "tls", "udp_diag"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "rocky9": { + HEAD_NODE: ["tls"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "ubuntu2204": { + HEAD_NODE: ["tls"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, + "ubuntu2404": { + HEAD_NODE: ["tls"], + COMPUTE_NODE: ["tls"], + LOGIN_NODE: ["tls"], + }, +} + def test_patching_cluster( region, @@ -83,7 +132,11 @@ def test_patching_cluster( # Start the cluster creation but do not block on it: the AMI patching below # runs concurrently while the cluster comes up. create_config = pcluster_config_reader( - output_file="pcluster.config.create.yaml", login_nodes_count=1, fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR + output_file="pcluster.config.create.yaml", + login_nodes_count=1, + head_node_instance=HEAD_NODE_INSTANCE, + login_node_instance=LOGIN_NODE_INSTANCE, + fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR, ) cluster = clusters_factory(create_config, wait=False) @@ -123,6 +176,8 @@ def test_patching_cluster( output_file="pcluster.config.stop-login.yaml", login_nodes_count=0, base_ami=base_ami_pin, + head_node_instance=HEAD_NODE_INSTANCE, + login_node_instance=LOGIN_NODE_INSTANCE, fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR, ) cluster.update(str(stop_login_config)) @@ -137,6 +192,8 @@ def test_patching_cluster( login_nodes_count=1, base_ami=base_ami_pin, patched_ami=patched_ami, + head_node_instance=HEAD_NODE_INSTANCE, + login_node_instance=LOGIN_NODE_INSTANCE, fsx_lustre_mount_dir=FSX_LUSTRE_MOUNT_DIR, ) cluster.update(str(update_config)) @@ -173,7 +230,7 @@ def test_patching_cluster( _run_gpu_workload(cluster, scheduler_commands_factory, use_login_node=True) # Trigger lazily-loaded modules so that we can compare pre v/s post reboot kernel modules. - _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory) + _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory, os) kernel_modules_after = _collect_loaded_kernel_modules(cluster, scheduler_commands_factory) logging.info("Kernel modules loaded after patching: %s", kernel_modules_after) with soft_assertions(): @@ -244,7 +301,7 @@ def _run_gpu_workload(cluster, scheduler_commands_factory, use_login_node): logging.info("GPU validation job %s submitted from the %s succeeded", job_id, source) -def _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory): +def _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory, os): """Ensure on-demand kernel modules are loaded before the post-reboot snapshot. Some modules load lazily and are absent right after the reboot until something @@ -255,8 +312,9 @@ def _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory): executor = _node_executor(cluster, scheduler_commands_factory, node_type) # Read the FSx for Lustre mountpoint to trigger the loading of lustre kernel module. executor.run_remote_command(f"ls {FSX_LUSTRE_MOUNT_DIR}") - # Load the kernel TLS module directly - executor.run_remote_command("sudo modprobe tls") + # Force-load known lazily-loaded modules for this OS and node type. + for module in LAZY_KERNEL_MODULES.get(os, {}).get(node_type, []): + executor.run_remote_command(f"sudo modprobe {module}") def _collect_loaded_kernel_modules(cluster, scheduler_commands_factory): diff --git a/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh b/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh index 9401417203..e520aa9d05 100644 --- a/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh +++ b/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh @@ -2,30 +2,43 @@ # # Patching script. # -# Applies OS package updates using the native package manager, in one of two +# Applies OS package updates using the native package manager, in one of four # flavours (mandatory first argument): # -# minimal: apply only the available *security* patches (smallest set). -# full: apply all available package updates (comprehensive upgrade). +# minimal: apply only the available *security* patches (smallest set). +# full: apply all available package updates (comprehensive upgrade). +# minimal-capped: like minimal, but cap the kernel (see below). +# full-capped: like full, but cap the kernel (see below). # -# Kernel packages are intentionally NOT excluded in either flavour: if an update -# requires a newer kernel, the bump is accepted. A reboot after this script runs -# is required to activate a new kernel. +# Kernel packages are NOT excluded: if an update requires a newer kernel, the bump is +# accepted. A reboot after this script runs is required to activate a new kernel. # -# Usage: patch_node.sh +# The "-capped" variants additionally constrain the kernel so it is never bumped past the +# newest version the FSx Lustre client supports, which prevents FSx mounts from breaking +# after reboot when the client lags the kernel. Capping applies on Ubuntu (per-kernel +# lustre-client-modules package) and RHEL/Rocky (minor-pinned FSx repo); AL2/AL2023 ship +# the Lustre module in-tree with the kernel, so there is nothing to cap there. +# +# Usage: patch_node.sh # # Supports dnf (AL2023/RHEL9/Rocky9), yum (AL2/RHEL8) and apt (Ubuntu). set -euo pipefail if [[ $# -lt 1 ]]; then - echo "ERROR: missing mandatory patching flavour argument (minimal|full)" >&2 + echo "ERROR: missing mandatory patching flavour argument (minimal|full|minimal-capped|full-capped)" >&2 exit 1 fi FLAVOUR="$1" -if [[ "${FLAVOUR}" != "minimal" && "${FLAVOUR}" != "full" ]]; then - echo "ERROR: invalid patching flavour '${FLAVOUR}', expected 'minimal' or 'full'" >&2 - exit 1 -fi +case "${FLAVOUR}" in + minimal | full | minimal-capped | full-capped) ;; + *) + echo "ERROR: invalid patching flavour '${FLAVOUR}', expected one of: minimal, full, minimal-capped, full-capped" >&2 + exit 1 + ;; +esac + +BASE_FLAVOUR="${FLAVOUR%-capped}" +[[ "${FLAVOUR}" == *-capped ]] && CAPPED=true || CAPPED=false # Recover the rpmdb in case a previously killed process left it corrupted (stale # Berkeley DB locks on EL8/AL2 cause "rpmdb open failed" on every rpm/dnf/yum call). @@ -77,6 +90,53 @@ refresh_lustre_client_debian() { || { echo "ERROR: no FSx Lustre client available for kernel ${new_kernel}" >&2; exit 1; } } +# Cap the kernel on Ubuntu (used only by the "-capped" flavours). The FSx Lustre client is a +# per-kernel package (lustre-client-modules-), so the newest such package is the +# highest kernel Lustre supports. Install exactly that kernel and hold the kernel +# meta-packages so neither the security nor the full upgrade can pull a newer one. Assumes +# the apt cache is already refreshed. No-op when FSx Lustre is not configured. +cap_kernel_debian() { + [[ -f /etc/apt/sources.list.d/fsxlustreclientrepo.list ]] || return 0 + local cap + cap=$(apt-cache pkgnames lustre-client-modules- \ + | sed -n 's/^lustre-client-modules-\([0-9].*-aws\)$/\1/p' | sort -V | tail -n1) + [[ -n "${cap}" ]] || { echo "ERROR: could not determine the max FSx Lustre-supported kernel" >&2; exit 1; } + echo "Capping kernel to the max FSx Lustre-supported version: ${cap}" + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + "linux-image-${cap}" "linux-headers-${cap}" "linux-modules-${cap}" "linux-modules-extra-${cap}" + sudo apt-mark hold linux-aws linux-image-aws linux-headers-aws +} + +# Cap the kernel on RHEL/Rocky (used only by the "-capped" flavours). FSx publishes the +# Lustre kmod per EL minor (/el/./), so a kernel is supported only if FSx ships +# a repo for that kernel's minor. Cap to the newest available kernel whose minor FSx already +# publishes and version-lock it so the upgrade stops there. AL2/AL2023 ship the Lustre module +# in-tree with the kernel, so there is nothing to cap. No-op when FSx Lustre is not configured. +cap_kernel_dnf() { + local id + id=$(. /etc/os-release && echo "${ID}") + if [[ ! "${id}" =~ ^(rhel|rocky)$ ]]; then + echo "Kernel cap not applicable on '${id}' (in-tree Lustre module); skipping cap" + return 0 + fi + [[ -f /etc/yum.repos.d/aws-fsx.repo ]] || return 0 + local maj arch cap + maj=$(. /etc/os-release && echo "${VERSION_ID%%.*}") + arch=$(uname -m) + cap=$(sudo dnf -q repoquery kernel --qf '%{version}-%{release}.%{arch}\n' 2>/dev/null | sort -rV | while read -r k; do + minor=$(printf '%s' "${k}" | sed -n "s/.*\.el${maj}_\([0-9]\+\)\..*/\1/p") + [[ -n "${minor}" ]] || continue + if curl -fsL -o /dev/null "https://fsx-lustre-client-repo.s3.amazonaws.com/el/${maj}.${minor}/${arch}/repodata/repomd.xml"; then + echo "${k}" + break + fi + done || true) + [[ -n "${cap}" ]] || { echo "ERROR: could not determine the max FSx Lustre-supported kernel" >&2; exit 1; } + echo "Capping kernel to the max FSx Lustre-supported version: ${cap}" + sudo dnf install -y python3-dnf-plugin-versionlock + sudo dnf versionlock add "kernel-${cap}" "kernel-core-${cap}" "kernel-modules-${cap}" +} + echo "===== Starting system ${FLAVOUR} patching on $(hostname) =====" # Report the running kernel before patching. The kernel after the reboot is # reported separately once the node has rebooted (the reboot is mandatory to @@ -88,7 +148,10 @@ if command -v dnf >/dev/null 2>&1; then fix_rpmdb sudo dnf clean all sudo dnf makecache --refresh -y || true - if [[ "${FLAVOUR}" == "minimal" ]]; then + if [[ "${CAPPED}" == "true" ]]; then + cap_kernel_dnf + fi + if [[ "${BASE_FLAVOUR}" == "minimal" ]]; then # Apply only security errata. Kernel packages are allowed to be upgraded. sudo dnf upgrade --security -y else @@ -100,7 +163,12 @@ elif command -v yum >/dev/null 2>&1; then fix_rpmdb sudo yum clean all sudo yum makecache -y || true - if [[ "${FLAVOUR}" == "minimal" ]]; then + if [[ "${CAPPED}" == "true" ]]; then + # The yum path is AL2, which ships the Lustre module in-tree/co-released with the + # kernel (RHEL/Rocky use the dnf path). Nothing to cap here. + echo "Kernel cap not applicable on this platform (in-tree/co-released Lustre); skipping cap" + fi + if [[ "${BASE_FLAVOUR}" == "minimal" ]]; then # update-minimal --security applies the smallest set of security errata. # Kernel bumps are allowed (no --exclude=kernel*). sudo yum update-minimal --security -y @@ -116,7 +184,10 @@ elif command -v apt-get >/dev/null 2>&1; then # so an exported var alone would not reach the root apt-get process. _envars=(DEBIAN_FRONTEND=noninteractive) sudo "${_envars[@]}" apt-get update -y - if [[ "${FLAVOUR}" == "minimal" ]]; then + if [[ "${CAPPED}" == "true" ]]; then + cap_kernel_debian + fi + if [[ "${BASE_FLAVOUR}" == "minimal" ]]; then # unattended-upgrades applies only the security pocket by default and will # upgrade linux-image-* (kernel) packages when needed. sudo "${_envars[@]}" apt-get install -y unattended-upgrades diff --git a/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/pcluster.config.yaml b/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/pcluster.config.yaml index f780ca2f25..72887fe1d7 100644 --- a/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/pcluster.config.yaml +++ b/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/pcluster.config.yaml @@ -4,7 +4,7 @@ Image: CustomAmi: {{ base_ami }} {% endif %} HeadNode: - InstanceType: {{ instance }} + InstanceType: {{ head_node_instance }} Networking: SubnetId: {{ public_subnet_id }} ElasticIp: true @@ -15,7 +15,7 @@ HeadNode: LoginNodes: Pools: - Name: login1 - InstanceType: {{ instance }} + InstanceType: {{ login_node_instance }} Count: {{ login_nodes_count }} GracetimePeriod: 3 {% if patched_ami %} @@ -50,7 +50,10 @@ Scheduling: - {{ private_subnet_id }} ComputeResources: - Name: cr1 - InstanceType: {{ instance }} + Instances: + {% for instance_type in flexible_instance_types %} + - InstanceType: {{ instance_type }} + {% endfor %} MinCount: 1 MaxCount: 1 Efa: