From 2f531765964725508459cee99b453060eccf9f56 Mon Sep 17 00:00:00 2001 From: Srikanth Myakam <374767+SRIKKANTH@users.noreply.github.com> Date: Mon, 3 Aug 2026 21:10:26 +0530 Subject: [PATCH 1/4] Add premium data-disk performance validation against rated SKU maximum Summary ------- Adds a post-run validation step for the premium data-disk fio performance tests that compares the measured random-read results against the Azure VM SKU's rated disk maximum, so the tests fail when a VM under-performs its published disk limits instead of only recording numbers. Changes ------- lisa/microsoft/testsuites/performance/common.py - Add `_get_azure_sku_capabilities()` to resolve the Azure SKU capability name/value map for the node's VM size (returns None on non-Azure platforms or when capabilities are unavailable). - Add `check_premium_datadisks_performance()` which: - Ignores IO depths at or below the saturation threshold (`PREMIUM_DATADISK_MIN_IODEPTH = 8`). - Averages the random-read value per IO depth across iterations. - Compares against `UncachedDiskIOPS` for 4K (IOPS-bound) runs and `UncachedDiskBytesPerSecond` for 1024K (bandwidth-bound) runs, converting IOPS to bytes/sec via block size where needed. - Requires every qualifying IO depth to reach at least `PREMIUM_DATADISK_PASS_RATIO` (95%) of the rated maximum. - Raise `SkippedException` (not a failure) when rated disk performance or the required capability is not published for the VM size/platform. - Derive the pass percentage in log/assertion messages from `PREMIUM_DATADISK_PASS_RATIO` instead of a hardcoded 95%. lisa/microsoft/testsuites/performance/storageperf.py - Capture the messages returned by `perf_premium_datadisks(...)` and invoke `check_premium_datadisks_performance()` in the 4K, 1024K, and io_uring (4K/1024K) premium data-disk test cases. Notes ----- - The rated-max validation only applies on Azure; other platforms are skipped. --- .../testsuites/performance/common.py | 171 +++++++++++++++++- .../testsuites/performance/storageperf.py | 13 +- 2 files changed, 177 insertions(+), 7 deletions(-) diff --git a/lisa/microsoft/testsuites/performance/common.py b/lisa/microsoft/testsuites/performance/common.py index 1680ee475d..2ee65345ef 100644 --- a/lisa/microsoft/testsuites/performance/common.py +++ b/lisa/microsoft/testsuites/performance/common.py @@ -131,7 +131,7 @@ def perf_disk( overwrite: bool = False, ioengine: IoEngine = IoEngine.LIBAIO, cwd: Optional[pathlib.PurePath] = None, -) -> None: +) -> List[DiskPerformanceMessage]: fio_result_list: List[FIOResult] = [] fio = node.tools[Fio] numjobiterator = 0 @@ -188,6 +188,7 @@ def perf_disk( ) for fio_message in fio_messages: notifier.notify(fio_message) + return fio_messages def get_nic_datapath(node: Node) -> str: @@ -926,7 +927,7 @@ def perf_premium_datadisks( start_iodepth: int = 1, max_iodepth: int = 256, ioengine: IoEngine = IoEngine.LIBAIO, -) -> None: +) -> List[DiskPerformanceMessage]: disk = node.features[Disk] data_disks = disk.get_raw_data_disks() disk_count = len(data_disks) @@ -937,7 +938,7 @@ def perf_premium_datadisks( filename = ":".join(partition_disks) cpu = node.tools[Lscpu] thread_count = cpu.get_thread_count() - perf_disk( + return perf_disk( node, start_iodepth, max_iodepth, @@ -956,6 +957,170 @@ def perf_premium_datadisks( ) +# Premium data-disk performance validation constants. +# IO depths at or below this value cannot saturate the device and are ignored. +PREMIUM_DATADISK_MIN_IODEPTH = 8 +# Every qualifying IO depth must reach at least this fraction of the rated max. +PREMIUM_DATADISK_PASS_RATIO = 0.95 +# Block sizes at or above this value (in KiB) are bandwidth-bound rather than +# IOPS-bound, so they are validated against the rated disk throughput. +PREMIUM_DATADISK_BANDWIDTH_BLOCK_SIZE_KB = 1024 + + +def _get_azure_sku_capabilities( + test_result: TestResult, node: Node +) -> Optional[Dict[str, str]]: + """Return the Azure SKU capability name/value map for the node's VM size. + + Returns ``None`` when the platform is not Azure or the VM size capabilities + are unavailable (for example on non-Azure platforms). + """ + try: + from lisa.sut_orchestrator import AZURE + from lisa.sut_orchestrator.azure.common import AzureNodeSchema + from lisa.sut_orchestrator.azure.platform_ import AzurePlatform + except ImportError: + return None + + environment = test_result.environment + if environment is None: + return None + platform = environment.platform + if not isinstance(platform, AzurePlatform): + return None + + node_runbook = node.capability.get_extended_runbook(AzureNodeSchema, AZURE) + vm_size = node_runbook.vm_size + location = node_runbook.location + if not vm_size or not location: + return None + + location_info = platform.get_location_info(location, node.log) + azure_capability = location_info.capabilities.get(vm_size) + if azure_capability is None: + return None + caps = azure_capability.resource_sku.get("capabilities", []) + return { + cap["name"]: cap["value"] + for cap in caps + if "name" in cap and "value" in cap + } + + +def check_premium_datadisks_performance( + test_result: TestResult, + perf_messages: List[DiskPerformanceMessage], +) -> None: + """Validate premium data-disk random-read performance against the rated max. + + Rules: + - Only IO depths greater than 8 are considered; lower queue depths cannot + saturate the device. + - For each qualifying IO depth, the random-read value is averaged across all + iterations at that depth to get one value per depth. + - The per-depth values are compared against the VM SKU's rated maximum. For + 4K (IOPS-bound) tests the rated maximum is ``UncachedDiskIOPS``; for 1024K + (bandwidth-bound) tests it is ``UncachedDiskBytesPerSecond`` and the + random-read IOPS are converted to bytes/sec using the block size. + - The VM passes when, at every IO depth above 8, the best observed value + reaches at least 95% of the rated maximum. + """ + environment = test_result.environment + assert environment, "fail to get environment from testresult" + node = cast(RemoteNode, environment.nodes[0]) + log = node.log + + sku_caps = _get_azure_sku_capabilities(test_result, node) + if sku_caps is None: + raise SkippedException( + "Skipping premium data-disk performance validation: rated disk " + "performance is not available for this VM size or platform. This " + "check requires an Azure VM size that publishes " + "UncachedDiskIOPS/UncachedDiskBytesPerSecond." + ) + + # Determine whether the run is bandwidth-bound (1024K) or IOPS-bound (4K) + # from the block size recorded on the messages. + block_size_kb = 0 + for message in perf_messages: + if message.block_size: + block_size_kb = message.block_size + break + bandwidth_bound = block_size_kb >= PREMIUM_DATADISK_BANDWIDTH_BLOCK_SIZE_KB + + if bandwidth_bound: + cap_name = "UncachedDiskBytesPerSecond" + rated_unit = "bytes/sec" + else: + cap_name = "UncachedDiskIOPS" + rated_unit = "IOPS" + if cap_name not in sku_caps: + raise SkippedException( + f"Skipping premium data-disk performance validation: rated disk " + f"capability '{cap_name}' is not published for this VM size. This " + f"check requires a VM size that publishes '{cap_name}'." + ) + rated_max = float(sku_caps[cap_name]) + block_size_bytes = block_size_kb * 1024 + + # Average the random-read value per IO depth across iterations, keeping only + # depths above the saturation threshold. + randread_by_iodepth: Dict[int, List[float]] = {} + for message in perf_messages: + if message.iodepth <= PREMIUM_DATADISK_MIN_IODEPTH: + continue + randread_iops = float(message.randread_iops) + if bandwidth_bound: + # Convert random-read IOPS to bytes/sec so it can be compared to the + # rated throughput. + measured = randread_iops * block_size_bytes + else: + measured = randread_iops + randread_by_iodepth.setdefault(message.iodepth, []).append(measured) + + if not randread_by_iodepth: + raise LisaException( + f"No random-read samples were found at an IO depth greater than " + f"{PREMIUM_DATADISK_MIN_IODEPTH}. Verify the test produced results " + f"for the expected IO depths." + ) + + required = rated_max * PREMIUM_DATADISK_PASS_RATIO + pass_percent = PREMIUM_DATADISK_PASS_RATIO * 100 + failing_iodepths: List[str] = [] + per_depth_percentages: List[float] = [] + for iodepth in sorted(randread_by_iodepth): + # The best observed value at this depth (max across iterations). + best = max(randread_by_iodepth[iodepth]) + percent_of_rated = best / rated_max * 100 + per_depth_percentages.append(percent_of_rated) + log.info( + f"premium data-disk check on {node.name} at iodepth {iodepth}: " + f"best random-read {best:.1f} {rated_unit} = " + f"{percent_of_rated:.1f}% of rated maximum {rated_max:.1f} " + f"{rated_unit} (required >= {pass_percent:.0f}%)." + ) + if best < required: + failing_iodepths.append( + f"iodepth {iodepth}: {best:.1f} {rated_unit} " + f"({percent_of_rated:.1f}%)" + ) + + average_percent = sum(per_depth_percentages) / len(per_depth_percentages) + log.info( + f"premium data-disk check on {node.name}: average random-read across " + f"{len(per_depth_percentages)} IO depths above " + f"{PREMIUM_DATADISK_MIN_IODEPTH} is {average_percent:.1f}% of the rated " + f"maximum {rated_max:.1f} {rated_unit}." + ) + assert_that(failing_iodepths).described_as( + f"every IO depth above {PREMIUM_DATADISK_MIN_IODEPTH} must reach at " + f"least {pass_percent:.0f}% ({required:.1f} {rated_unit}) of the rated " + f"maximum ({rated_max:.1f} {rated_unit}), but these IO depths fell " + f"short: {failing_iodepths}" + ).is_empty() + + def perf_resource_disks( node: Node, test_result: TestResult, diff --git a/lisa/microsoft/testsuites/performance/storageperf.py b/lisa/microsoft/testsuites/performance/storageperf.py index b22cc91727..3cc57816f9 100644 --- a/lisa/microsoft/testsuites/performance/storageperf.py +++ b/lisa/microsoft/testsuites/performance/storageperf.py @@ -23,6 +23,7 @@ from lisa.features.network_interface import Sriov, Synthetic from lisa.messages import DiskSetupType, DiskType from lisa.microsoft.testsuites.performance.common import ( + check_premium_datadisks_performance, perf_disk, perf_premium_datadisks, perf_resource_disks, @@ -161,7 +162,8 @@ def perf_premiumv2_datadisks_1024k(self, node: Node, result: TestResult) -> None ), ) def perf_premium_datadisks_4k(self, node: Node, result: TestResult) -> None: - perf_premium_datadisks(node, result) + perf_messages = perf_premium_datadisks(node, result) + check_premium_datadisks_performance(result, perf_messages) @TestCaseMetadata( description=""" @@ -180,7 +182,8 @@ def perf_premium_datadisks_4k(self, node: Node, result: TestResult) -> None: ), ) def perf_premium_datadisks_1024k(self, node: Node, result: TestResult) -> None: - perf_premium_datadisks(node, result, block_size=1024) + perf_messages = perf_premium_datadisks(node, result, block_size=1024) + check_premium_datadisks_performance(result, perf_messages) @TestCaseMetadata( description=""" @@ -205,9 +208,10 @@ def perf_premium_datadisks_4k_io_uring( kernel_config = node.tools[KernelConfig] if not kernel_config.is_enabled("CONFIG_IO_URING"): raise SkippedException("io_uring is not available in kernel configuration") - perf_premium_datadisks( + perf_messages = perf_premium_datadisks( node, ioengine=IoEngine.IO_URING, test_result=result, max_iodepth=1024 ) + check_premium_datadisks_performance(result, perf_messages) @TestCaseMetadata( description=""" @@ -232,13 +236,14 @@ def perf_premium_datadisks_1024k_io_uring( kernel_config = node.tools[KernelConfig] if not kernel_config.is_enabled("CONFIG_IO_URING"): raise SkippedException("io_uring is not available in kernel configuration") - perf_premium_datadisks( + perf_messages = perf_premium_datadisks( node, ioengine=IoEngine.IO_URING, test_result=result, max_iodepth=1024, block_size=1024, ) + check_premium_datadisks_performance(result, perf_messages) @TestCaseMetadata( description=""" From 34242f5880010ab79894ada7f0ca9651d840730b Mon Sep 17 00:00:00 2001 From: SrikanthMyakam Date: Mon, 3 Aug 2026 22:06:39 +0530 Subject: [PATCH 2/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- lisa/microsoft/testsuites/performance/common.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lisa/microsoft/testsuites/performance/common.py b/lisa/microsoft/testsuites/performance/common.py index 2ee65345ef..e26e9840b4 100644 --- a/lisa/microsoft/testsuites/performance/common.py +++ b/lisa/microsoft/testsuites/performance/common.py @@ -1014,16 +1014,17 @@ def check_premium_datadisks_performance( """Validate premium data-disk random-read performance against the rated max. Rules: - - Only IO depths greater than 8 are considered; lower queue depths cannot - saturate the device. + - Only IO depths greater than ``PREMIUM_DATADISK_MIN_IODEPTH`` are considered; + lower queue depths cannot saturate the device. - For each qualifying IO depth, the random-read value is averaged across all iterations at that depth to get one value per depth. - The per-depth values are compared against the VM SKU's rated maximum. For 4K (IOPS-bound) tests the rated maximum is ``UncachedDiskIOPS``; for 1024K (bandwidth-bound) tests it is ``UncachedDiskBytesPerSecond`` and the random-read IOPS are converted to bytes/sec using the block size. - - The VM passes when, at every IO depth above 8, the best observed value - reaches at least 95% of the rated maximum. + - The VM passes when, at every IO depth above ``PREMIUM_DATADISK_MIN_IODEPTH``, + the averaged value reaches at least ``PREMIUM_DATADISK_PASS_RATIO`` of the + rated maximum. """ environment = test_result.environment assert environment, "fail to get environment from testresult" From 1a0d9c07422deab8c521a2687bc7e9aef0d69026 Mon Sep 17 00:00:00 2001 From: SrikanthMyakam Date: Mon, 3 Aug 2026 22:06:53 +0530 Subject: [PATCH 3/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- lisa/microsoft/testsuites/performance/common.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lisa/microsoft/testsuites/performance/common.py b/lisa/microsoft/testsuites/performance/common.py index e26e9840b4..92fffdc4e5 100644 --- a/lisa/microsoft/testsuites/performance/common.py +++ b/lisa/microsoft/testsuites/performance/common.py @@ -1061,7 +1061,13 @@ def check_premium_datadisks_performance( f"capability '{cap_name}' is not published for this VM size. This " f"check requires a VM size that publishes '{cap_name}'." ) - rated_max = float(sku_caps[cap_name]) + try: + rated_max = float(sku_caps[cap_name]) + except (TypeError, ValueError) as identifier: + raise SkippedException( + f"Skipping premium data-disk performance validation: rated disk " + f"capability '{cap_name}' value '{sku_caps.get(cap_name)}' is not a number." + ) from identifier block_size_bytes = block_size_kb * 1024 # Average the random-read value per IO depth across iterations, keeping only From ade2002e6969dbed17646a385fbcd0740d697686 Mon Sep 17 00:00:00 2001 From: Srikanth Myakam <374767+SRIKKANTH@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:07:51 +0530 Subject: [PATCH 4/4] Update common.py --- lisa/microsoft/testsuites/performance/common.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lisa/microsoft/testsuites/performance/common.py b/lisa/microsoft/testsuites/performance/common.py index 92fffdc4e5..4c0e89178e 100644 --- a/lisa/microsoft/testsuites/performance/common.py +++ b/lisa/microsoft/testsuites/performance/common.py @@ -1001,9 +1001,7 @@ def _get_azure_sku_capabilities( return None caps = azure_capability.resource_sku.get("capabilities", []) return { - cap["name"]: cap["value"] - for cap in caps - if "name" in cap and "value" in cap + cap["name"]: cap["value"] for cap in caps if "name" in cap and "value" in cap }