diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index f1e23b6a8d5..9d2597e114a 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -3080,3 +3080,69 @@ func Test_ACL_SecondaryNIC_DualStack(t *testing.T) { }, }) } + +func Test_Ubuntu2404_MANA(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that MANA (Accelerated Networking) is properly configured on Ubuntu 24.04 with a V6 SKU", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2404Gen2Containerd, + UseNVMe: true, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.ContainerService.Properties.AgentPoolProfiles[0].VMSize = ensureMinVMGeneration("Standard_D2ds_v6") + nbc.AgentPoolProfile.VMSize = ensureMinVMGeneration("Standard_D2ds_v6") + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr(ensureMinVMGeneration("Standard_D2ds_v6")) + enableAcceleratedNetworking(vmss) + }, + }, + }) +} + +func Test_Ubuntu2204_MANA(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that MANA (Accelerated Networking) is properly configured on Ubuntu 22.04 with a V6 SKU", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2204Gen2Containerd, + UseNVMe: true, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.ContainerService.Properties.AgentPoolProfiles[0].VMSize = ensureMinVMGeneration("Standard_D2ds_v6") + nbc.AgentPoolProfile.VMSize = ensureMinVMGeneration("Standard_D2ds_v6") + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr(ensureMinVMGeneration("Standard_D2ds_v6")) + enableAcceleratedNetworking(vmss) + }, + }, + }) +} + +func Test_AzureLinuxV3_MANA(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that MANA (Accelerated Networking) is properly configured on Azure Linux V3 with a V6 SKU", + Tags: Tags{ + VMSeriesCoverageTest: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3Gen2, + UseNVMe: true, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.ContainerService.Properties.AgentPoolProfiles[0].VMSize = ensureMinVMGeneration("Standard_D2ds_v6") + nbc.AgentPoolProfile.VMSize = ensureMinVMGeneration("Standard_D2ds_v6") + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr(ensureMinVMGeneration("Standard_D2ds_v6")) + enableAcceleratedNetworking(vmss) + }, + }, + }) +} diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index 1dca0a2be2b..828fad77aa9 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -10,6 +10,7 @@ import ( "os/signal" "path/filepath" "regexp" + "strconv" "strings" "syscall" "testing" @@ -1081,3 +1082,35 @@ func runScenarioGPUNPD(t *testing.T, vmSize, location, k8sSystemPoolSKU string) }, }} } + +func vmSKUGeneration(sku string) (int, error) { + // Extract the generation number from the SKU string (e.g., "Standard_D2s_v3" -> 3) + sku = strings.ToLower(sku) + idx := strings.LastIndex(sku, "_v") + if idx < 0 { + return 0, fmt.Errorf("invalid SKU format: %s", sku) + } + gen, err := strconv.Atoi(sku[idx+2:]) + if err != nil { + return 0, fmt.Errorf("SKU %q has non-numeric generation suffix: %w", sku, err) + } + return gen, nil +} + +func ensureMinVMGeneration(minSku string) string { + // Ensure that the VM SKU used is at least the minimum generation required for the test + // Get the minimum generation for the specified SKU + defaultGen, err := vmSKUGeneration(config.Config.DefaultVMSKU) + if err != nil { + panic(fmt.Sprintf("Warning: No minimum generation found for SKU %s", config.Config.DefaultVMSKU)) + } + minGen, err := vmSKUGeneration(minSku) + if err != nil { + panic(fmt.Sprintf("Warning: No minimum generation found for SKU %s", minSku)) + } + if defaultGen < minGen { + return minSku + } else { + return config.Config.DefaultVMSKU + } +} diff --git a/e2e/validation.go b/e2e/validation.go index 998ae13b2da..014cbd59632 100644 --- a/e2e/validation.go +++ b/e2e/validation.go @@ -45,6 +45,13 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) { ValidateLeakedSecrets(ctx, s) ValidateIPTablesCompatibleWithCiliumEBPF(ctx, s) ValidateRxBufferDefault(ctx, s) + + // Validate MANA (Accelerated Networking) when hardware is present. + // MANA is the standard network adapter on V5+ VM series. + if hasMANAHardware(ctx, s) { + ValidateMANA(ctx, s) + } + ValidateKernelLogs(ctx, s) ValidateWaagentLog(ctx, s) ValidateScriptlessCSECmd(ctx, s) diff --git a/e2e/validators.go b/e2e/validators.go index 1db27e0aa6e..e5bbc3351cf 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -22,6 +22,7 @@ import ( "github.com/Azure/agentbaker/e2e/components" "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/e2e/toolkit" "github.com/Azure/agentbaker/pkg/agent" "github.com/Azure/agentbaker/pkg/agent/datamodel" "github.com/stretchr/testify/assert" @@ -2818,6 +2819,111 @@ func ValidateRxBufferDefault(ctx context.Context, s *Scenario) { ValidateNetworkInterfaceConfig(ctx, s, customNicConfig) } +// ValidateMANAPCIDevice checks that the MANA PCI device is exposed to the VM. +// MANA hardware is identified by PCI device ID 0x00ba (Microsoft Corporation). +func ValidateMANAPCIDevice(ctx context.Context, s *Scenario) { + s.T.Helper() + defer toolkit.LogStep(s.T, "validating MANA PCI device is present")() + cmd := "grep -Rqi '^0x00ba$' /sys/bus/pci/devices/*/device 2>/dev/null" + execScriptOnVMForScenarioValidateExitCode(ctx, s, cmd, 0, + "MANA PCI device (0x00ba) not found in /sys/bus/pci/devices") +} + +// ValidateMANADriverLoaded checks that the MANA Ethernet driver (mana) is loaded +// in the running kernel. For built-in drivers they appear in modules.builtin; +// for loadable modules they must be present in lsmod. +func ValidateMANADriverLoaded(ctx context.Context, s *Scenario) { + s.T.Helper() + defer toolkit.LogStep(s.T, "validating MANA kernel driver is loaded")() + cmd := `lsmod | grep -q '^mana ' || grep -q '/mana\.ko' /lib/modules/$(uname -r)/modules.builtin` + execScriptOnVMForScenarioValidateExitCode(ctx, s, cmd, 0, + "MANA kernel driver (mana) not found in lsmod or modules.builtin") +} + +// ValidateMANAVFBonded checks that the MANA Virtual Function (VF) interface exists +// and is properly bonded to the primary eth0 interface. +// When Accelerated Networking is enabled with MANA, a VF interface should appear +// as a subordinate (SLAVE) of eth0. The VF name varies by VM generation: +// - V5: enP* (e.g., enP30832p0s0) +// - V6+: ens1 or enp0s0 +func ValidateMANAVFBonded(ctx context.Context, s *Scenario) { + s.T.Helper() + defer toolkit.LogStep(s.T, "validating MANA VF is bonded to eth0")() + // Look for any interface that has "master eth0" in ip link output, + // indicating it is bonded as a VF to the primary synthetic NIC. + cmd := `ip link show | grep 'master eth0'` + result := execScriptOnVMForScenarioValidateExitCode(ctx, s, cmd, 0, + "no VF interface found bonded to eth0 — accelerated networking may not be working") + s.T.Logf("MANA VF bonding: %s", strings.TrimSpace(result.stdout)) +} + +// ValidateMANATrafficFlowing checks that network traffic is actually flowing through +// the MANA Virtual Function rather than the slower synthetic (NetVSC) path. +// It sends HTTP requests from a pod to the node's default gateway and verifies +// that the VF TX packet counters increase by at least that amount. +func ValidateMANATrafficFlowing(ctx context.Context, s *Scenario) { + s.T.Helper() + defer toolkit.LogStep(s.T, "validating traffic is flowing through MANA VF")() + + const requestCount = 10 + getVFTxPackets := `val=$(sudo ethtool -S eth0 | awk '/^[[:space:]]*vf_tx_packets:/{print $2; exit}'); [ -n "$val" ] && echo "$val" || { echo "vf_tx_packets not found in ethtool -S eth0 output" >&2; exit 1; }` + // Read VF tx counter before generating traffic + resultBefore := execScriptOnVMForScenarioValidateExitCode(ctx, s, getVFTxPackets, 0, + "could not read VF tx packet counter from ethtool -S eth0") + countBefore, err := strconv.Atoi(strings.TrimSpace(resultBefore.stdout)) + require.NoError(s.T, err, "failed to parse vf_tx_packets before value %q", resultBefore.stdout) + s.T.Logf("MANA VF tx packets before: %d", countBefore) + + // Generate traffic from a pod on this node using curl to the node's default + // gateway. We use vf_tx_packets (not rx) so the test passes regardless of + // whether the target responds — what matters is that outbound packets from + // the pod traverse the MANA VF path. curl is pre-installed in the Mariner + // debug image, so no package install is needed. + gatewayResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, + "ip route | awk '/default/{print $3}'", 0, + "could not determine default gateway from ip route") + gatewayIP := strings.TrimSpace(gatewayResult.stdout) + require.NotEmpty(s.T, gatewayIP, "default gateway IP is empty") + s.T.Logf("MANA traffic test: using gateway %s as target", gatewayIP) + + // The "; true" ensures exit 0 regardless of curl's result — the gateway has + // no HTTP server so connections will fail, but TCP SYN packets still traverse + // the VF (incrementing vf_tx_packets). The real assertion is the counter delta below. + curlCmd := fmt.Sprintf("for i in $(seq 1 %d); do curl -s -o /dev/null -m 1 http://%s/ 2>/dev/null; done; true", requestCount, gatewayIP) + execOnVMForScenarioOnUnprivilegedPod(ctx, s, curlCmd) + + // Read VF tx counter after generating traffic + resultAfter := execScriptOnVMForScenarioValidateExitCode(ctx, s, getVFTxPackets, 0, + "could not read VF tx packet counter from ethtool -S eth0") + countAfter, err := strconv.Atoi(strings.TrimSpace(resultAfter.stdout)) + require.NoError(s.T, err, "failed to parse vf_tx_packets after value %q", resultAfter.stdout) + + delta := countAfter - countBefore + s.T.Logf("MANA VF tx packets after: %d (delta: %d, expected >= %d)", countAfter, delta, requestCount) + + require.GreaterOrEqual(s.T, delta, requestCount, + "vf_tx_packets increased by %d but expected at least %d \u2014 traffic may not be flowing through the MANA VF", delta, requestCount) +} + +// ValidateMANA runs all MANA (Microsoft Azure Network Adapter) checks. +// It verifies that the MANA PCI device is present, the kernel driver is loaded, +// the VF interface is bonded to eth0, and traffic is flowing through the VF. +func ValidateMANA(ctx context.Context, s *Scenario) { + s.T.Helper() + ValidateMANAPCIDevice(ctx, s) + ValidateMANADriverLoaded(ctx, s) + ValidateMANAVFBonded(ctx, s) + ValidateMANATrafficFlowing(ctx, s) +} + +// hasMANAHardware checks if the VM has MANA PCI hardware available. +// Returns true if the MANA device (0x00ba) is found in sysfs. +// This is used to conditionally run MANA validations on VMs that support it. +func hasMANAHardware(ctx context.Context, s *Scenario) bool { + result := execScriptOnVMForScenario(ctx, s, "grep -Rqi '^0x00ba$' /sys/bus/pci/devices/*/device 2>/dev/null") + return result.exitCode == "0" +} + // ValidateKernelLogs checks kernel logs for critical errors across multiple categories: // - Kernel panics/crashes (panic, oops, call trace, BUG, etc.) // - CPU lockups/stalls (soft/hard lockup, RCU stall, hung task, watchdog) diff --git a/e2e/vmss.go b/e2e/vmss.go index a14d5f7609a..d69afac7c6d 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -1286,6 +1286,20 @@ func addSecondaryNIC(vmss *armcompute.VirtualMachineScaleSet) { ) } +// enableAcceleratedNetworking explicitly enables Accelerated Networking on the +// primary NIC of the VMSS. This ensures MANA (Microsoft Azure Network Adapter) +// is active on the VM, which is required for V5+ VM series. +func enableAcceleratedNetworking(vmss *armcompute.VirtualMachineScaleSet) { + primaryNIC, err := getVMSSNICConfig(vmss) + if err != nil { + panic(fmt.Sprintf("enableAcceleratedNetworking: unable to get primary NIC config: %v", err)) + } + if primaryNIC.Properties == nil { + primaryNIC.Properties = &armcompute.VirtualMachineScaleSetNetworkConfigurationProperties{} + } + primaryNIC.Properties.EnableAcceleratedNetworking = to.Ptr(true) +} + // addDualStackSecondaryNIC appends a secondary (non-primary) NIC with both IPv4 and IPv6 // IP configurations to the VMSS model, using the same subnet as the primary NIC. func addDualStackSecondaryNIC(vmss *armcompute.VirtualMachineScaleSet) {