From c97403f624aa03610f0e0fe184d864aa75709f25 Mon Sep 17 00:00:00 2001 From: Vivek Yadav Date: Tue, 4 Aug 2026 17:32:24 -0700 Subject: [PATCH] Add configurable OpenVMM virtio device support Add PCIe virtio-net and virtio-blk device support for OpenVMM guests. Expose virtual IOMMU, shared TAP networking, network queue count, and processor topology settings through the runbook schema and launch configuration. Validate incompatible settings and extend the OpenVMM tool, schema, and node selftests to cover the new behavior. Keep Python 3.9 CI bootstrap compatible by excluding argcomplete 3.7.1, which evaluates PEP 604 unions at import time. --- .../continuous-integration-workflow.yml | 3 +- lisa/sut_orchestrator/openvmm/context.py | 17 + lisa/sut_orchestrator/openvmm/node.py | 482 ++++++++++++++---- lisa/sut_orchestrator/openvmm/schema.py | 82 +++ lisa/tools/openvmm.py | 174 ++++++- selftests/test_openvmm_node.py | 88 +++- selftests/test_openvmm_schema.py | 29 ++ selftests/test_openvmm_tool.py | 72 +++ 8 files changed, 827 insertions(+), 120 deletions(-) create mode 100644 selftests/test_openvmm_tool.py diff --git a/.github/workflows/continuous-integration-workflow.yml b/.github/workflows/continuous-integration-workflow.yml index 841694e079..d463fd23d2 100644 --- a/.github/workflows/continuous-integration-workflow.yml +++ b/.github/workflows/continuous-integration-workflow.yml @@ -69,7 +69,8 @@ jobs: if: startsWith(matrix.os, 'ubuntu') - name: Install Nox - run: pip install nox toml + # argcomplete 3.7.1 evaluates PEP 604 unions at import time on Python 3.9. + run: pip install nox toml "argcomplete!=3.7.1; python_version < '3.10'" if: matrix.nox-session - name: Run Nox diff --git a/lisa/sut_orchestrator/openvmm/context.py b/lisa/sut_orchestrator/openvmm/context.py index 38dc91421e..dbf49f2405 100644 --- a/lisa/sut_orchestrator/openvmm/context.py +++ b/lisa/sut_orchestrator/openvmm/context.py @@ -41,6 +41,17 @@ class DevicePassthroughContext: requested_count: int = 0 +@dataclass +class SharedTapNetworkContext: + tap_host_cidr: str = "" + address_mode: str = "" + reference_count: int = 0 + bridge_created: bool = False + dnsmasq_pid_file: str = "" + dnsmasq_lease_file: str = "" + input_rules_added: List[str] = field(default_factory=_new_str_list) + + @dataclass class NodeContext: vm_name: str = "" @@ -67,6 +78,7 @@ class NodeContext: tap_input_rules_added: List[str] = field(default_factory=_new_str_list) tap_dnsmasq_pid_file: str = "" tap_dnsmasq_lease_file: str = "" + shared_tap_network_key: str = "" effective_network: Optional[OpenVmmNetworkSchema] = None process_id: str = "" command_line: str = "" @@ -81,12 +93,17 @@ class OpenVmmHostContext: default_factory=_new_str_dict ) active_bridge_netfilter_count: int = 0 + bridge_netfilter_lock: RLock = field(default_factory=RLock) ssh_forwarding_lock: RLock = field(default_factory=RLock) artifact_copy_lock: Lock = field(default_factory=Lock) artifact_cache: Dict[str, str] = field(default_factory=_new_str_dict) device_pool_lock: Lock = field(default_factory=Lock) device_pool: Optional[Any] = None device_pool_config_key: str = "" + tap_network_lock: RLock = field(default_factory=RLock) + shared_tap_networks: Dict[str, SharedTapNetworkContext] = field( + default_factory=dict + ) def get_node_context(node: Node) -> NodeContext: diff --git a/lisa/sut_orchestrator/openvmm/node.py b/lisa/sut_orchestrator/openvmm/node.py index 5fb32001f6..f82242c00f 100644 --- a/lisa/sut_orchestrator/openvmm/node.py +++ b/lisa/sut_orchestrator/openvmm/node.py @@ -41,6 +41,7 @@ DeviceAddressSchema, DevicePassthroughContext, NodeContext, + SharedTapNetworkContext, get_host_context, get_node_context, ) @@ -170,6 +171,28 @@ def _shift_ip_address(address: str, base_cidr: str, index: int) -> str: return str(ipaddress.ip_address(new_address)) +def _increment_ip_address(address: str, cidr: str, index: int) -> str: + if not address or not cidr or index == 0: + return address + + host_interface = ipaddress.ip_interface(cidr) + network = host_interface.network + new_address = ipaddress.ip_address(int(ipaddress.ip_address(address)) + index) + if ( + new_address not in network + or new_address == network.network_address + or new_address == network.broadcast_address + or new_address == host_interface.ip + ): + raise LisaException( + f"cannot derive a shared OpenVMM guest address from '{address}' " + f"for guest index {index} within '{cidr}'. Use a lower base guest " + "address or a larger shared subnet." + ) + + return str(new_address) + + class GuestIpResolver(ABC): @abstractmethod def resolve( @@ -380,6 +403,30 @@ def create_effective_network( effective_network.tap_name = _increment_name_suffix( effective_network.tap_name, guest_index ) + if effective_network.shared_subnet: + try: + effective_network.validate_tap_interface_names() + except LisaException as identifier: + raise LisaException( + "cannot derive OpenVMM tap network interface names for guest " + f"index {guest_index}: {identifier}" + ) from identifier + if effective_network.address_mode == OPENVMM_ADDRESS_MODE_STATIC: + effective_network.guest_address = _increment_ip_address( + effective_network.guest_address, + effective_network.tap_host_cidr, + guest_index, + ) + if effective_network.forward_ssh_port: + effective_network.forwarded_port += guest_index + if effective_network.forwarded_port > 65535: + raise LisaException( + "cannot derive OpenVMM forwarded SSH port from " + f"'{network.forwarded_port}' for guest index {guest_index}: " + "derived port exceeds 65535. Use a lower base forwarded_port." + ) + return effective_network + effective_network.bridge_name = _increment_name_suffix( effective_network.bridge_name, guest_index ) @@ -695,17 +742,28 @@ def launch(self, node: "OpenVmmGuestNode", log: Logger) -> None: node_context = get_node_context(node) network = self._get_node_network(node, node_context) self._prepare_tap_network(network, node_context) + processor_count = _countspace_to_int(node.capability.core_count) launch_config = OpenVmmLaunchConfig( uefi_firmware_path=node_context.uefi_firmware_path, disk_img_path=node_context.disk_img_path, + disk_device=runbook.disk_device, + iommu=runbook.iommu, dvd_disk_paths=( [node_context.cloud_init_file_path] if node_context.cloud_init_file_path else [] ), - processors=_countspace_to_int(node.capability.core_count), + processors=processor_count, + vps_per_socket=( + runbook.vps_per_socket + if runbook.vps_per_socket is not None + else processor_count + ), + smt=runbook.smt, memory_mb=_countspace_to_int(node.capability.memory_mb), network_mode=network.mode, + network_device=network.device, + network_queue_count=network.queue_count, tap_name=getattr(network, "tap_name", ""), network_cidr=network.consomme_cidr, serial_mode=runbook.serial.mode, @@ -833,6 +891,59 @@ def _prepare_tap_network( if network.mode != OPENVMM_NETWORK_MODE_TAP: return + if network.shared_subnet: + host_context = get_host_context(self.host_node) + network_key = network.bridge_name + with host_context.tap_network_lock: + shared_context = host_context.shared_tap_networks.get(network_key) + if shared_context is None: + shared_context = SharedTapNetworkContext( + tap_host_cidr=network.tap_host_cidr, + address_mode=network.address_mode, + ) + host_context.shared_tap_networks[network_key] = shared_context + elif ( + shared_context.tap_host_cidr != network.tap_host_cidr + or shared_context.address_mode != network.address_mode + ): + raise LisaException( + f"OpenVMM shared bridge '{network_key}' is already using " + f"CIDR '{shared_context.tap_host_cidr}' and address mode " + f"'{shared_context.address_mode}'. Use matching shared " + "network settings for every guest on this bridge." + ) + + reuse_shared_resources = shared_context.reference_count > 0 + shared_context.reference_count += 1 + node_context.shared_tap_network_key = network_key + try: + self._prepare_tap_network_resources( + network, + node_context, + shared_context, + reuse_shared_resources, + ) + except Exception: + try: + self._teardown_tap_network(node_context, network) + except Exception as cleanup_identifier: + self._log.warning( + "failed to roll back OpenVMM shared TAP network " + f"'{network_key}' after setup failed: " + f"{cleanup_identifier}" + ) + raise + return + + self._prepare_tap_network_resources(network, node_context) + + def _prepare_tap_network_resources( + self, + network: OpenVmmNetworkSchema, + node_context: NodeContext, + shared_context: Optional[SharedTapNetworkContext] = None, + reuse_shared_resources: bool = False, + ) -> None: tap_name = network.tap_name bridge_name = network.bridge_name host = self.host_node @@ -845,7 +956,10 @@ def _prepare_tap_network( if not ip_tool.nic_exists(bridge_name): ip_tool.create_virtual_interface(bridge_name, "bridge") - node_context.tap_bridge_created = True + if shared_context: + shared_context.bridge_created = True + else: + node_context.tap_bridge_created = True host.execute( f"ip link set dev {shlex.quote(bridge_name)} type bridge stp_state 0", shell=True, @@ -948,41 +1062,52 @@ def _prepare_tap_network( ip_tool.up(tap_name) if network.address_mode != OPENVMM_ADDRESS_MODE_STATIC: - self._ensure_tap_host_services_input_allowed( - host_interface_name, node_context - ) - pid_file = f"/var/run/qemu-dnsmasq-{host_interface_name}.pid" - lease_file = f"/var/run/qemu-dnsmasq-{host_interface_name}.leases" - host.execute( - ( - f"test -f {shlex.quote(pid_file)} && " - f"kill $(cat {shlex.quote(pid_file)}) || true; " - f"rm -f {shlex.quote(pid_file)}; " - f"cp /dev/null {shlex.quote(lease_file)}" - ), - shell=True, - sudo=True, - expected_exit_code=0, - expected_exit_code_failure_message=( - "failed to reset OpenVMM dnsmasq state before starting " - f"DHCP on interface {host_interface_name}" - ), - ) - host.tools[Dnsmasq].start( - host_interface_name, - tap_gateway, - dhcp_range, - stop_firewall=False, - kill_existing=False, - pid_file=pid_file, - lease_file=lease_file, - dhcp_options=[ - f"option:router,{tap_gateway}", - f"option:dns-server,{tap_gateway}", - ], - ) - node_context.tap_dnsmasq_pid_file = pid_file - node_context.tap_dnsmasq_lease_file = lease_file + if shared_context and reuse_shared_resources: + node_context.tap_dnsmasq_pid_file = shared_context.dnsmasq_pid_file + node_context.tap_dnsmasq_lease_file = shared_context.dnsmasq_lease_file + else: + self._ensure_tap_host_services_input_allowed( + host_interface_name, node_context + ) + pid_file = f"/var/run/qemu-dnsmasq-{host_interface_name}.pid" + lease_file = f"/var/run/qemu-dnsmasq-{host_interface_name}.leases" + host.execute( + ( + f"test -f {shlex.quote(pid_file)} && " + f"kill $(cat {shlex.quote(pid_file)}) || true; " + f"rm -f {shlex.quote(pid_file)}; " + f"cp /dev/null {shlex.quote(lease_file)}" + ), + shell=True, + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "failed to reset OpenVMM dnsmasq state before starting " + f"DHCP on interface {host_interface_name}" + ), + ) + host.tools[Dnsmasq].start( + host_interface_name, + tap_gateway, + dhcp_range, + stop_firewall=False, + kill_existing=False, + pid_file=pid_file, + lease_file=lease_file, + dhcp_options=[ + f"option:router,{tap_gateway}", + f"option:dns-server,{tap_gateway}", + ], + ) + node_context.tap_dnsmasq_pid_file = pid_file + node_context.tap_dnsmasq_lease_file = lease_file + if shared_context: + shared_context.dnsmasq_pid_file = pid_file + shared_context.dnsmasq_lease_file = lease_file + shared_context.input_rules_added.extend( + node_context.tap_input_rules_added + ) + node_context.tap_input_rules_added.clear() self._log_tap_network_state(network, node_context) if node_context.tap_dnsmasq_pid_file: @@ -991,61 +1116,62 @@ def _prepare_tap_network( def _disable_bridge_netfilter(self, node_context: NodeContext) -> None: host = self.host_node host_context = get_host_context(host) - modprobe = host.tools[Modprobe] - if modprobe.module_exists("br_netfilter") and not modprobe.is_module_loaded( - "br_netfilter", force_run=True - ): - modprobe.load("br_netfilter") - - if host_context.active_bridge_netfilter_count > 0: - host_context.active_bridge_netfilter_count += 1 - node_context.tap_bridge_netfilter_disabled = True - return + with host_context.bridge_netfilter_lock: + modprobe = host.tools[Modprobe] + if modprobe.module_exists("br_netfilter") and not modprobe.is_module_loaded( + "br_netfilter", force_run=True + ): + modprobe.load("br_netfilter") - original_values = {} - for key in OPENVMM_BRIDGE_NETFILTER_KEYS: - value_result = host.execute( - f"sysctl -n {shlex.quote(key)}", - shell=True, - sudo=True, - no_info_log=True, - no_error_log=True, - expected_exit_code=None, - ) - if value_result.exit_code == 0: - original_values[key] = value_result.stdout.strip() + if host_context.active_bridge_netfilter_count > 0: + host_context.active_bridge_netfilter_count += 1 + node_context.tap_bridge_netfilter_disabled = True + return + + original_values = {} + for key in OPENVMM_BRIDGE_NETFILTER_KEYS: + value_result = host.execute( + f"sysctl -n {shlex.quote(key)}", + shell=True, + sudo=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=None, + ) + if value_result.exit_code == 0: + original_values[key] = value_result.stdout.strip() - if not original_values: - return + if not original_values: + return - host_context.original_bridge_netfilter_values = original_values - host_context.active_bridge_netfilter_count = 1 - node_context.tap_bridge_netfilter_disabled = True - try: - self._set_bridge_netfilter_values( - {key: "0" for key in original_values}, - failure_message=( - "failed to disable bridge netfilter on the OpenVMM host" - ), - ) - except Exception: + host_context.original_bridge_netfilter_values = original_values + host_context.active_bridge_netfilter_count = 1 + node_context.tap_bridge_netfilter_disabled = True try: self._set_bridge_netfilter_values( - original_values, + {key: "0" for key in original_values}, failure_message=( - "failed to roll back bridge netfilter after an OpenVMM " - "setup error" + "failed to disable bridge netfilter on the OpenVMM host" ), ) - except Exception as cleanup_identifier: - self._log.debug( - "failed to roll back bridge netfilter after setup error: " - f"{cleanup_identifier}" - ) - host_context.original_bridge_netfilter_values = {} - host_context.active_bridge_netfilter_count = 0 - node_context.tap_bridge_netfilter_disabled = False - raise + except Exception: + try: + self._set_bridge_netfilter_values( + original_values, + failure_message=( + "failed to roll back bridge netfilter after an " + "OpenVMM setup error" + ), + ) + except Exception as cleanup_identifier: + self._log.debug( + "failed to roll back bridge netfilter after setup error: " + f"{cleanup_identifier}" + ) + host_context.original_bridge_netfilter_values = {} + host_context.active_bridge_netfilter_count = 0 + node_context.tap_bridge_netfilter_disabled = False + raise def _set_bridge_netfilter_values( self, @@ -1118,6 +1244,26 @@ def _get_tap_host_service_input_rules(self, host_interface_name: str) -> List[st def _get_tap_network_config(self, network: OpenVmmNetworkSchema) -> tuple[str, str]: host_interface = ipaddress.ip_interface(network.tap_host_cidr) + if network.shared_subnet: + first_address = int(host_interface.network.network_address) + 1 + last_address = int(host_interface.network.broadcast_address) - 1 + gateway = int(host_interface.ip) + if first_address == gateway: + first_address += 1 + elif last_address == gateway: + last_address -= 1 + elif first_address < gateway < last_address: + first_address = gateway + 1 + if first_address > last_address: + raise LisaException( + f"OpenVMM shared subnet '{network.tap_host_cidr}' has no " + "address available for a guest. Use a larger subnet." + ) + return str(host_interface.ip), ( + f"{ipaddress.ip_address(first_address)}," + f"{ipaddress.ip_address(last_address)}" + ) + guest_ip = network.guest_address if not guest_ip: for address in host_interface.network.hosts(): @@ -1266,6 +1412,9 @@ def _get_tap_guest_address( network: OpenVmmNetworkSchema, log: Logger, ) -> str: + if network.shared_subnet: + return self._wait_for_shared_tap_lease(node_context, network, log) + _, dhcp_range = self._get_tap_network_config(network) guest_address = dhcp_range.split(",", maxsplit=1)[0].strip() if not guest_address: @@ -1276,6 +1425,91 @@ def _get_tap_guest_address( self._wait_for_tap_lease(node_context, guest_address, log, network) return guest_address + def _wait_for_shared_tap_lease( + self, + node_context: NodeContext, + network: OpenVmmNetworkSchema, + log: Logger, + timeout: int = OPENVMM_IP_DISCOVERY_TIMEOUT, + ) -> str: + lease_file = node_context.tap_dnsmasq_lease_file + if not lease_file: + raise LisaException( + "OpenVMM shared TAP DHCP lease tracking is not configured. " + "dnsmasq lease file path was not recorded." + ) + + guest_address = "" + + def _lease_is_ready() -> bool: + nonlocal guest_address + lease_result = self.host_node.execute( + ( + f"test -f {shlex.quote(lease_file)} && " + f"cat {shlex.quote(lease_file)} || true" + ), + shell=True, + sudo=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=0, + ) + fdb_result = self.host_node.execute( + f"bridge fdb show dev {shlex.quote(network.tap_name)}", + shell=True, + sudo=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "failed to inspect the OpenVMM shared bridge forwarding " + f"database for TAP interface {network.tap_name}" + ), + ) + tap_mac_addresses = { + line.split()[0].lower() + for line in fdb_result.stdout.splitlines() + if line.split() + } + for lease_line in lease_result.stdout.splitlines(): + lease_fields = lease_line.split() + if ( + len(lease_fields) >= 3 + and lease_fields[1].lower() in tap_mac_addresses + ): + guest_address = lease_fields[2] + log.debug( + "matched OpenVMM shared TAP interface " + f"'{network.tap_name}' to DHCP lease " + f"'{guest_address}' in {lease_file}" + ) + return True + if not self._is_process_running(node_context.process_id): + raise LisaException( + "OpenVMM process exited before the guest acquired a DHCP " + f"lease on shared TAP interface '{network.tap_name}'. " + f"{self._get_openvmm_failure_context(node_context, network)}" + ) + return False + + try: + check_till_timeout( + _lease_is_ready, + timeout_message=( + "wait for OpenVMM guest DHCP lease on shared TAP interface " + f"'{network.tap_name}'" + ), + timeout=timeout, + ) + except LisaTimeoutException as identifier: + raise LisaException( + "OpenVMM guest did not acquire a DHCP lease on shared TAP " + f"interface '{network.tap_name}'. " + f"{self._get_openvmm_failure_context(node_context, network)}" + ) from identifier + + return guest_address + def _wait_for_tap_lease( self, node_context: Any, @@ -1904,6 +2138,8 @@ def _teardown_tap_network( if network.mode != OPENVMM_NETWORK_MODE_TAP: return + shared_network_key = node_context.shared_tap_network_key + if node_context.tap_input_rules_added: rules_to_remove = node_context.tap_input_rules_added for rule in rules_to_remove: @@ -1915,7 +2151,7 @@ def _teardown_tap_network( ) node_context.tap_input_rules_added.clear() - if node_context.tap_dnsmasq_pid_file: + if not shared_network_key and node_context.tap_dnsmasq_pid_file: self.host_node.execute( ( f"test -f {shlex.quote(node_context.tap_dnsmasq_pid_file)} && " @@ -1938,7 +2174,9 @@ def _teardown_tap_network( ) node_context.tap_created = False - if node_context.tap_bridge_created and network.bridge_name: + if shared_network_key: + self._release_shared_tap_network(node_context, shared_network_key) + elif node_context.tap_bridge_created and network.bridge_name: self.host_node.execute( f"ip link delete {shlex.quote(network.bridge_name)} || true", shell=True, @@ -1949,23 +2187,67 @@ def _teardown_tap_network( if node_context.tap_bridge_netfilter_disabled: host_context = get_host_context(self.host_node) - if host_context.active_bridge_netfilter_count > 0: - host_context.active_bridge_netfilter_count -= 1 + with host_context.bridge_netfilter_lock: + if host_context.active_bridge_netfilter_count > 0: + host_context.active_bridge_netfilter_count -= 1 + + if ( + host_context.active_bridge_netfilter_count == 0 + and host_context.original_bridge_netfilter_values + ): + self._set_bridge_netfilter_values( + host_context.original_bridge_netfilter_values, + failure_message=( + "failed to restore bridge netfilter state on the " + "OpenVMM host" + ), + ) + host_context.original_bridge_netfilter_values = {} - if ( - host_context.active_bridge_netfilter_count == 0 - and host_context.original_bridge_netfilter_values - ): - self._set_bridge_netfilter_values( - host_context.original_bridge_netfilter_values, - failure_message=( - "failed to restore bridge netfilter state on the " - "OpenVMM host" + node_context.tap_bridge_netfilter_disabled = False + + def _release_shared_tap_network( + self, node_context: NodeContext, network_key: str + ) -> None: + host_context = get_host_context(self.host_node) + with host_context.tap_network_lock: + shared_context = host_context.shared_tap_networks.get(network_key) + node_context.shared_tap_network_key = "" + node_context.tap_dnsmasq_pid_file = "" + node_context.tap_dnsmasq_lease_file = "" + if shared_context is None: + return + + shared_context.reference_count -= 1 + if shared_context.reference_count > 0: + return + + for rule in shared_context.input_rules_added: + self.host_node.execute( + f"iptables -D {rule} || true", + shell=True, + sudo=True, + expected_exit_code=0, + ) + if shared_context.dnsmasq_pid_file: + self.host_node.execute( + ( + f"test -f {shlex.quote(shared_context.dnsmasq_pid_file)} " + "&& kill $(cat " + f"{shlex.quote(shared_context.dnsmasq_pid_file)}) || true" ), + shell=True, + sudo=True, + expected_exit_code=0, ) - host_context.original_bridge_netfilter_values = {} - - node_context.tap_bridge_netfilter_disabled = False + if shared_context.bridge_created: + self.host_node.execute( + f"ip link delete {shlex.quote(network_key)} || true", + shell=True, + sudo=True, + expected_exit_code=0, + ) + del host_context.shared_tap_networks[network_key] class OpenVmmGuestNode(RemoteNode): diff --git a/lisa/sut_orchestrator/openvmm/schema.py b/lisa/sut_orchestrator/openvmm/schema.py index 686c38db4d..735f36aaca 100644 --- a/lisa/sut_orchestrator/openvmm/schema.py +++ b/lisa/sut_orchestrator/openvmm/schema.py @@ -16,6 +16,18 @@ DevicePassthroughSchema, HostDevicePoolSchema, ) +from lisa.tools.openvmm import ( + OPENVMM_DISK_DEVICE_SCSI, + OPENVMM_DISK_DEVICE_VIRTIO_BLK, + OPENVMM_IOMMU_AMD, + OPENVMM_IOMMU_INTEL, + OPENVMM_IOMMU_NONE, + OPENVMM_NETWORK_DEVICE_SYNTHETIC, + OPENVMM_NETWORK_DEVICE_VIRTIO, + OPENVMM_SMT_AUTO, + OPENVMM_SMT_FORCE, + OPENVMM_SMT_OFF, +) from lisa.util import LisaException from .. import OPENVMM @@ -99,6 +111,15 @@ def __post_init__(self) -> None: @dataclass class OpenVmmNetworkSchema: mode: str = OPENVMM_NETWORK_MODE_USER + device: str = OPENVMM_NETWORK_DEVICE_SYNTHETIC + queue_count: Optional[int] = field( + default=None, + metadata=schema.field_metadata( + field_function=schema.fields.Int, + validate=schema.validate.Range(min=1, max=65535), + ), + ) + shared_subnet: bool = False connection_mode: str = OPENVMM_CONNECTION_MODE_FORWARDED_PORT address_mode: str = OPENVMM_ADDRESS_MODE_DISCOVER tap_name: str = "" @@ -160,7 +181,33 @@ def validate_tap_interface_names(self) -> None: if self.bridge_name: self._validate_interface_name("bridge_name", self.bridge_name) + def _validate_device(self) -> None: + if self.device not in [ + OPENVMM_NETWORK_DEVICE_SYNTHETIC, + OPENVMM_NETWORK_DEVICE_VIRTIO, + ]: + raise LisaException( + f"network device '{self.device}' is not supported for OpenVMM " + f"guests. Supported values: {OPENVMM_NETWORK_DEVICE_SYNTHETIC}, " + f"{OPENVMM_NETWORK_DEVICE_VIRTIO}" + ) + + def _validate_shared_subnet(self) -> None: + if not self.shared_subnet: + return + if self.mode != OPENVMM_NETWORK_MODE_TAP or not self.bridge_name: + raise LisaException( + "shared_subnet requires tap network mode and bridge_name" + ) + if self.address_mode != OPENVMM_ADDRESS_MODE_STATIC and self.guest_address: + raise LisaException( + "guest_address cannot be fixed when shared_subnet uses DHCP. " + "Remove guest_address or use address_mode 'static'." + ) + def __post_init__(self) -> None: + self._validate_device() + self._validate_shared_subnet() if self.connection_mode not in [ OPENVMM_CONNECTION_MODE_FORWARDED_PORT, OPENVMM_CONNECTION_MODE_HOST_PROXY, @@ -258,6 +305,16 @@ class OpenVmmGuestNodeSchema(schema.GuestNode): uefi: Optional[OpenVmmUefiSchema] = None disk_img: str = "" disk_img_is_remote_path: bool = False + disk_device: str = OPENVMM_DISK_DEVICE_SCSI + iommu: str = OPENVMM_IOMMU_NONE + vps_per_socket: Optional[int] = field( + default=None, + metadata=schema.field_metadata( + field_function=schema.fields.Int, + validate=schema.validate.Range(min=1), + ), + ) + smt: str = OPENVMM_SMT_OFF min_raw_disk_size_gb: int = field( default=OPENVMM_DEFAULT_MIN_RAW_DISK_SIZE_GB, metadata=schema.field_metadata( @@ -290,6 +347,31 @@ def __post_init__(self) -> None: ) if not self.disk_img: raise LisaException("disk_img is required for UEFI OpenVMM guests") + if self.disk_device not in [ + OPENVMM_DISK_DEVICE_SCSI, + OPENVMM_DISK_DEVICE_VIRTIO_BLK, + ]: + raise LisaException( + f"disk device '{self.disk_device}' is not supported for OpenVMM " + f"guests. Supported values: {OPENVMM_DISK_DEVICE_SCSI}, " + f"{OPENVMM_DISK_DEVICE_VIRTIO_BLK}" + ) + if self.iommu not in [ + OPENVMM_IOMMU_NONE, + OPENVMM_IOMMU_INTEL, + OPENVMM_IOMMU_AMD, + ]: + raise LisaException( + f"IOMMU '{self.iommu}' is not supported for OpenVMM guests. " + f"Supported values: {OPENVMM_IOMMU_NONE}, " + f"{OPENVMM_IOMMU_INTEL}, {OPENVMM_IOMMU_AMD}" + ) + if self.smt not in [OPENVMM_SMT_AUTO, OPENVMM_SMT_FORCE, OPENVMM_SMT_OFF]: + raise LisaException( + f"SMT mode '{self.smt}' is not supported for OpenVMM guests. " + f"Supported values: {OPENVMM_SMT_AUTO}, {OPENVMM_SMT_FORCE}, " + f"{OPENVMM_SMT_OFF}" + ) if ( self.cloud_init and not self.private_key_file diff --git a/lisa/tools/openvmm.py b/lisa/tools/openvmm.py index a1f5404541..697ec24f3c 100644 --- a/lisa/tools/openvmm.py +++ b/lisa/tools/openvmm.py @@ -14,6 +14,19 @@ OPENVMM_NETWORK_BACKEND_CONSOMME = "consomme" OPENVMM_DEFAULT_SCSI_CONTROLLER = "lisa_scsi0" +OPENVMM_DISK_DEVICE_SCSI = "scsi" +OPENVMM_DISK_DEVICE_VIRTIO_BLK = "virtio-blk" +OPENVMM_IOMMU_AMD = "amd-iommu" +OPENVMM_IOMMU_INTEL = "intel-vtd" +OPENVMM_IOMMU_NONE = "none" +OPENVMM_NETWORK_DEVICE_SYNTHETIC = "synthetic" +OPENVMM_NETWORK_DEVICE_VIRTIO = "virtio" +OPENVMM_SMT_AUTO = "auto" +OPENVMM_SMT_FORCE = "force" +OPENVMM_SMT_OFF = "off" +OPENVMM_VIRTIO_ROOT_COMPLEX = "lisa_virtio_rc0" +OPENVMM_VIRTIO_DISK_PORT = "lisa_virtio_disk" +OPENVMM_VIRTIO_NETWORK_PORT = "lisa_virtio_net" _COMMAND_NOT_FOUND_MARKERS = ( "command not found", @@ -37,10 +50,16 @@ class OpenVmmLaunchConfig: with_hv: bool = True hypervisor: str = "mshv" disk_img_path: str = "" + disk_device: str = OPENVMM_DISK_DEVICE_SCSI + iommu: str = OPENVMM_IOMMU_NONE dvd_disk_paths: List[str] = field(default_factory=_new_str_list) processors: int = 1 + vps_per_socket: Optional[int] = None + smt: str = "" memory_mb: int = 1024 network_mode: str = "user" + network_device: str = OPENVMM_NETWORK_DEVICE_SYNTHETIC + network_queue_count: Optional[int] = None tap_name: str = "" network_cidr: str = "" serial_mode: str = "file" @@ -100,7 +119,12 @@ def build_command(self, config: OpenVmmLaunchConfig) -> str: args.append("--hv") if config.hypervisor: args.extend(["--hypervisor", config.hypervisor]) + self._validate_processor_topology(config) args.extend(["--processors", str(config.processors)]) + if config.vps_per_socket is not None: + args.extend(["--vps-per-socket", str(config.vps_per_socket)]) + if config.smt: + args.extend(["--smt", config.smt]) args.extend(["--memory", f"{config.memory_mb}MB"]) if not config.uefi_firmware_path: @@ -108,17 +132,124 @@ def build_command(self, config: OpenVmmLaunchConfig) -> str: args.append("--uefi") args.extend(["--uefi-firmware", config.uefi_firmware_path]) - if config.disk_img_path or config.dvd_disk_paths: - args.extend(["--vmbus-scsi", f"id={OPENVMM_DEFAULT_SCSI_CONTROLLER}"]) + self._validate_device_types(config) + self._add_pcie_args(args, config) + self._add_disk_args(args, config) + network_backend = self._get_network_backend(config) + self._add_network_args(args, config, network_backend) - if config.disk_img_path: + if config.serial_mode == "stderr": + args.extend(["--com1", "stderr"]) + elif config.serial_mode == "file": + if not config.serial_path: + raise LisaException("serial_path must be provided for file serial mode") + args.extend(["--com1", f"file={config.serial_path}"]) + else: + raise LisaException(f"Unsupported serial mode: {config.serial_mode}") + + args.extend(config.extra_args) + return " ".join(shlex.quote(arg) for arg in args) + + def _validate_processor_topology(self, config: OpenVmmLaunchConfig) -> None: + if config.vps_per_socket is not None and config.vps_per_socket < 1: + raise LisaException( + "OpenVMM vps_per_socket must be at least 1. " + "Set it to the number of virtual processors in each socket." + ) + if config.smt and config.smt not in [ + OPENVMM_SMT_AUTO, + OPENVMM_SMT_FORCE, + OPENVMM_SMT_OFF, + ]: + raise LisaException( + f"OpenVMM SMT mode '{config.smt}' is not supported. " + f"Use {OPENVMM_SMT_AUTO}, {OPENVMM_SMT_FORCE}, or " + f"{OPENVMM_SMT_OFF}." + ) + + def _validate_device_types(self, config: OpenVmmLaunchConfig) -> None: + if config.disk_device not in [ + OPENVMM_DISK_DEVICE_SCSI, + OPENVMM_DISK_DEVICE_VIRTIO_BLK, + ]: + raise LisaException( + f"Unsupported OpenVMM disk device: {config.disk_device}" + ) + if config.network_device not in [ + OPENVMM_NETWORK_DEVICE_SYNTHETIC, + OPENVMM_NETWORK_DEVICE_VIRTIO, + ]: + raise LisaException( + f"Unsupported OpenVMM network device: {config.network_device}" + ) + if config.network_queue_count is not None and not ( + 1 <= config.network_queue_count <= 65535 + ): + raise LisaException( + "OpenVMM network queue count must be between 1 and 65535. " + "Set network.queue_count to a supported positive value." + ) + if config.iommu not in [ + OPENVMM_IOMMU_NONE, + OPENVMM_IOMMU_INTEL, + OPENVMM_IOMMU_AMD, + ]: + raise LisaException(f"Unsupported OpenVMM IOMMU: {config.iommu}") + + def _add_pcie_args(self, args: List[str], config: OpenVmmLaunchConfig) -> None: + use_virtio_disk = bool(config.disk_img_path) and ( + config.disk_device == OPENVMM_DISK_DEVICE_VIRTIO_BLK + ) + use_virtio_network = config.network_device == OPENVMM_NETWORK_DEVICE_VIRTIO + if use_virtio_disk or use_virtio_network: + args.extend(["--pcie-root-complex", OPENVMM_VIRTIO_ROOT_COMPLEX]) + if config.iommu != OPENVMM_IOMMU_NONE: + args.extend([f"--{config.iommu}", OPENVMM_VIRTIO_ROOT_COMPLEX]) + elif config.iommu != OPENVMM_IOMMU_NONE: + raise LisaException( + "OpenVMM IOMMU requires a virtio disk or network device on PCIe" + ) + if use_virtio_disk: args.extend( [ - "--disk", - f"file:{config.disk_img_path}," - f"on={OPENVMM_DEFAULT_SCSI_CONTROLLER},lun=0", + "--pcie-root-port", + f"{OPENVMM_VIRTIO_ROOT_COMPLEX}:{OPENVMM_VIRTIO_DISK_PORT}", ] ) + if use_virtio_network: + network_root_port = ( + f"{OPENVMM_VIRTIO_ROOT_COMPLEX}:{OPENVMM_VIRTIO_NETWORK_PORT}" + ) + args.extend( + [ + "--pcie-root-port", + network_root_port, + ] + ) + + def _add_disk_args(self, args: List[str], config: OpenVmmLaunchConfig) -> None: + if config.dvd_disk_paths or ( + config.disk_img_path and config.disk_device == OPENVMM_DISK_DEVICE_SCSI + ): + args.extend(["--vmbus-scsi", f"id={OPENVMM_DEFAULT_SCSI_CONTROLLER}"]) + + if config.disk_img_path: + if config.disk_device == OPENVMM_DISK_DEVICE_SCSI: + args.extend( + [ + "--disk", + f"file:{config.disk_img_path}," + f"on={OPENVMM_DEFAULT_SCSI_CONTROLLER},lun=0", + ] + ) + else: + args.extend( + [ + "--virtio-blk", + f"file:{config.disk_img_path}," + f"pcie_port={OPENVMM_VIRTIO_DISK_PORT}", + ] + ) for lun, dvd_disk_path in enumerate(config.dvd_disk_paths, start=1): args.extend( @@ -129,29 +260,36 @@ def build_command(self, config: OpenVmmLaunchConfig) -> str: ] ) + def _get_network_backend(self, config: OpenVmmLaunchConfig) -> str: if config.network_mode == "user": network_backend = OPENVMM_NETWORK_BACKEND_CONSOMME if config.network_cidr: network_backend = f"{network_backend}:{config.network_cidr}" - args.extend(["--net", network_backend]) elif config.network_mode == "tap": if not config.tap_name: raise LisaException("tap_name must be provided for tap networking") - args.extend(["--net", f"tap:{config.tap_name}"]) + network_backend = f"tap:{config.tap_name}" else: raise LisaException(f"Unsupported network mode: {config.network_mode}") + return network_backend - if config.serial_mode == "stderr": - args.extend(["--com1", "stderr"]) - elif config.serial_mode == "file": - if not config.serial_path: - raise LisaException("serial_path must be provided for file serial mode") - args.extend(["--com1", f"file={config.serial_path}"]) + def _add_network_args( + self, + args: List[str], + config: OpenVmmLaunchConfig, + network_backend: str, + ) -> None: + if config.network_queue_count is not None: + network_backend = f"queues={config.network_queue_count}:{network_backend}" + if config.network_device == OPENVMM_NETWORK_DEVICE_SYNTHETIC: + args.extend(["--net", network_backend]) else: - raise LisaException(f"Unsupported serial mode: {config.serial_mode}") - - args.extend(config.extra_args) - return " ".join(shlex.quote(arg) for arg in args) + args.extend( + [ + "--virtio-net", + f"pcie_port={OPENVMM_VIRTIO_NETWORK_PORT}:{network_backend}", + ] + ) def launch_vm( self, diff --git a/selftests/test_openvmm_node.py b/selftests/test_openvmm_node.py index e1cafde91d..72d8d28041 100644 --- a/selftests/test_openvmm_node.py +++ b/selftests/test_openvmm_node.py @@ -15,6 +15,7 @@ from lisa.sut_orchestrator.openvmm.context import NodeContext from lisa.sut_orchestrator.openvmm.node import OpenVmmController, OpenVmmGuestNode from lisa.sut_orchestrator.openvmm.schema import ( + OPENVMM_ADDRESS_MODE_STATIC, OPENVMM_CONNECTION_MODE_HOST_PROXY, OPENVMM_NETWORK_MODE_TAP, OpenVmmGuestNodeSchema, @@ -26,6 +27,11 @@ SerialConsole as OpenVmmSerialConsole, ) from lisa.tools import Cat, Ip, Kill, Mkdir +from lisa.tools.openvmm import ( + OPENVMM_DISK_DEVICE_SCSI, + OPENVMM_IOMMU_NONE, + OPENVMM_NETWORK_DEVICE_SYNTHETIC, +) from lisa.util import LisaException @@ -131,7 +137,16 @@ def test_launch_uses_host_pure_path_for_cwd(self) -> None: node = SimpleNamespace( runbook=SimpleNamespace( openvmm_binary="/usr/local/bin/openvmm", - network=SimpleNamespace(mode="user", consomme_cidr=""), + disk_device=OPENVMM_DISK_DEVICE_SCSI, + iommu=OPENVMM_IOMMU_NONE, + vps_per_socket=None, + smt="off", + network=SimpleNamespace( + mode="user", + device=OPENVMM_NETWORK_DEVICE_SYNTHETIC, + queue_count=1, + consomme_cidr="", + ), serial=SimpleNamespace(mode="file"), extra_args=[], ), @@ -158,6 +173,10 @@ def test_launch_uses_host_pure_path_for_cwd(self) -> None: cwd=PurePosixPath("/var/tmp/openvmm-host-g0"), sudo=False, ) + launch_config = openvmm.launch_vm.call_args.args[0] + self.assertEqual(OPENVMM_DISK_DEVICE_SCSI, launch_config.disk_device) + self.assertEqual(OPENVMM_NETWORK_DEVICE_SYNTHETIC, launch_config.network_device) + self.assertEqual(1, launch_config.network_queue_count) def test_create_effective_network_derives_unique_tap_settings(self) -> None: controller, _, _, _ = self._create_controller() @@ -189,6 +208,73 @@ def test_create_effective_network_derives_unique_tap_settings(self) -> None: self.assertEqual("tap0", network.tap_name) self.assertEqual("10.0.0.1/24", network.tap_host_cidr) + def test_create_effective_network_reuses_shared_tap_subnet(self) -> None: + controller, _, _, _ = self._create_controller() + network = OpenVmmNetworkSchema( + mode=OPENVMM_NETWORK_MODE_TAP, + shared_subnet=True, + address_mode=OPENVMM_ADDRESS_MODE_STATIC, + tap_name="tap0", + bridge_name="ovmbr0", + tap_host_cidr="10.0.0.1/24", + guest_address="10.0.0.2", + forward_ssh_port=True, + forwarded_port=60022, + ) + + third_guest_network = controller.create_effective_network(network, 2) + + self.assertEqual("tap2", third_guest_network.tap_name) + self.assertEqual("ovmbr0", third_guest_network.bridge_name) + self.assertEqual("10.0.0.1/24", third_guest_network.tap_host_cidr) + self.assertEqual("10.0.0.4", third_guest_network.guest_address) + self.assertEqual(60024, third_guest_network.forwarded_port) + + def test_create_effective_network_rejects_shared_tap_name_overflow(self) -> None: + controller, _, _, _ = self._create_controller() + network = OpenVmmNetworkSchema( + mode=OPENVMM_NETWORK_MODE_TAP, + shared_subnet=True, + tap_name="abcdefghijklmno", + bridge_name="ovmbr0", + ) + + with self.assertRaisesRegex( + LisaException, + "cannot derive OpenVMM tap network interface names", + ): + controller.create_effective_network(network, 1) + + def test_shared_tap_setup_failure_removes_node_input_rules(self) -> None: + controller, _, _, _ = self._create_controller() + network = OpenVmmNetworkSchema( + mode=OPENVMM_NETWORK_MODE_TAP, + shared_subnet=True, + tap_name="tap0", + bridge_name="ovmbr0", + ) + node_context = NodeContext() + input_rule = "INPUT -i ovmbr0 -p udp --dport 67 -j ACCEPT" + + def _fail_after_adding_input_rule(*args: Any, **kwargs: Any) -> None: + node_context.tap_input_rules_added.append(input_rule) + raise LisaException("dnsmasq failed") + + with patch.object( + controller, + "_prepare_tap_network_resources", + side_effect=_fail_after_adding_input_rule, + ), self.assertRaisesRegex(LisaException, "dnsmasq failed"): + controller._prepare_tap_network(network, node_context) + + execute = cast(MagicMock, controller.host_node.execute) + self.assertIn( + f"iptables -D {input_rule} || true", + [call.args[0] for call in execute.call_args_list], + ) + self.assertEqual([], node_context.tap_input_rules_added) + self.assertEqual("", node_context.shared_tap_network_key) + def test_supported_features_include_serial_console(self) -> None: supported_feature_names = [ feature.name() for feature in OpenVmmController.supported_features() diff --git a/selftests/test_openvmm_schema.py b/selftests/test_openvmm_schema.py index b889213c84..0348022574 100644 --- a/selftests/test_openvmm_schema.py +++ b/selftests/test_openvmm_schema.py @@ -13,6 +13,10 @@ OpenVmmGuestNodeSchema, OpenVmmNetworkSchema, ) +from lisa.tools.openvmm import ( + OPENVMM_DISK_DEVICE_VIRTIO_BLK, + OPENVMM_NETWORK_DEVICE_VIRTIO, +) class OpenVmmSchemaTestCase(TestCase): @@ -39,6 +43,17 @@ def test_network_schema_rejects_invalid_ssh_port(self) -> None: } ) + def test_network_schema_accepts_valid_queue_count(self) -> None: + network_schema = cast(Any, OpenVmmNetworkSchema).schema() + network = network_schema.load({"queue_count": 1}) + + self.assertEqual(1, network.queue_count) + + def test_network_schema_rejects_invalid_queue_count(self) -> None: + network_schema = cast(Any, OpenVmmNetworkSchema).schema() + with self.assertRaises(ValidationError): + network_schema.load({"queue_count": 0}) + def test_guest_schema_splits_extra_args_string(self) -> None: guest_schema = cast(Any, OpenVmmGuestNodeSchema).schema() guest = guest_schema.load( @@ -51,6 +66,20 @@ def test_guest_schema_splits_extra_args_string(self) -> None: self.assertEqual(["--foo", "bar baz"], guest.extra_args) + def test_guest_schema_accepts_virtio_devices(self) -> None: + guest_schema = cast(Any, OpenVmmGuestNodeSchema).schema() + guest = guest_schema.load( + { + "uefi": {"firmware_path": "/firmware"}, + "disk_img": "/disk.raw", + "disk_device": OPENVMM_DISK_DEVICE_VIRTIO_BLK, + "network": {"device": OPENVMM_NETWORK_DEVICE_VIRTIO}, + } + ) + + self.assertEqual(OPENVMM_DISK_DEVICE_VIRTIO_BLK, guest.disk_device) + self.assertEqual(OPENVMM_NETWORK_DEVICE_VIRTIO, guest.network.device) + def test_host_proxy_connection_mode_disables_forwarded_port(self) -> None: network_schema = cast(Any, OpenVmmNetworkSchema).schema() network = network_schema.load( diff --git a/selftests/test_openvmm_tool.py b/selftests/test_openvmm_tool.py new file mode 100644 index 0000000000..b26f819b1f --- /dev/null +++ b/selftests/test_openvmm_tool.py @@ -0,0 +1,72 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from unittest import TestCase + +from lisa.tools.openvmm import ( + OPENVMM_DISK_DEVICE_VIRTIO_BLK, + OPENVMM_IOMMU_INTEL, + OPENVMM_NETWORK_DEVICE_VIRTIO, + OpenVmm, + OpenVmmLaunchConfig, +) + + +class OpenVmmToolTestCase(TestCase): + def _create_tool(self) -> OpenVmm: + tool = OpenVmm.__new__(OpenVmm) + tool.set_binary_path("openvmm") + return tool + + def test_build_command_uses_default_devices(self) -> None: + command = self._create_tool().build_command( + OpenVmmLaunchConfig( + uefi_firmware_path="/firmware/MSVM.fd", + disk_img_path="/disks/guest.raw", + network_mode="tap", + tap_name="tap0", + serial_path="/logs/console.log", + ) + ) + + self.assertIn("--vmbus-scsi id=lisa_scsi0", command) + self.assertIn("--disk file:/disks/guest.raw,on=lisa_scsi0,lun=0", command) + self.assertIn("--net tap:tap0", command) + self.assertNotIn("--pcie-root-complex", command) + self.assertNotIn("--virtio-blk", command) + self.assertNotIn("--virtio-net", command) + self.assertNotIn("queues=", command) + + def test_build_command_uses_virtio_devices_over_pcie(self) -> None: + command = self._create_tool().build_command( + OpenVmmLaunchConfig( + uefi_firmware_path="/firmware/MSVM.fd", + disk_img_path="/disks/guest.raw", + disk_device=OPENVMM_DISK_DEVICE_VIRTIO_BLK, + iommu=OPENVMM_IOMMU_INTEL, + dvd_disk_paths=["/disks/cloud-init.iso"], + network_mode="tap", + network_device=OPENVMM_NETWORK_DEVICE_VIRTIO, + network_queue_count=1, + tap_name="tap0", + serial_path="/logs/console.log", + ) + ) + + self.assertIn("--pcie-root-complex lisa_virtio_rc0", command) + self.assertIn("--intel-vtd lisa_virtio_rc0", command) + self.assertIn("--pcie-root-port lisa_virtio_rc0:lisa_virtio_disk", command) + self.assertIn("--pcie-root-port lisa_virtio_rc0:lisa_virtio_net", command) + self.assertIn( + "--virtio-blk file:/disks/guest.raw,pcie_port=lisa_virtio_disk", + command, + ) + self.assertIn( + "--virtio-net pcie_port=lisa_virtio_net:queues=1:tap:tap0", command + ) + self.assertIn("--vmbus-scsi id=lisa_scsi0", command) + self.assertIn( + "--disk file:/disks/cloud-init.iso,on=lisa_scsi0,lun=1,dvd", command + ) + self.assertNotIn("--disk file:/disks/guest.raw,on=lisa_scsi0,lun=0", command) + self.assertNotIn("--net tap:tap0", command)