diff --git a/.github/workflows/continuous-integration-workflow.yml b/.github/workflows/continuous-integration-workflow.yml index 3c4e0e24fb..16b454d31b 100644 --- a/.github/workflows/continuous-integration-workflow.yml +++ b/.github/workflows/continuous-integration-workflow.yml @@ -71,10 +71,8 @@ jobs: if: startsWith(matrix.os, 'ubuntu') && matrix.additional-sys-deps - name: Install Nox - # argcomplete >= 3.6 uses Python 3.10 union-type syntax. Keep the - # nox CLI importable on the Python 3.9 matrix jobs while allowing - # newer jobs to use the unconstrained latest version. - run: python -m pip install nox toml "argcomplete<3.6; python_version < '3.10'" + # argcomplete 3.7.1 evaluates PEP 604 unions at import time on Python 3.9. + run: python -m pip install nox toml "argcomplete!=3.7.1; python_version < '3.10'" if: matrix.nox-session - name: Run Nox diff --git a/lisa/sut_orchestrator/openvmm/context.py b/lisa/sut_orchestrator/openvmm/context.py index 38dc91421e..dbf49f2405 100644 --- a/lisa/sut_orchestrator/openvmm/context.py +++ b/lisa/sut_orchestrator/openvmm/context.py @@ -41,6 +41,17 @@ class DevicePassthroughContext: requested_count: int = 0 +@dataclass +class SharedTapNetworkContext: + tap_host_cidr: str = "" + address_mode: str = "" + reference_count: int = 0 + bridge_created: bool = False + dnsmasq_pid_file: str = "" + dnsmasq_lease_file: str = "" + input_rules_added: List[str] = field(default_factory=_new_str_list) + + @dataclass class NodeContext: vm_name: str = "" @@ -67,6 +78,7 @@ class NodeContext: tap_input_rules_added: List[str] = field(default_factory=_new_str_list) tap_dnsmasq_pid_file: str = "" tap_dnsmasq_lease_file: str = "" + shared_tap_network_key: str = "" effective_network: Optional[OpenVmmNetworkSchema] = None process_id: str = "" command_line: str = "" @@ -81,12 +93,17 @@ class OpenVmmHostContext: default_factory=_new_str_dict ) active_bridge_netfilter_count: int = 0 + bridge_netfilter_lock: RLock = field(default_factory=RLock) ssh_forwarding_lock: RLock = field(default_factory=RLock) artifact_copy_lock: Lock = field(default_factory=Lock) artifact_cache: Dict[str, str] = field(default_factory=_new_str_dict) device_pool_lock: Lock = field(default_factory=Lock) device_pool: Optional[Any] = None device_pool_config_key: str = "" + tap_network_lock: RLock = field(default_factory=RLock) + shared_tap_networks: Dict[str, SharedTapNetworkContext] = field( + default_factory=dict + ) def get_node_context(node: Node) -> NodeContext: diff --git a/lisa/sut_orchestrator/openvmm/node.py b/lisa/sut_orchestrator/openvmm/node.py index 5fb32001f6..f82242c00f 100644 --- a/lisa/sut_orchestrator/openvmm/node.py +++ b/lisa/sut_orchestrator/openvmm/node.py @@ -41,6 +41,7 @@ DeviceAddressSchema, DevicePassthroughContext, NodeContext, + SharedTapNetworkContext, get_host_context, get_node_context, ) @@ -170,6 +171,28 @@ def _shift_ip_address(address: str, base_cidr: str, index: int) -> str: return str(ipaddress.ip_address(new_address)) +def _increment_ip_address(address: str, cidr: str, index: int) -> str: + if not address or not cidr or index == 0: + return address + + host_interface = ipaddress.ip_interface(cidr) + network = host_interface.network + new_address = ipaddress.ip_address(int(ipaddress.ip_address(address)) + index) + if ( + new_address not in network + or new_address == network.network_address + or new_address == network.broadcast_address + or new_address == host_interface.ip + ): + raise LisaException( + f"cannot derive a shared OpenVMM guest address from '{address}' " + f"for guest index {index} within '{cidr}'. Use a lower base guest " + "address or a larger shared subnet." + ) + + return str(new_address) + + class GuestIpResolver(ABC): @abstractmethod def resolve( @@ -380,6 +403,30 @@ def create_effective_network( effective_network.tap_name = _increment_name_suffix( effective_network.tap_name, guest_index ) + if effective_network.shared_subnet: + try: + effective_network.validate_tap_interface_names() + except LisaException as identifier: + raise LisaException( + "cannot derive OpenVMM tap network interface names for guest " + f"index {guest_index}: {identifier}" + ) from identifier + if effective_network.address_mode == OPENVMM_ADDRESS_MODE_STATIC: + effective_network.guest_address = _increment_ip_address( + effective_network.guest_address, + effective_network.tap_host_cidr, + guest_index, + ) + if effective_network.forward_ssh_port: + effective_network.forwarded_port += guest_index + if effective_network.forwarded_port > 65535: + raise LisaException( + "cannot derive OpenVMM forwarded SSH port from " + f"'{network.forwarded_port}' for guest index {guest_index}: " + "derived port exceeds 65535. Use a lower base forwarded_port." + ) + return effective_network + effective_network.bridge_name = _increment_name_suffix( effective_network.bridge_name, guest_index ) @@ -695,17 +742,28 @@ def launch(self, node: "OpenVmmGuestNode", log: Logger) -> None: node_context = get_node_context(node) network = self._get_node_network(node, node_context) self._prepare_tap_network(network, node_context) + processor_count = _countspace_to_int(node.capability.core_count) launch_config = OpenVmmLaunchConfig( uefi_firmware_path=node_context.uefi_firmware_path, disk_img_path=node_context.disk_img_path, + disk_device=runbook.disk_device, + iommu=runbook.iommu, dvd_disk_paths=( [node_context.cloud_init_file_path] if node_context.cloud_init_file_path else [] ), - processors=_countspace_to_int(node.capability.core_count), + processors=processor_count, + vps_per_socket=( + runbook.vps_per_socket + if runbook.vps_per_socket is not None + else processor_count + ), + smt=runbook.smt, memory_mb=_countspace_to_int(node.capability.memory_mb), network_mode=network.mode, + network_device=network.device, + network_queue_count=network.queue_count, tap_name=getattr(network, "tap_name", ""), network_cidr=network.consomme_cidr, serial_mode=runbook.serial.mode, @@ -833,6 +891,59 @@ def _prepare_tap_network( if network.mode != OPENVMM_NETWORK_MODE_TAP: return + if network.shared_subnet: + host_context = get_host_context(self.host_node) + network_key = network.bridge_name + with host_context.tap_network_lock: + shared_context = host_context.shared_tap_networks.get(network_key) + if shared_context is None: + shared_context = SharedTapNetworkContext( + tap_host_cidr=network.tap_host_cidr, + address_mode=network.address_mode, + ) + host_context.shared_tap_networks[network_key] = shared_context + elif ( + shared_context.tap_host_cidr != network.tap_host_cidr + or shared_context.address_mode != network.address_mode + ): + raise LisaException( + f"OpenVMM shared bridge '{network_key}' is already using " + f"CIDR '{shared_context.tap_host_cidr}' and address mode " + f"'{shared_context.address_mode}'. Use matching shared " + "network settings for every guest on this bridge." + ) + + reuse_shared_resources = shared_context.reference_count > 0 + shared_context.reference_count += 1 + node_context.shared_tap_network_key = network_key + try: + self._prepare_tap_network_resources( + network, + node_context, + shared_context, + reuse_shared_resources, + ) + except Exception: + try: + self._teardown_tap_network(node_context, network) + except Exception as cleanup_identifier: + self._log.warning( + "failed to roll back OpenVMM shared TAP network " + f"'{network_key}' after setup failed: " + f"{cleanup_identifier}" + ) + raise + return + + self._prepare_tap_network_resources(network, node_context) + + def _prepare_tap_network_resources( + self, + network: OpenVmmNetworkSchema, + node_context: NodeContext, + shared_context: Optional[SharedTapNetworkContext] = None, + reuse_shared_resources: bool = False, + ) -> None: tap_name = network.tap_name bridge_name = network.bridge_name host = self.host_node @@ -845,7 +956,10 @@ def _prepare_tap_network( if not ip_tool.nic_exists(bridge_name): ip_tool.create_virtual_interface(bridge_name, "bridge") - node_context.tap_bridge_created = True + if shared_context: + shared_context.bridge_created = True + else: + node_context.tap_bridge_created = True host.execute( f"ip link set dev {shlex.quote(bridge_name)} type bridge stp_state 0", shell=True, @@ -948,41 +1062,52 @@ def _prepare_tap_network( ip_tool.up(tap_name) if network.address_mode != OPENVMM_ADDRESS_MODE_STATIC: - self._ensure_tap_host_services_input_allowed( - host_interface_name, node_context - ) - pid_file = f"/var/run/qemu-dnsmasq-{host_interface_name}.pid" - lease_file = f"/var/run/qemu-dnsmasq-{host_interface_name}.leases" - host.execute( - ( - f"test -f {shlex.quote(pid_file)} && " - f"kill $(cat {shlex.quote(pid_file)}) || true; " - f"rm -f {shlex.quote(pid_file)}; " - f"cp /dev/null {shlex.quote(lease_file)}" - ), - shell=True, - sudo=True, - expected_exit_code=0, - expected_exit_code_failure_message=( - "failed to reset OpenVMM dnsmasq state before starting " - f"DHCP on interface {host_interface_name}" - ), - ) - host.tools[Dnsmasq].start( - host_interface_name, - tap_gateway, - dhcp_range, - stop_firewall=False, - kill_existing=False, - pid_file=pid_file, - lease_file=lease_file, - dhcp_options=[ - f"option:router,{tap_gateway}", - f"option:dns-server,{tap_gateway}", - ], - ) - node_context.tap_dnsmasq_pid_file = pid_file - node_context.tap_dnsmasq_lease_file = lease_file + if shared_context and reuse_shared_resources: + node_context.tap_dnsmasq_pid_file = shared_context.dnsmasq_pid_file + node_context.tap_dnsmasq_lease_file = shared_context.dnsmasq_lease_file + else: + self._ensure_tap_host_services_input_allowed( + host_interface_name, node_context + ) + pid_file = f"/var/run/qemu-dnsmasq-{host_interface_name}.pid" + lease_file = f"/var/run/qemu-dnsmasq-{host_interface_name}.leases" + host.execute( + ( + f"test -f {shlex.quote(pid_file)} && " + f"kill $(cat {shlex.quote(pid_file)}) || true; " + f"rm -f {shlex.quote(pid_file)}; " + f"cp /dev/null {shlex.quote(lease_file)}" + ), + shell=True, + sudo=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "failed to reset OpenVMM dnsmasq state before starting " + f"DHCP on interface {host_interface_name}" + ), + ) + host.tools[Dnsmasq].start( + host_interface_name, + tap_gateway, + dhcp_range, + stop_firewall=False, + kill_existing=False, + pid_file=pid_file, + lease_file=lease_file, + dhcp_options=[ + f"option:router,{tap_gateway}", + f"option:dns-server,{tap_gateway}", + ], + ) + node_context.tap_dnsmasq_pid_file = pid_file + node_context.tap_dnsmasq_lease_file = lease_file + if shared_context: + shared_context.dnsmasq_pid_file = pid_file + shared_context.dnsmasq_lease_file = lease_file + shared_context.input_rules_added.extend( + node_context.tap_input_rules_added + ) + node_context.tap_input_rules_added.clear() self._log_tap_network_state(network, node_context) if node_context.tap_dnsmasq_pid_file: @@ -991,61 +1116,62 @@ def _prepare_tap_network( def _disable_bridge_netfilter(self, node_context: NodeContext) -> None: host = self.host_node host_context = get_host_context(host) - modprobe = host.tools[Modprobe] - if modprobe.module_exists("br_netfilter") and not modprobe.is_module_loaded( - "br_netfilter", force_run=True - ): - modprobe.load("br_netfilter") - - if host_context.active_bridge_netfilter_count > 0: - host_context.active_bridge_netfilter_count += 1 - node_context.tap_bridge_netfilter_disabled = True - return + with host_context.bridge_netfilter_lock: + modprobe = host.tools[Modprobe] + if modprobe.module_exists("br_netfilter") and not modprobe.is_module_loaded( + "br_netfilter", force_run=True + ): + modprobe.load("br_netfilter") - original_values = {} - for key in OPENVMM_BRIDGE_NETFILTER_KEYS: - value_result = host.execute( - f"sysctl -n {shlex.quote(key)}", - shell=True, - sudo=True, - no_info_log=True, - no_error_log=True, - expected_exit_code=None, - ) - if value_result.exit_code == 0: - original_values[key] = value_result.stdout.strip() + if host_context.active_bridge_netfilter_count > 0: + host_context.active_bridge_netfilter_count += 1 + node_context.tap_bridge_netfilter_disabled = True + return + + original_values = {} + for key in OPENVMM_BRIDGE_NETFILTER_KEYS: + value_result = host.execute( + f"sysctl -n {shlex.quote(key)}", + shell=True, + sudo=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=None, + ) + if value_result.exit_code == 0: + original_values[key] = value_result.stdout.strip() - if not original_values: - return + if not original_values: + return - host_context.original_bridge_netfilter_values = original_values - host_context.active_bridge_netfilter_count = 1 - node_context.tap_bridge_netfilter_disabled = True - try: - self._set_bridge_netfilter_values( - {key: "0" for key in original_values}, - failure_message=( - "failed to disable bridge netfilter on the OpenVMM host" - ), - ) - except Exception: + host_context.original_bridge_netfilter_values = original_values + host_context.active_bridge_netfilter_count = 1 + node_context.tap_bridge_netfilter_disabled = True try: self._set_bridge_netfilter_values( - original_values, + {key: "0" for key in original_values}, failure_message=( - "failed to roll back bridge netfilter after an OpenVMM " - "setup error" + "failed to disable bridge netfilter on the OpenVMM host" ), ) - except Exception as cleanup_identifier: - self._log.debug( - "failed to roll back bridge netfilter after setup error: " - f"{cleanup_identifier}" - ) - host_context.original_bridge_netfilter_values = {} - host_context.active_bridge_netfilter_count = 0 - node_context.tap_bridge_netfilter_disabled = False - raise + except Exception: + try: + self._set_bridge_netfilter_values( + original_values, + failure_message=( + "failed to roll back bridge netfilter after an " + "OpenVMM setup error" + ), + ) + except Exception as cleanup_identifier: + self._log.debug( + "failed to roll back bridge netfilter after setup error: " + f"{cleanup_identifier}" + ) + host_context.original_bridge_netfilter_values = {} + host_context.active_bridge_netfilter_count = 0 + node_context.tap_bridge_netfilter_disabled = False + raise def _set_bridge_netfilter_values( self, @@ -1118,6 +1244,26 @@ def _get_tap_host_service_input_rules(self, host_interface_name: str) -> List[st def _get_tap_network_config(self, network: OpenVmmNetworkSchema) -> tuple[str, str]: host_interface = ipaddress.ip_interface(network.tap_host_cidr) + if network.shared_subnet: + first_address = int(host_interface.network.network_address) + 1 + last_address = int(host_interface.network.broadcast_address) - 1 + gateway = int(host_interface.ip) + if first_address == gateway: + first_address += 1 + elif last_address == gateway: + last_address -= 1 + elif first_address < gateway < last_address: + first_address = gateway + 1 + if first_address > last_address: + raise LisaException( + f"OpenVMM shared subnet '{network.tap_host_cidr}' has no " + "address available for a guest. Use a larger subnet." + ) + return str(host_interface.ip), ( + f"{ipaddress.ip_address(first_address)}," + f"{ipaddress.ip_address(last_address)}" + ) + guest_ip = network.guest_address if not guest_ip: for address in host_interface.network.hosts(): @@ -1266,6 +1412,9 @@ def _get_tap_guest_address( network: OpenVmmNetworkSchema, log: Logger, ) -> str: + if network.shared_subnet: + return self._wait_for_shared_tap_lease(node_context, network, log) + _, dhcp_range = self._get_tap_network_config(network) guest_address = dhcp_range.split(",", maxsplit=1)[0].strip() if not guest_address: @@ -1276,6 +1425,91 @@ def _get_tap_guest_address( self._wait_for_tap_lease(node_context, guest_address, log, network) return guest_address + def _wait_for_shared_tap_lease( + self, + node_context: NodeContext, + network: OpenVmmNetworkSchema, + log: Logger, + timeout: int = OPENVMM_IP_DISCOVERY_TIMEOUT, + ) -> str: + lease_file = node_context.tap_dnsmasq_lease_file + if not lease_file: + raise LisaException( + "OpenVMM shared TAP DHCP lease tracking is not configured. " + "dnsmasq lease file path was not recorded." + ) + + guest_address = "" + + def _lease_is_ready() -> bool: + nonlocal guest_address + lease_result = self.host_node.execute( + ( + f"test -f {shlex.quote(lease_file)} && " + f"cat {shlex.quote(lease_file)} || true" + ), + shell=True, + sudo=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=0, + ) + fdb_result = self.host_node.execute( + f"bridge fdb show dev {shlex.quote(network.tap_name)}", + shell=True, + sudo=True, + no_info_log=True, + no_error_log=True, + expected_exit_code=0, + expected_exit_code_failure_message=( + "failed to inspect the OpenVMM shared bridge forwarding " + f"database for TAP interface {network.tap_name}" + ), + ) + tap_mac_addresses = { + line.split()[0].lower() + for line in fdb_result.stdout.splitlines() + if line.split() + } + for lease_line in lease_result.stdout.splitlines(): + lease_fields = lease_line.split() + if ( + len(lease_fields) >= 3 + and lease_fields[1].lower() in tap_mac_addresses + ): + guest_address = lease_fields[2] + log.debug( + "matched OpenVMM shared TAP interface " + f"'{network.tap_name}' to DHCP lease " + f"'{guest_address}' in {lease_file}" + ) + return True + if not self._is_process_running(node_context.process_id): + raise LisaException( + "OpenVMM process exited before the guest acquired a DHCP " + f"lease on shared TAP interface '{network.tap_name}'. " + f"{self._get_openvmm_failure_context(node_context, network)}" + ) + return False + + try: + check_till_timeout( + _lease_is_ready, + timeout_message=( + "wait for OpenVMM guest DHCP lease on shared TAP interface " + f"'{network.tap_name}'" + ), + timeout=timeout, + ) + except LisaTimeoutException as identifier: + raise LisaException( + "OpenVMM guest did not acquire a DHCP lease on shared TAP " + f"interface '{network.tap_name}'. " + f"{self._get_openvmm_failure_context(node_context, network)}" + ) from identifier + + return guest_address + def _wait_for_tap_lease( self, node_context: Any, @@ -1904,6 +2138,8 @@ def _teardown_tap_network( if network.mode != OPENVMM_NETWORK_MODE_TAP: return + shared_network_key = node_context.shared_tap_network_key + if node_context.tap_input_rules_added: rules_to_remove = node_context.tap_input_rules_added for rule in rules_to_remove: @@ -1915,7 +2151,7 @@ def _teardown_tap_network( ) node_context.tap_input_rules_added.clear() - if node_context.tap_dnsmasq_pid_file: + if not shared_network_key and node_context.tap_dnsmasq_pid_file: self.host_node.execute( ( f"test -f {shlex.quote(node_context.tap_dnsmasq_pid_file)} && " @@ -1938,7 +2174,9 @@ def _teardown_tap_network( ) node_context.tap_created = False - if node_context.tap_bridge_created and network.bridge_name: + if shared_network_key: + self._release_shared_tap_network(node_context, shared_network_key) + elif node_context.tap_bridge_created and network.bridge_name: self.host_node.execute( f"ip link delete {shlex.quote(network.bridge_name)} || true", shell=True, @@ -1949,23 +2187,67 @@ def _teardown_tap_network( if node_context.tap_bridge_netfilter_disabled: host_context = get_host_context(self.host_node) - if host_context.active_bridge_netfilter_count > 0: - host_context.active_bridge_netfilter_count -= 1 + with host_context.bridge_netfilter_lock: + if host_context.active_bridge_netfilter_count > 0: + host_context.active_bridge_netfilter_count -= 1 + + if ( + host_context.active_bridge_netfilter_count == 0 + and host_context.original_bridge_netfilter_values + ): + self._set_bridge_netfilter_values( + host_context.original_bridge_netfilter_values, + failure_message=( + "failed to restore bridge netfilter state on the " + "OpenVMM host" + ), + ) + host_context.original_bridge_netfilter_values = {} - if ( - host_context.active_bridge_netfilter_count == 0 - and host_context.original_bridge_netfilter_values - ): - self._set_bridge_netfilter_values( - host_context.original_bridge_netfilter_values, - failure_message=( - "failed to restore bridge netfilter state on the " - "OpenVMM host" + node_context.tap_bridge_netfilter_disabled = False + + def _release_shared_tap_network( + self, node_context: NodeContext, network_key: str + ) -> None: + host_context = get_host_context(self.host_node) + with host_context.tap_network_lock: + shared_context = host_context.shared_tap_networks.get(network_key) + node_context.shared_tap_network_key = "" + node_context.tap_dnsmasq_pid_file = "" + node_context.tap_dnsmasq_lease_file = "" + if shared_context is None: + return + + shared_context.reference_count -= 1 + if shared_context.reference_count > 0: + return + + for rule in shared_context.input_rules_added: + self.host_node.execute( + f"iptables -D {rule} || true", + shell=True, + sudo=True, + expected_exit_code=0, + ) + if shared_context.dnsmasq_pid_file: + self.host_node.execute( + ( + f"test -f {shlex.quote(shared_context.dnsmasq_pid_file)} " + "&& kill $(cat " + f"{shlex.quote(shared_context.dnsmasq_pid_file)}) || true" ), + shell=True, + sudo=True, + expected_exit_code=0, ) - host_context.original_bridge_netfilter_values = {} - - node_context.tap_bridge_netfilter_disabled = False + if shared_context.bridge_created: + self.host_node.execute( + f"ip link delete {shlex.quote(network_key)} || true", + shell=True, + sudo=True, + expected_exit_code=0, + ) + del host_context.shared_tap_networks[network_key] class OpenVmmGuestNode(RemoteNode): diff --git a/lisa/sut_orchestrator/openvmm/schema.py b/lisa/sut_orchestrator/openvmm/schema.py index 686c38db4d..8810d17bae 100644 --- a/lisa/sut_orchestrator/openvmm/schema.py +++ b/lisa/sut_orchestrator/openvmm/schema.py @@ -16,6 +16,18 @@ DevicePassthroughSchema, HostDevicePoolSchema, ) +from lisa.tools.openvmm import ( + OPENVMM_DISK_DEVICE_SCSI, + OPENVMM_DISK_DEVICE_VIRTIO_BLK, + OPENVMM_IOMMU_AMD, + OPENVMM_IOMMU_INTEL, + OPENVMM_IOMMU_NONE, + OPENVMM_NETWORK_DEVICE_SYNTHETIC, + OPENVMM_NETWORK_DEVICE_VIRTIO, + OPENVMM_SMT_AUTO, + OPENVMM_SMT_FORCE, + OPENVMM_SMT_OFF, +) from lisa.util import LisaException from .. import OPENVMM @@ -99,6 +111,16 @@ def __post_init__(self) -> None: @dataclass class OpenVmmNetworkSchema: mode: str = OPENVMM_NETWORK_MODE_USER + device: str = OPENVMM_NETWORK_DEVICE_SYNTHETIC + queue_count: Optional[int] = field( + default=None, + metadata=schema.field_metadata( + field_function=schema.fields.Int, + validate=schema.validate.Range(min=1, max=65535), + allow_none=True, + ), + ) + shared_subnet: bool = False connection_mode: str = OPENVMM_CONNECTION_MODE_FORWARDED_PORT address_mode: str = OPENVMM_ADDRESS_MODE_DISCOVER tap_name: str = "" @@ -160,7 +182,33 @@ def validate_tap_interface_names(self) -> None: if self.bridge_name: self._validate_interface_name("bridge_name", self.bridge_name) + def _validate_device(self) -> None: + if self.device not in [ + OPENVMM_NETWORK_DEVICE_SYNTHETIC, + OPENVMM_NETWORK_DEVICE_VIRTIO, + ]: + raise LisaException( + f"network device '{self.device}' is not supported for OpenVMM " + f"guests. Supported values: {OPENVMM_NETWORK_DEVICE_SYNTHETIC}, " + f"{OPENVMM_NETWORK_DEVICE_VIRTIO}" + ) + + def _validate_shared_subnet(self) -> None: + if not self.shared_subnet: + return + if self.mode != OPENVMM_NETWORK_MODE_TAP or not self.bridge_name: + raise LisaException( + "shared_subnet requires tap network mode and bridge_name" + ) + if self.address_mode != OPENVMM_ADDRESS_MODE_STATIC and self.guest_address: + raise LisaException( + "guest_address cannot be fixed when shared_subnet uses DHCP. " + "Remove guest_address or use address_mode 'static'." + ) + def __post_init__(self) -> None: + self._validate_device() + self._validate_shared_subnet() if self.connection_mode not in [ OPENVMM_CONNECTION_MODE_FORWARDED_PORT, OPENVMM_CONNECTION_MODE_HOST_PROXY, @@ -258,6 +306,17 @@ class OpenVmmGuestNodeSchema(schema.GuestNode): uefi: Optional[OpenVmmUefiSchema] = None disk_img: str = "" disk_img_is_remote_path: bool = False + disk_device: str = OPENVMM_DISK_DEVICE_SCSI + iommu: str = OPENVMM_IOMMU_NONE + vps_per_socket: Optional[int] = field( + default=None, + metadata=schema.field_metadata( + field_function=schema.fields.Int, + validate=schema.validate.Range(min=1), + allow_none=True, + ), + ) + smt: str = OPENVMM_SMT_OFF min_raw_disk_size_gb: int = field( default=OPENVMM_DEFAULT_MIN_RAW_DISK_SIZE_GB, metadata=schema.field_metadata( @@ -290,6 +349,31 @@ def __post_init__(self) -> None: ) if not self.disk_img: raise LisaException("disk_img is required for UEFI OpenVMM guests") + if self.disk_device not in [ + OPENVMM_DISK_DEVICE_SCSI, + OPENVMM_DISK_DEVICE_VIRTIO_BLK, + ]: + raise LisaException( + f"disk device '{self.disk_device}' is not supported for OpenVMM " + f"guests. Supported values: {OPENVMM_DISK_DEVICE_SCSI}, " + f"{OPENVMM_DISK_DEVICE_VIRTIO_BLK}" + ) + if self.iommu not in [ + OPENVMM_IOMMU_NONE, + OPENVMM_IOMMU_INTEL, + OPENVMM_IOMMU_AMD, + ]: + raise LisaException( + f"IOMMU '{self.iommu}' is not supported for OpenVMM guests. " + f"Supported values: {OPENVMM_IOMMU_NONE}, " + f"{OPENVMM_IOMMU_INTEL}, {OPENVMM_IOMMU_AMD}" + ) + if self.smt not in [OPENVMM_SMT_AUTO, OPENVMM_SMT_FORCE, OPENVMM_SMT_OFF]: + raise LisaException( + f"SMT mode '{self.smt}' is not supported for OpenVMM guests. " + f"Supported values: {OPENVMM_SMT_AUTO}, {OPENVMM_SMT_FORCE}, " + f"{OPENVMM_SMT_OFF}" + ) if ( self.cloud_init and not self.private_key_file diff --git a/lisa/tools/openvmm.py b/lisa/tools/openvmm.py index a1f5404541..697ec24f3c 100644 --- a/lisa/tools/openvmm.py +++ b/lisa/tools/openvmm.py @@ -14,6 +14,19 @@ OPENVMM_NETWORK_BACKEND_CONSOMME = "consomme" OPENVMM_DEFAULT_SCSI_CONTROLLER = "lisa_scsi0" +OPENVMM_DISK_DEVICE_SCSI = "scsi" +OPENVMM_DISK_DEVICE_VIRTIO_BLK = "virtio-blk" +OPENVMM_IOMMU_AMD = "amd-iommu" +OPENVMM_IOMMU_INTEL = "intel-vtd" +OPENVMM_IOMMU_NONE = "none" +OPENVMM_NETWORK_DEVICE_SYNTHETIC = "synthetic" +OPENVMM_NETWORK_DEVICE_VIRTIO = "virtio" +OPENVMM_SMT_AUTO = "auto" +OPENVMM_SMT_FORCE = "force" +OPENVMM_SMT_OFF = "off" +OPENVMM_VIRTIO_ROOT_COMPLEX = "lisa_virtio_rc0" +OPENVMM_VIRTIO_DISK_PORT = "lisa_virtio_disk" +OPENVMM_VIRTIO_NETWORK_PORT = "lisa_virtio_net" _COMMAND_NOT_FOUND_MARKERS = ( "command not found", @@ -37,10 +50,16 @@ class OpenVmmLaunchConfig: with_hv: bool = True hypervisor: str = "mshv" disk_img_path: str = "" + disk_device: str = OPENVMM_DISK_DEVICE_SCSI + iommu: str = OPENVMM_IOMMU_NONE dvd_disk_paths: List[str] = field(default_factory=_new_str_list) processors: int = 1 + vps_per_socket: Optional[int] = None + smt: str = "" memory_mb: int = 1024 network_mode: str = "user" + network_device: str = OPENVMM_NETWORK_DEVICE_SYNTHETIC + network_queue_count: Optional[int] = None tap_name: str = "" network_cidr: str = "" serial_mode: str = "file" @@ -100,7 +119,12 @@ def build_command(self, config: OpenVmmLaunchConfig) -> str: args.append("--hv") if config.hypervisor: args.extend(["--hypervisor", config.hypervisor]) + self._validate_processor_topology(config) args.extend(["--processors", str(config.processors)]) + if config.vps_per_socket is not None: + args.extend(["--vps-per-socket", str(config.vps_per_socket)]) + if config.smt: + args.extend(["--smt", config.smt]) args.extend(["--memory", f"{config.memory_mb}MB"]) if not config.uefi_firmware_path: @@ -108,17 +132,124 @@ def build_command(self, config: OpenVmmLaunchConfig) -> str: args.append("--uefi") args.extend(["--uefi-firmware", config.uefi_firmware_path]) - if config.disk_img_path or config.dvd_disk_paths: - args.extend(["--vmbus-scsi", f"id={OPENVMM_DEFAULT_SCSI_CONTROLLER}"]) + self._validate_device_types(config) + self._add_pcie_args(args, config) + self._add_disk_args(args, config) + network_backend = self._get_network_backend(config) + self._add_network_args(args, config, network_backend) - if config.disk_img_path: + if config.serial_mode == "stderr": + args.extend(["--com1", "stderr"]) + elif config.serial_mode == "file": + if not config.serial_path: + raise LisaException("serial_path must be provided for file serial mode") + args.extend(["--com1", f"file={config.serial_path}"]) + else: + raise LisaException(f"Unsupported serial mode: {config.serial_mode}") + + args.extend(config.extra_args) + return " ".join(shlex.quote(arg) for arg in args) + + def _validate_processor_topology(self, config: OpenVmmLaunchConfig) -> None: + if config.vps_per_socket is not None and config.vps_per_socket < 1: + raise LisaException( + "OpenVMM vps_per_socket must be at least 1. " + "Set it to the number of virtual processors in each socket." + ) + if config.smt and config.smt not in [ + OPENVMM_SMT_AUTO, + OPENVMM_SMT_FORCE, + OPENVMM_SMT_OFF, + ]: + raise LisaException( + f"OpenVMM SMT mode '{config.smt}' is not supported. " + f"Use {OPENVMM_SMT_AUTO}, {OPENVMM_SMT_FORCE}, or " + f"{OPENVMM_SMT_OFF}." + ) + + def _validate_device_types(self, config: OpenVmmLaunchConfig) -> None: + if config.disk_device not in [ + OPENVMM_DISK_DEVICE_SCSI, + OPENVMM_DISK_DEVICE_VIRTIO_BLK, + ]: + raise LisaException( + f"Unsupported OpenVMM disk device: {config.disk_device}" + ) + if config.network_device not in [ + OPENVMM_NETWORK_DEVICE_SYNTHETIC, + OPENVMM_NETWORK_DEVICE_VIRTIO, + ]: + raise LisaException( + f"Unsupported OpenVMM network device: {config.network_device}" + ) + if config.network_queue_count is not None and not ( + 1 <= config.network_queue_count <= 65535 + ): + raise LisaException( + "OpenVMM network queue count must be between 1 and 65535. " + "Set network.queue_count to a supported positive value." + ) + if config.iommu not in [ + OPENVMM_IOMMU_NONE, + OPENVMM_IOMMU_INTEL, + OPENVMM_IOMMU_AMD, + ]: + raise LisaException(f"Unsupported OpenVMM IOMMU: {config.iommu}") + + def _add_pcie_args(self, args: List[str], config: OpenVmmLaunchConfig) -> None: + use_virtio_disk = bool(config.disk_img_path) and ( + config.disk_device == OPENVMM_DISK_DEVICE_VIRTIO_BLK + ) + use_virtio_network = config.network_device == OPENVMM_NETWORK_DEVICE_VIRTIO + if use_virtio_disk or use_virtio_network: + args.extend(["--pcie-root-complex", OPENVMM_VIRTIO_ROOT_COMPLEX]) + if config.iommu != OPENVMM_IOMMU_NONE: + args.extend([f"--{config.iommu}", OPENVMM_VIRTIO_ROOT_COMPLEX]) + elif config.iommu != OPENVMM_IOMMU_NONE: + raise LisaException( + "OpenVMM IOMMU requires a virtio disk or network device on PCIe" + ) + if use_virtio_disk: args.extend( [ - "--disk", - f"file:{config.disk_img_path}," - f"on={OPENVMM_DEFAULT_SCSI_CONTROLLER},lun=0", + "--pcie-root-port", + f"{OPENVMM_VIRTIO_ROOT_COMPLEX}:{OPENVMM_VIRTIO_DISK_PORT}", ] ) + if use_virtio_network: + network_root_port = ( + f"{OPENVMM_VIRTIO_ROOT_COMPLEX}:{OPENVMM_VIRTIO_NETWORK_PORT}" + ) + args.extend( + [ + "--pcie-root-port", + network_root_port, + ] + ) + + def _add_disk_args(self, args: List[str], config: OpenVmmLaunchConfig) -> None: + if config.dvd_disk_paths or ( + config.disk_img_path and config.disk_device == OPENVMM_DISK_DEVICE_SCSI + ): + args.extend(["--vmbus-scsi", f"id={OPENVMM_DEFAULT_SCSI_CONTROLLER}"]) + + if config.disk_img_path: + if config.disk_device == OPENVMM_DISK_DEVICE_SCSI: + args.extend( + [ + "--disk", + f"file:{config.disk_img_path}," + f"on={OPENVMM_DEFAULT_SCSI_CONTROLLER},lun=0", + ] + ) + else: + args.extend( + [ + "--virtio-blk", + f"file:{config.disk_img_path}," + f"pcie_port={OPENVMM_VIRTIO_DISK_PORT}", + ] + ) for lun, dvd_disk_path in enumerate(config.dvd_disk_paths, start=1): args.extend( @@ -129,29 +260,36 @@ def build_command(self, config: OpenVmmLaunchConfig) -> str: ] ) + def _get_network_backend(self, config: OpenVmmLaunchConfig) -> str: if config.network_mode == "user": network_backend = OPENVMM_NETWORK_BACKEND_CONSOMME if config.network_cidr: network_backend = f"{network_backend}:{config.network_cidr}" - args.extend(["--net", network_backend]) elif config.network_mode == "tap": if not config.tap_name: raise LisaException("tap_name must be provided for tap networking") - args.extend(["--net", f"tap:{config.tap_name}"]) + network_backend = f"tap:{config.tap_name}" else: raise LisaException(f"Unsupported network mode: {config.network_mode}") + return network_backend - if config.serial_mode == "stderr": - args.extend(["--com1", "stderr"]) - elif config.serial_mode == "file": - if not config.serial_path: - raise LisaException("serial_path must be provided for file serial mode") - args.extend(["--com1", f"file={config.serial_path}"]) + def _add_network_args( + self, + args: List[str], + config: OpenVmmLaunchConfig, + network_backend: str, + ) -> None: + if config.network_queue_count is not None: + network_backend = f"queues={config.network_queue_count}:{network_backend}" + if config.network_device == OPENVMM_NETWORK_DEVICE_SYNTHETIC: + args.extend(["--net", network_backend]) else: - raise LisaException(f"Unsupported serial mode: {config.serial_mode}") - - args.extend(config.extra_args) - return " ".join(shlex.quote(arg) for arg in args) + args.extend( + [ + "--virtio-net", + f"pcie_port={OPENVMM_VIRTIO_NETWORK_PORT}:{network_backend}", + ] + ) def launch_vm( self, diff --git a/selftests/test_guest_node_schema.py b/selftests/test_guest_node_schema.py index 89c4dbf07b..d0ad8d0638 100644 --- a/selftests/test_guest_node_schema.py +++ b/selftests/test_guest_node_schema.py @@ -4,6 +4,7 @@ from typing import Any, cast from unittest import TestCase +import lisa.sut_orchestrator.openvmm.node # noqa: F401 from lisa import constants, schema from lisa.sut_orchestrator.openvmm.schema import ( OpenVmmGuestNodeSchema, diff --git a/selftests/test_openvmm_node.py b/selftests/test_openvmm_node.py index e1cafde91d..72d8d28041 100644 --- a/selftests/test_openvmm_node.py +++ b/selftests/test_openvmm_node.py @@ -15,6 +15,7 @@ from lisa.sut_orchestrator.openvmm.context import NodeContext from lisa.sut_orchestrator.openvmm.node import OpenVmmController, OpenVmmGuestNode from lisa.sut_orchestrator.openvmm.schema import ( + OPENVMM_ADDRESS_MODE_STATIC, OPENVMM_CONNECTION_MODE_HOST_PROXY, OPENVMM_NETWORK_MODE_TAP, OpenVmmGuestNodeSchema, @@ -26,6 +27,11 @@ SerialConsole as OpenVmmSerialConsole, ) from lisa.tools import Cat, Ip, Kill, Mkdir +from lisa.tools.openvmm import ( + OPENVMM_DISK_DEVICE_SCSI, + OPENVMM_IOMMU_NONE, + OPENVMM_NETWORK_DEVICE_SYNTHETIC, +) from lisa.util import LisaException @@ -131,7 +137,16 @@ def test_launch_uses_host_pure_path_for_cwd(self) -> None: node = SimpleNamespace( runbook=SimpleNamespace( openvmm_binary="/usr/local/bin/openvmm", - network=SimpleNamespace(mode="user", consomme_cidr=""), + disk_device=OPENVMM_DISK_DEVICE_SCSI, + iommu=OPENVMM_IOMMU_NONE, + vps_per_socket=None, + smt="off", + network=SimpleNamespace( + mode="user", + device=OPENVMM_NETWORK_DEVICE_SYNTHETIC, + queue_count=1, + consomme_cidr="", + ), serial=SimpleNamespace(mode="file"), extra_args=[], ), @@ -158,6 +173,10 @@ def test_launch_uses_host_pure_path_for_cwd(self) -> None: cwd=PurePosixPath("/var/tmp/openvmm-host-g0"), sudo=False, ) + launch_config = openvmm.launch_vm.call_args.args[0] + self.assertEqual(OPENVMM_DISK_DEVICE_SCSI, launch_config.disk_device) + self.assertEqual(OPENVMM_NETWORK_DEVICE_SYNTHETIC, launch_config.network_device) + self.assertEqual(1, launch_config.network_queue_count) def test_create_effective_network_derives_unique_tap_settings(self) -> None: controller, _, _, _ = self._create_controller() @@ -189,6 +208,73 @@ def test_create_effective_network_derives_unique_tap_settings(self) -> None: self.assertEqual("tap0", network.tap_name) self.assertEqual("10.0.0.1/24", network.tap_host_cidr) + def test_create_effective_network_reuses_shared_tap_subnet(self) -> None: + controller, _, _, _ = self._create_controller() + network = OpenVmmNetworkSchema( + mode=OPENVMM_NETWORK_MODE_TAP, + shared_subnet=True, + address_mode=OPENVMM_ADDRESS_MODE_STATIC, + tap_name="tap0", + bridge_name="ovmbr0", + tap_host_cidr="10.0.0.1/24", + guest_address="10.0.0.2", + forward_ssh_port=True, + forwarded_port=60022, + ) + + third_guest_network = controller.create_effective_network(network, 2) + + self.assertEqual("tap2", third_guest_network.tap_name) + self.assertEqual("ovmbr0", third_guest_network.bridge_name) + self.assertEqual("10.0.0.1/24", third_guest_network.tap_host_cidr) + self.assertEqual("10.0.0.4", third_guest_network.guest_address) + self.assertEqual(60024, third_guest_network.forwarded_port) + + def test_create_effective_network_rejects_shared_tap_name_overflow(self) -> None: + controller, _, _, _ = self._create_controller() + network = OpenVmmNetworkSchema( + mode=OPENVMM_NETWORK_MODE_TAP, + shared_subnet=True, + tap_name="abcdefghijklmno", + bridge_name="ovmbr0", + ) + + with self.assertRaisesRegex( + LisaException, + "cannot derive OpenVMM tap network interface names", + ): + controller.create_effective_network(network, 1) + + def test_shared_tap_setup_failure_removes_node_input_rules(self) -> None: + controller, _, _, _ = self._create_controller() + network = OpenVmmNetworkSchema( + mode=OPENVMM_NETWORK_MODE_TAP, + shared_subnet=True, + tap_name="tap0", + bridge_name="ovmbr0", + ) + node_context = NodeContext() + input_rule = "INPUT -i ovmbr0 -p udp --dport 67 -j ACCEPT" + + def _fail_after_adding_input_rule(*args: Any, **kwargs: Any) -> None: + node_context.tap_input_rules_added.append(input_rule) + raise LisaException("dnsmasq failed") + + with patch.object( + controller, + "_prepare_tap_network_resources", + side_effect=_fail_after_adding_input_rule, + ), self.assertRaisesRegex(LisaException, "dnsmasq failed"): + controller._prepare_tap_network(network, node_context) + + execute = cast(MagicMock, controller.host_node.execute) + self.assertIn( + f"iptables -D {input_rule} || true", + [call.args[0] for call in execute.call_args_list], + ) + self.assertEqual([], node_context.tap_input_rules_added) + self.assertEqual("", node_context.shared_tap_network_key) + def test_supported_features_include_serial_console(self) -> None: supported_feature_names = [ feature.name() for feature in OpenVmmController.supported_features() diff --git a/selftests/test_openvmm_schema.py b/selftests/test_openvmm_schema.py index b889213c84..0348022574 100644 --- a/selftests/test_openvmm_schema.py +++ b/selftests/test_openvmm_schema.py @@ -13,6 +13,10 @@ OpenVmmGuestNodeSchema, OpenVmmNetworkSchema, ) +from lisa.tools.openvmm import ( + OPENVMM_DISK_DEVICE_VIRTIO_BLK, + OPENVMM_NETWORK_DEVICE_VIRTIO, +) class OpenVmmSchemaTestCase(TestCase): @@ -39,6 +43,17 @@ def test_network_schema_rejects_invalid_ssh_port(self) -> None: } ) + def test_network_schema_accepts_valid_queue_count(self) -> None: + network_schema = cast(Any, OpenVmmNetworkSchema).schema() + network = network_schema.load({"queue_count": 1}) + + self.assertEqual(1, network.queue_count) + + def test_network_schema_rejects_invalid_queue_count(self) -> None: + network_schema = cast(Any, OpenVmmNetworkSchema).schema() + with self.assertRaises(ValidationError): + network_schema.load({"queue_count": 0}) + def test_guest_schema_splits_extra_args_string(self) -> None: guest_schema = cast(Any, OpenVmmGuestNodeSchema).schema() guest = guest_schema.load( @@ -51,6 +66,20 @@ def test_guest_schema_splits_extra_args_string(self) -> None: self.assertEqual(["--foo", "bar baz"], guest.extra_args) + def test_guest_schema_accepts_virtio_devices(self) -> None: + guest_schema = cast(Any, OpenVmmGuestNodeSchema).schema() + guest = guest_schema.load( + { + "uefi": {"firmware_path": "/firmware"}, + "disk_img": "/disk.raw", + "disk_device": OPENVMM_DISK_DEVICE_VIRTIO_BLK, + "network": {"device": OPENVMM_NETWORK_DEVICE_VIRTIO}, + } + ) + + self.assertEqual(OPENVMM_DISK_DEVICE_VIRTIO_BLK, guest.disk_device) + self.assertEqual(OPENVMM_NETWORK_DEVICE_VIRTIO, guest.network.device) + def test_host_proxy_connection_mode_disables_forwarded_port(self) -> None: network_schema = cast(Any, OpenVmmNetworkSchema).schema() network = network_schema.load( diff --git a/selftests/test_openvmm_tool.py b/selftests/test_openvmm_tool.py new file mode 100644 index 0000000000..b26f819b1f --- /dev/null +++ b/selftests/test_openvmm_tool.py @@ -0,0 +1,72 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from unittest import TestCase + +from lisa.tools.openvmm import ( + OPENVMM_DISK_DEVICE_VIRTIO_BLK, + OPENVMM_IOMMU_INTEL, + OPENVMM_NETWORK_DEVICE_VIRTIO, + OpenVmm, + OpenVmmLaunchConfig, +) + + +class OpenVmmToolTestCase(TestCase): + def _create_tool(self) -> OpenVmm: + tool = OpenVmm.__new__(OpenVmm) + tool.set_binary_path("openvmm") + return tool + + def test_build_command_uses_default_devices(self) -> None: + command = self._create_tool().build_command( + OpenVmmLaunchConfig( + uefi_firmware_path="/firmware/MSVM.fd", + disk_img_path="/disks/guest.raw", + network_mode="tap", + tap_name="tap0", + serial_path="/logs/console.log", + ) + ) + + self.assertIn("--vmbus-scsi id=lisa_scsi0", command) + self.assertIn("--disk file:/disks/guest.raw,on=lisa_scsi0,lun=0", command) + self.assertIn("--net tap:tap0", command) + self.assertNotIn("--pcie-root-complex", command) + self.assertNotIn("--virtio-blk", command) + self.assertNotIn("--virtio-net", command) + self.assertNotIn("queues=", command) + + def test_build_command_uses_virtio_devices_over_pcie(self) -> None: + command = self._create_tool().build_command( + OpenVmmLaunchConfig( + uefi_firmware_path="/firmware/MSVM.fd", + disk_img_path="/disks/guest.raw", + disk_device=OPENVMM_DISK_DEVICE_VIRTIO_BLK, + iommu=OPENVMM_IOMMU_INTEL, + dvd_disk_paths=["/disks/cloud-init.iso"], + network_mode="tap", + network_device=OPENVMM_NETWORK_DEVICE_VIRTIO, + network_queue_count=1, + tap_name="tap0", + serial_path="/logs/console.log", + ) + ) + + self.assertIn("--pcie-root-complex lisa_virtio_rc0", command) + self.assertIn("--intel-vtd lisa_virtio_rc0", command) + self.assertIn("--pcie-root-port lisa_virtio_rc0:lisa_virtio_disk", command) + self.assertIn("--pcie-root-port lisa_virtio_rc0:lisa_virtio_net", command) + self.assertIn( + "--virtio-blk file:/disks/guest.raw,pcie_port=lisa_virtio_disk", + command, + ) + self.assertIn( + "--virtio-net pcie_port=lisa_virtio_net:queues=1:tap:tap0", command + ) + self.assertIn("--vmbus-scsi id=lisa_scsi0", command) + self.assertIn( + "--disk file:/disks/cloud-init.iso,on=lisa_scsi0,lun=1,dvd", command + ) + self.assertNotIn("--disk file:/disks/guest.raw,on=lisa_scsi0,lun=0", command) + self.assertNotIn("--net tap:tap0", command)