diff --git a/alioth/src/board/board.rs b/alioth/src/board/board.rs index 507a58c8..fe983c40 100644 --- a/alioth/src/board/board.rs +++ b/alioth/src/board/board.rs @@ -20,6 +20,7 @@ mod aarch64; mod x86_64; use std::ffi::CStr; +use std::os::fd::BorrowedFd; use std::sync::Arc; use libc::{MAP_PRIVATE, MAP_SHARED}; @@ -187,11 +188,21 @@ where Ok(()) } - fn dev_mem_added(&self, gpa: u64, pages: &ArcMemPages) -> mem::Result<()> { + fn dev_mem_added( + &self, + gpa: u64, + pages: &ArcMemPages, + _: Option, + ) -> mem::Result<()> { self.ram_added(gpa, pages) } - fn dev_mem_removed(&self, gpa: u64, pages: &ArcMemPages) -> mem::Result<()> { + fn dev_mem_removed( + &self, + gpa: u64, + pages: &ArcMemPages, + _: Option, + ) -> mem::Result<()> { self.ram_removed(gpa, pages) } } diff --git a/alioth/src/board/board_amd64/sev.rs b/alioth/src/board/board_amd64/sev.rs index cc2385be..b0a8db04 100644 --- a/alioth/src/board/board_amd64/sev.rs +++ b/alioth/src/board/board_amd64/sev.rs @@ -14,6 +14,7 @@ use std::arch::x86_64::{__cpuid, CpuidResult}; use std::collections::HashMap; +use std::os::fd::BorrowedFd; use std::sync::Arc; use crate::arch::cpuid::{ @@ -149,11 +150,11 @@ where Ok(()) } - fn dev_mem_added(&self, _: u64, _: &ArcMemPages) -> mem::Result<()> { + fn dev_mem_added(&self, _: u64, _: &ArcMemPages, _: Option) -> mem::Result<()> { Ok(()) } - fn dev_mem_removed(&self, _: u64, _: &ArcMemPages) -> mem::Result<()> { + fn dev_mem_removed(&self, _: u64, _: &ArcMemPages, _: Option) -> mem::Result<()> { Ok(()) } } diff --git a/alioth/src/mem/mem.rs b/alioth/src/mem/mem.rs index fdfe5919..04f3a46c 100644 --- a/alioth/src/mem/mem.rs +++ b/alioth/src/mem/mem.rs @@ -18,6 +18,7 @@ pub mod mapped; use std::any::{Any, type_name}; use std::fmt::Debug; +use std::os::fd::{AsFd, BorrowedFd, OwnedFd}; use std::sync::Arc; use parking_lot::{Mutex, RwLock}; @@ -124,7 +125,10 @@ impl MemSpec { #[derive(Debug)] pub enum MemRange { Ram(ArcMemPages), - DevMem(ArcMemPages), + DevMem { + pages: ArcMemPages, + dma_buf: Option, + }, Emulated(Arc), Span(u64), } @@ -132,7 +136,7 @@ pub enum MemRange { impl MemRange { pub fn size(&self) -> u64 { match self { - MemRange::Ram(pages) | MemRange::DevMem(pages) => pages.size(), + MemRange::Ram(pages) | MemRange::DevMem { pages, .. } => pages.size(), MemRange::Emulated(range) => Mmio::size(range), MemRange::Span(size) => *size, } @@ -186,7 +190,10 @@ impl MemRegion { pub fn with_dev_mem(pages: ArcMemPages, type_: MemRegionType) -> MemRegion { let size = pages.size(); MemRegion { - ranges: vec![MemRange::DevMem(pages)], + ranges: vec![MemRange::DevMem { + pages, + dma_buf: None, + }], entries: vec![MemRegionEntry { type_, size }], callbacks: Mutex::new(vec![]), } @@ -245,8 +252,8 @@ impl SlotBackend for Arc { pub trait LayoutChanged: Debug + Send + Sync + 'static { fn ram_added(&self, gpa: u64, pages: &ArcMemPages) -> Result<()>; fn ram_removed(&self, gpa: u64, pages: &ArcMemPages) -> Result<()>; - fn dev_mem_added(&self, gpa: u64, pages: &ArcMemPages) -> Result<()>; - fn dev_mem_removed(&self, gpa: u64, pages: &ArcMemPages) -> Result<()>; + fn dev_mem_added(&self, gpa: u64, pages: &ArcMemPages, fd: Option) -> Result<()>; + fn dev_mem_removed(&self, gpa: u64, pages: &ArcMemPages, fd: Option) -> Result<()>; } pub trait LayoutUpdated: Debug + Send + Sync + 'static { @@ -285,7 +292,10 @@ impl Memory { let gpa = addr + offset; match range { MemRange::Ram(r) => callback.ram_added(gpa, r)?, - MemRange::Span(_) | MemRange::Emulated(_) | MemRange::DevMem(_) => {} + MemRange::DevMem { pages, dma_buf } => { + callback.dev_mem_added(gpa, pages, dma_buf.as_ref().map(AsFd::as_fd))? + } + MemRange::Span(_) | MemRange::Emulated(_) => {} } offset += range.size(); } @@ -327,8 +337,8 @@ impl Memory { let mut ram_updated = false; for range in ®ion.ranges { let gpa = addr + offset; - if let MemRange::Ram(r) | MemRange::DevMem(r) = range { - self.ram_bus.add(gpa, r.clone())?; + if let MemRange::Ram(pages) | MemRange::DevMem { pages, .. } = range { + self.ram_bus.add(gpa, pages.clone())?; ram_updated = true; } match range { @@ -341,9 +351,9 @@ impl Memory { callback.ram_added(gpa, r)?; } } - MemRange::DevMem(r) => { + MemRange::DevMem { pages: r, dma_buf } => { for callback in &callbacks.changed { - callback.dev_mem_added(gpa, r)?; + callback.dev_mem_added(gpa, r, dma_buf.as_ref().map(AsFd::as_fd))?; } } MemRange::Span(_) => {} @@ -369,7 +379,7 @@ impl Memory { let mut ram_updated = false; for range in ®ion.ranges { let gpa = addr + offset; - if let MemRange::Ram(_) | MemRange::DevMem(_) = range { + if let MemRange::Ram(_) | MemRange::DevMem { .. } = range { self.ram_bus.remove(gpa)?; ram_updated = true; } @@ -383,9 +393,9 @@ impl Memory { callback.ram_removed(gpa, r)?; } } - MemRange::DevMem(r) => { + MemRange::DevMem { pages, dma_buf } => { for callback in callbacks.changed.iter().rev() { - callback.dev_mem_removed(gpa, r)?; + callback.dev_mem_removed(gpa, pages, dma_buf.as_ref().map(AsFd::as_fd))?; } } MemRange::Span(_) => {} @@ -511,11 +521,11 @@ impl Memory { 'out: while let Some((mut addr, region)) = regions.search_next(start) { let next_start = addr + region.size(); for range in ®ion.ranges { - let (MemRange::DevMem(r) | MemRange::Ram(r)) = range else { + let (MemRange::DevMem { pages, .. } | MemRange::Ram(pages)) = range else { addr += range.size(); continue; }; - let range_end = addr + r.size(); + let range_end = addr + pages.size(); if range_end <= start { addr = range_end; continue; @@ -602,11 +612,11 @@ where Ok(()) } - fn dev_mem_added(&self, _: u64, _: &ArcMemPages) -> Result<()> { + fn dev_mem_added(&self, _: u64, _: &ArcMemPages, _: Option) -> Result<()> { Ok(()) } - fn dev_mem_removed(&self, _: u64, _: &ArcMemPages) -> Result<()> { + fn dev_mem_removed(&self, _: u64, _: &ArcMemPages, _: Option) -> Result<()> { Ok(()) } } diff --git a/alioth/src/sys/linux/ioctl.rs b/alioth/src/sys/linux/ioctl.rs index bf3debbb..a50a601a 100644 --- a/alioth/src/sys/linux/ioctl.rs +++ b/alioth/src/sys/linux/ioctl.rs @@ -80,9 +80,9 @@ macro_rules! ioctl_write_val { #[macro_export] macro_rules! ioctl_write_ptr { - ($name:ident, $code:expr, $ty:ty) => { + ($name:ident, $code:expr, $ty:ty, $($g:ident),*) => { #[allow(clippy::missing_safety_doc)] - pub unsafe fn $name( + pub unsafe fn $name( fd: &F, val: &$ty, ) -> ::std::io::Result { @@ -96,6 +96,9 @@ macro_rules! ioctl_write_ptr { }) } }; + ($name:ident, $code:expr, $ty:ty) => { + $crate::ioctl_write_ptr!($name, $code, $ty, ); + }; ($name:ident, $type_:expr, $nr:expr, $ty:ty) => { $crate::ioctl_write_ptr!( $name, diff --git a/alioth/src/sys/linux/iommufd.rs b/alioth/src/sys/linux/iommufd.rs new file mode 100644 index 00000000..4d763efb --- /dev/null +++ b/alioth/src/sys/linux/iommufd.rs @@ -0,0 +1,36 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::ioctl_write_ptr; +use crate::sys::ioctl::ioctl_io; +use crate::sys::vfio::IommuIoasMapFlag; + +pub const IOMMUFD_TYPE: u8 = b';'; + +#[repr(C)] +pub struct IommuIoasMapFile { + pub size: u32, + pub flags: IommuIoasMapFlag, + pub ioas_id: u32, + pub fd: i32, + pub start: u64, + pub length: u64, + pub iova: u64, +} + +ioctl_write_ptr! { + iommu_ioas_map_file, + ioctl_io(IOMMUFD_TYPE, 0x8f), + IommuIoasMapFile +} diff --git a/alioth/src/sys/linux/linux.rs b/alioth/src/sys/linux/linux.rs index 46a9637d..3585acc7 100644 --- a/alioth/src/sys/linux/linux.rs +++ b/alioth/src/sys/linux/linux.rs @@ -14,6 +14,7 @@ pub mod if_tun; pub mod ioctl; +pub mod iommufd; pub mod kvm; pub mod sev; pub mod tdx; diff --git a/alioth/src/sys/linux/vfio.rs b/alioth/src/sys/linux/vfio.rs index ea3e7c1d..4d7b7edb 100644 --- a/alioth/src/sys/linux/vfio.rs +++ b/alioth/src/sys/linux/vfio.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use bitfield::bitfield; + use crate::sys::ioctl::ioctl_io; use crate::{ bitflags, consts, ioctl_none, ioctl_write_buf, ioctl_write_ptr, ioctl_write_val, @@ -262,6 +264,48 @@ consts! { } } +consts! { + pub struct DeviceFeature(u16) { + DMA_BUF = 11; + } +} + +#[derive(Debug, Clone)] +#[repr(C)] +pub struct VfioRegionDmaRange { + pub offset: u64, + pub length: u64, +} + +#[derive(Debug, Clone)] +#[repr(C)] +pub struct VfioDeviceFeatureDmaBuf { + pub region_index: u32, + pub open_flags: u32, + pub flags: u32, + pub nr_ranges: u32, + pub dma_ranges: [VfioRegionDmaRange; N], +} + +bitfield! { + #[derive(Copy, Clone, Default, PartialEq, Eq, Hash)] + pub struct VfioDeviceFeatureFlag(u32); + impl Debug; + impl new; + pub u16, from into DeviceFeature, index, set_index: 15, 0; + pub get, set_get: 16; + pub set, set_set: 17; + pub probe, set_probe: 18; +} + +#[repr(C)] +#[derive(Debug, Clone)] +pub struct VfioDeviceFeature { + pub argsz: u32, + pub flags: VfioDeviceFeatureFlag, + pub data: T, +} + ioctl_writeread!( vfio_device_get_info, ioctl_io(VFIO_TYPE, 107), @@ -282,6 +326,13 @@ ioctl_write_buf!(vfio_device_set_irqs, ioctl_io(VFIO_TYPE, 110), VfioIrqSet); ioctl_none!(vfio_device_reset, VFIO_TYPE, 111); +ioctl_write_ptr!( + vfio_device_feature, + ioctl_io(VFIO_TYPE, 117), + VfioDeviceFeature, + T +); + ioctl_write_ptr!( vfio_device_bind_iommufd, ioctl_io(VFIO_TYPE, 118), diff --git a/alioth/src/vfio/container.rs b/alioth/src/vfio/container.rs index 30dfe99e..0d57c4db 100644 --- a/alioth/src/vfio/container.rs +++ b/alioth/src/vfio/container.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::fs::File; -use std::os::fd::AsRawFd; +use std::os::fd::{AsRawFd, BorrowedFd}; use std::path::Path; use std::sync::Arc; @@ -120,11 +120,21 @@ impl LayoutChanged for UpdateContainerMapping { Ok(()) } - fn dev_mem_added(&self, gpa: u64, pages: &ArcMemPages) -> mem::Result<()> { + fn dev_mem_added( + &self, + gpa: u64, + pages: &ArcMemPages, + _: Option, + ) -> mem::Result<()> { self.ram_added(gpa, pages) } - fn dev_mem_removed(&self, gpa: u64, pages: &ArcMemPages) -> mem::Result<()> { + fn dev_mem_removed( + &self, + gpa: u64, + pages: &ArcMemPages, + _: Option, + ) -> mem::Result<()> { self.ram_removed(gpa, pages) } } diff --git a/alioth/src/vfio/device.rs b/alioth/src/vfio/device.rs index c321a656..d31ecbfe 100644 --- a/alioth/src/vfio/device.rs +++ b/alioth/src/vfio/device.rs @@ -15,14 +15,15 @@ use std::fmt::Debug; use std::fs::File; use std::mem::size_of; -use std::os::fd::AsRawFd; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; use std::os::unix::fs::FileExt; use crate::mem; use crate::sys::vfio::{ + DeviceFeature, VfioDeviceFeature, VfioDeviceFeatureDmaBuf, VfioDeviceFeatureFlag, VfioDeviceInfo, VfioIrqInfo, VfioIrqSet, VfioIrqSetData, VfioIrqSetFlag, VfioPciIrq, - VfioRegionInfo, vfio_device_get_info, vfio_device_get_irq_info, vfio_device_get_region_info, - vfio_device_reset, vfio_device_set_irqs, + VfioRegionDmaRange, VfioRegionInfo, vfio_device_feature, vfio_device_get_info, + vfio_device_get_irq_info, vfio_device_get_region_info, vfio_device_reset, vfio_device_set_irqs, }; use crate::vfio::Result; @@ -38,6 +39,25 @@ pub trait Device: Debug + Send + Sync + 'static { Ok(device_info) } + fn get_dma_buf_fd(&self, index: u32, offset: u64, size: usize) -> Result { + let req = VfioDeviceFeature { + argsz: size_of::>>() as u32, + flags: VfioDeviceFeatureFlag::new(DeviceFeature::DMA_BUF, true, false, false), + data: VfioDeviceFeatureDmaBuf { + region_index: index, + open_flags: (libc::O_RDWR | libc::O_CLOEXEC) as u32, + flags: 0, + nr_ranges: 1, + dma_ranges: [VfioRegionDmaRange { + offset, + length: size as u64, + }], + }, + }; + let fd = unsafe { vfio_device_feature(self.fd(), &req) }?; + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) + } + fn get_region_info(&self, index: u32) -> Result { let mut region_config = VfioRegionInfo { argsz: size_of::() as u32, diff --git a/alioth/src/vfio/iommu.rs b/alioth/src/vfio/iommu.rs index a45bbd7b..58d979a0 100644 --- a/alioth/src/vfio/iommu.rs +++ b/alioth/src/vfio/iommu.rs @@ -14,7 +14,7 @@ use std::fs::File; use std::mem::size_of; -use std::os::fd::AsRawFd; +use std::os::fd::{AsRawFd, BorrowedFd}; use std::path::Path; use std::sync::Arc; @@ -23,6 +23,7 @@ use snafu::ResultExt; use crate::errors::BoxTrace; use crate::mem::mapped::ArcMemPages; use crate::mem::{self, LayoutChanged}; +use crate::sys::iommufd::{IommuIoasMapFile, iommu_ioas_map_file}; use crate::sys::vfio::{ IommuDestroy, IommuIoasAlloc, IommuIoasMap, IommuIoasMapFlag, IommuIoasUnmap, iommu_destroy, iommu_ioas_alloc, iommu_ioas_map, iommu_ioas_unmap, @@ -99,6 +100,28 @@ impl Ioas { Ok(()) } + pub fn map_file(&self, fd: BorrowedFd, start: u64, iova: u64, len: u64) -> Result<()> { + let ioas_map_file = IommuIoasMapFile { + size: size_of::() as u32, + flags: IommuIoasMapFlag::READABLE + | IommuIoasMapFlag::WRITEABLE + | IommuIoasMapFlag::FIXED_IOVA, + ioas_id: self.id, + fd: fd.as_raw_fd(), + start, + length: len, + iova, + }; + log::debug!( + "ioas-{}-{}: mapped file: {iova:#018x} -> fd = {:?}, offset = {start:#x}, size = {len:#x}", + fd.as_raw_fd(), + self.iommu.fd.as_raw_fd(), + self.id, + ); + unsafe { iommu_ioas_map_file(&self.iommu.fd, &ioas_map_file) }?; + Ok(()) + } + pub fn unmap(&self, iova: u64, len: u64) -> Result<()> { let ioas_unmap = IommuIoasUnmap { size: size_of::() as u32, @@ -138,12 +161,31 @@ impl LayoutChanged for UpdateIommuIoas { Ok(()) } - fn dev_mem_added(&self, _gpa: u64, _pages: &ArcMemPages) -> mem::Result<()> { - // Iommufd does not support mapping device memory into IOAS. - Ok(()) + fn dev_mem_added( + &self, + gpa: u64, + pages: &ArcMemPages, + dma_buf: Option, + ) -> mem::Result<()> { + if let Some(fd) = dma_buf { + self.ioas + .map_file(fd, 0, gpa, pages.size()) + .box_trace(mem::error::ChangeLayout) + } else { + self.ioas + .map(pages.addr(), gpa, pages.size()) + .box_trace(mem::error::ChangeLayout) + } } - fn dev_mem_removed(&self, _gpa: u64, _pages: &ArcMemPages) -> mem::Result<()> { - Ok(()) + fn dev_mem_removed( + &self, + gpa: u64, + pages: &ArcMemPages, + _: Option, + ) -> mem::Result<()> { + self.ioas + .unmap(gpa, pages.size()) + .box_trace(mem::error::ChangeLayout) } } diff --git a/alioth/src/vfio/pci.rs b/alioth/src/vfio/pci.rs index b2286166..df2e15d1 100644 --- a/alioth/src/vfio/pci.rs +++ b/alioth/src/vfio/pci.rs @@ -13,11 +13,10 @@ // limitations under the License. use std::cmp::min; -use std::fs::File; use std::iter::zip; use std::mem::size_of; use std::ops::Range; -use std::os::fd::{AsFd, AsRawFd}; +use std::os::fd::{AsFd, AsRawFd, OwnedFd}; use std::os::unix::fs::FileExt; use std::sync::Arc; use std::sync::atomic::AtomicU64; @@ -53,21 +52,31 @@ fn round_up_range(range: Range) -> Range { (align_down!(range.start, 12))..(align_up!(range.end, 12)) } -fn create_mapped_bar_pages( - fd: &File, - region_flags: VfioRegionInfoFlag, - offset: i64, +fn create_mapped_bar_pages( + dev: &VfioDev, + region: &VfioRegionInfo, + offset: u64, size: usize, -) -> Result { +) -> Result<(ArcMemPages, Option)> { + let dma_buf = match dev.dev.get_dma_buf_fd(region.index, offset, size) { + Ok(fd) => Some(fd), + Err(e) => { + log::warn!("{}: failed to get dma buf fd: {e:?}", dev.name); + None + } + }; + let mut prot = 0; - if region_flags.contains(VfioRegionInfoFlag::READ) { + if region.flags.contains(VfioRegionInfoFlag::READ) { prot |= PROT_READ; } - if region_flags.contains(VfioRegionInfoFlag::WRITE) { + if region.flags.contains(VfioRegionInfoFlag::WRITE) { prot |= PROT_WRITE; } - let mapped_pages = ArcMemPages::from_file(fd.try_clone()?, offset, size, prot)?; - Ok(mapped_pages) + let dev_fd = dev.dev.fd().try_clone()?; + let dev_fd_offset = region.offset + offset; + let mapped_pages = ArcMemPages::from_file(dev_fd, dev_fd_offset as i64, size, prot)?; + Ok((mapped_pages, dma_buf)) } fn create_splitted_bar_region( @@ -107,12 +116,8 @@ where ranges: vec![], }; if excluded_page1.start > 0 { - region.ranges.push(MemRange::DevMem(create_mapped_bar_pages( - dev.dev.fd(), - region_info.flags, - region_info.offset as i64, - excluded_page1.start, - )?)); + let (pages, dma_buf) = create_mapped_bar_pages(&dev, region_info, 0, excluded_page1.start)?; + region.ranges.push(MemRange::DevMem { pages, dma_buf }); } if excluded_page1.end - excluded_page1.start > 0 { region.ranges.push(MemRange::Emulated(Arc::new(MsixBarMmio { @@ -128,12 +133,13 @@ where }))); } if excluded_page2.start - excluded_page1.end > 0 { - region.ranges.push(MemRange::DevMem(create_mapped_bar_pages( - dev.dev.fd(), - region_info.flags, - region_info.offset as i64 + excluded_page1.end as i64, + let (pages, dma_buf) = create_mapped_bar_pages( + &dev, + region_info, + excluded_page1.end as u64, excluded_page2.start - excluded_page1.end, - )?)); + )?; + region.ranges.push(MemRange::DevMem { pages, dma_buf }); } if excluded_page2.end - excluded_page2.start > 0 { region.ranges.push(MemRange::Emulated(Arc::new(MsixBarMmio { @@ -149,12 +155,13 @@ where }))); } if excluded_page2.end < region_info.size as usize { - region.ranges.push(MemRange::DevMem(create_mapped_bar_pages( - dev.dev.fd(), - region_info.flags, - region_info.offset as i64 + excluded_page2.end as i64, + let (pages, dma_buf) = create_mapped_bar_pages( + &dev, + region_info, + excluded_page2.end as u64, region_info.size as usize - excluded_page2.end, - )?)); + )?; + region.ranges.push(MemRange::DevMem { pages, dma_buf }); } Ok(region) } diff --git a/alioth/src/virtio/vu/frontend.rs b/alioth/src/virtio/vu/frontend.rs index e83f822b..faf610b3 100644 --- a/alioth/src/virtio/vu/frontend.rs +++ b/alioth/src/virtio/vu/frontend.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::os::fd::{AsFd, AsRawFd, FromRawFd, OwnedFd}; +use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, OwnedFd}; use std::path::Path; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -90,11 +90,21 @@ impl LayoutChanged for UpdateVuMem { Ok(()) } - fn dev_mem_added(&self, gpa: u64, pages: &ArcMemPages) -> mem::Result<()> { + fn dev_mem_added( + &self, + gpa: u64, + pages: &ArcMemPages, + _: Option, + ) -> mem::Result<()> { self.ram_added(gpa, pages) } - fn dev_mem_removed(&self, gpa: u64, pages: &ArcMemPages) -> mem::Result<()> { + fn dev_mem_removed( + &self, + gpa: u64, + pages: &ArcMemPages, + _: Option, + ) -> mem::Result<()> { self.ram_removed(gpa, pages) } }