devices: vfio_pci: MMIO BAR allocation for devices on non-root buses
Since we'll be putting all PCI devices on non-root buses behind virtual
PCI-E root ports, MMIO BARs in such devices must be inside the forward
windows of their root ports. This presents additional requirements for
their MMIO BAR allocation:
1. All non-prefetchable BARs must be inside the same 32-bit MMIO window
2. All prefetchable BARs must be inside the same MMIO window, but
different than the non-prefetchable MMIO window
3. Both windows must be 1MB-aligned
4. No other PCI devices should occupy MMIO space in these windows
Allocate the entire window from the system resource allocator to prevent
any space within the window from being used elsewhere. To maximize
memory space efficiency, use VfioResourceAllocator for BAR allocation.
BUG=b:185084350
TEST=passthrough a vfio-pci device with bus_number > 0 and static connet it
behind a pcie root port, then check pcie RP and vfio-pci device function in
guest.
Change-Id: Ic9865afc48eb3ff9fa475dbcfdf90642b012980c
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3166888
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>
Commit-Queue: Daniel Verkamp <dverkamp@chromium.org>
diff --git a/src/linux/mod.rs b/src/linux/mod.rs
index ed3eb4c..e128660 100644
--- a/src/linux/mod.rs
+++ b/src/linux/mod.rs
@@ -684,10 +684,29 @@
if host_pcie_rp.is_empty() {
// user doesn't specify host pcie root port which link to this virtual pcie rp,
// find the empty bus and create a total virtual pcie rp
- let sec_bus = (1..255)
- .find(|&bus_num| sys_allocator.pci_bus_empty(bus_num))
- .context("failed to find empty bus for Pci hotplug")?;
- let pcie_root_port = Arc::new(Mutex::new(PcieRootPort::new(sec_bus, true)));
+ let mut hp_sec_bus = 0u8;
+ // Create Pcie Root Port for non-root buses, each non-root bus device will be
+ // connected behind a virtual pcie root port.
+ for i in 1..255 {
+ if sys_allocator.pci_bus_empty(i) {
+ if hp_sec_bus == 0 {
+ hp_sec_bus = i;
+ }
+ continue;
+ }
+ let pcie_root_port = Arc::new(Mutex::new(PcieRootPort::new(i, false)));
+ let (msi_host_tube, msi_device_tube) = Tube::pair().context("failed to create tube")?;
+ control_tubes.push(TaggedControlTube::VmIrq(msi_host_tube));
+ let pci_bridge = Box::new(PciBridge::new(pcie_root_port.clone(), msi_device_tube));
+ // no ipc is used if the root port disables hotplug
+ devices.push((pci_bridge, None));
+ }
+
+ // Create Pcie Root Port for hot-plug
+ if hp_sec_bus == 0 {
+ return Err(anyhow!("no more addresses are available"));
+ }
+ let pcie_root_port = Arc::new(Mutex::new(PcieRootPort::new(hp_sec_bus, true)));
let (msi_host_tube, msi_device_tube) = Tube::pair().context("failed to create tube")?;
control_tubes.push(TaggedControlTube::VmIrq(msi_host_tube));
let pci_bridge = Box::new(PciBridge::new(pcie_root_port.clone(), msi_device_tube));