Skip to content

Commit d695d39

Browse files
bchaliosbchalios
committed
refactor: move Buses to Vm object
We had previously added MMIO and Port IO buses inside ResourceAllocator so that we could implement DeviceRelocation for the type. Now, we will delegate device relocation responsibilities to ArchVm instead. That is because device relocation requires access to the Vm file descriptor as well. As a result, we can move buses to the Vm object itself. Add MMIO bus to VmCommon as both architectures use it. Add PortIO bus for x86 architecture only. Signed-off-by: Babis Chalios <[email protected]>
1 parent 9f839e8 commit d695d39

File tree

10 files changed

+73
-93
lines changed

10 files changed

+73
-93
lines changed

‎src/vmm/src/arch/x86_64/vm.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-License-Identifier: Apache-2.0
33

44
use std::fmt;
5+
use std::sync::Arc;
56

67
use kvm_bindings::{
78
KVM_CLOCK_TSC_STABLE, KVM_IRQCHIP_IOAPIC, KVM_IRQCHIP_PIC_MASTER, KVM_IRQCHIP_PIC_SLAVE,
@@ -58,6 +59,8 @@ pub struct ArchVm {
5859
///
5960
/// `None` if `KVM_CAP_XSAVE2` not supported.
6061
xsave2_size: Option<usize>,
62+
/// Port IO bus
63+
pub pio_bus: Arc<vm_device::Bus>,
6164
}
6265

6366
impl ArchVm {
@@ -92,10 +95,13 @@ impl ArchVm {
9295
.set_tss_address(u64_to_usize(crate::arch::x86_64::layout::KVM_TSS_ADDRESS))
9396
.map_err(ArchVmError::SetTssAddress)?;
9497

98+
let pio_bus = Arc::new(vm_device::Bus::new());
99+
95100
Ok(ArchVm {
96101
common,
97102
msrs_to_save,
98103
xsave2_size,
104+
pio_bus,
99105
})
100106
}
101107

‎src/vmm/src/device_manager/legacy.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ impl PortIODeviceManager {
119119
input: None,
120120
}));
121121

122-
let io_bus = &vm.common.resource_allocator.pio_bus;
122+
let io_bus = &vm.pio_bus;
123123
io_bus.insert(
124124
self.stdio_serial.clone(),
125125
Self::SERIAL_PORT_ADDRESSES[0],

‎src/vmm/src/device_manager/mmio.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ impl MMIODeviceManager {
209209
.map_err(MmioError::RegisterIrqFd)?;
210210
}
211211

212-
vm.common.resource_allocator.mmio_bus.insert(
212+
vm.common.mmio_bus.insert(
213213
device.inner.clone(),
214214
device.resources.addr,
215215
device.resources.len,
@@ -303,7 +303,7 @@ impl MMIODeviceManager {
303303
inner: serial,
304304
};
305305

306-
vm.common.resource_allocator.mmio_bus.insert(
306+
vm.common.mmio_bus.insert(
307307
device.inner.clone(),
308308
device.resources.addr,
309309
device.resources.len,
@@ -334,7 +334,7 @@ impl MMIODeviceManager {
334334
/// given as parameter, otherwise allocate a new MMIO resources for it.
335335
pub fn register_mmio_rtc(
336336
&mut self,
337-
resource_allocator: &ResourceAllocator,
337+
vm: &Vm,
338338
rtc: Arc<Mutex<RTCDevice>>,
339339
device_info_opt: Option<MMIODeviceInfo>,
340340
) -> Result<(), MmioError> {
@@ -343,7 +343,7 @@ impl MMIODeviceManager {
343343
let device_info = if let Some(device_info) = device_info_opt {
344344
device_info
345345
} else {
346-
let gsi = resource_allocator.allocate_gsi(1)?;
346+
let gsi = vm.common.resource_allocator.allocate_gsi(1)?;
347347
MMIODeviceInfo {
348348
addr: RTC_MEM_START,
349349
len: MMIO_LEN,
@@ -356,7 +356,7 @@ impl MMIODeviceManager {
356356
inner: rtc,
357357
};
358358

359-
resource_allocator.mmio_bus.insert(
359+
vm.common.mmio_bus.insert(
360360
device.inner.clone(),
361361
device.resources.addr,
362362
device.resources.len,

‎src/vmm/src/device_manager/mod.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ impl DeviceManager {
210210
let boot_timer = Arc::new(Mutex::new(BootTimer::new(request_ts)));
211211

212212
self.mmio_devices
213-
.register_mmio_boot_timer(&vm.common.resource_allocator.mmio_bus, boot_timer)?;
213+
.register_mmio_boot_timer(&vm.common.mmio_bus, boot_timer)?;
214214

215215
Ok(())
216216
}
@@ -249,8 +249,7 @@ impl DeviceManager {
249249
}
250250

251251
let rtc = Arc::new(Mutex::new(RTCDevice::new()));
252-
self.mmio_devices
253-
.register_mmio_rtc(&vm.common.resource_allocator, rtc, None)?;
252+
self.mmio_devices.register_mmio_rtc(vm, rtc, None)?;
254253
Ok(())
255254
}
256255

‎src/vmm/src/device_manager/pci_mngr.rs

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ impl PciDevices {
7575

7676
// Currently we don't assign any IRQs to PCI devices. We will be using MSI-X interrupts
7777
// only.
78-
let pci_segment = PciSegment::new(0, &vm.common.resource_allocator, &[0u8; 32])?;
78+
let pci_segment = PciSegment::new(0, vm, &[0u8; 32])?;
7979
self.pci_segment = Some(pci_segment);
8080

8181
Ok(())
@@ -140,20 +140,15 @@ impl PciDevices {
140140
match bar.region_type() {
141141
PciBarRegionType::IoRegion => {
142142
#[cfg(target_arch = "x86_64")]
143-
resource_allocator.pio_bus.insert(
144-
virtio_device.clone(),
145-
bar.addr(),
146-
bar.size(),
147-
)?;
143+
vm.pio_bus
144+
.insert(virtio_device.clone(), bar.addr(), bar.size())?;
148145
#[cfg(target_arch = "aarch64")]
149146
log::error!("pci: We do not support I/O region allocation")
150147
}
151148
PciBarRegionType::Memory32BitRegion | PciBarRegionType::Memory64BitRegion => {
152-
resource_allocator.mmio_bus.insert(
153-
virtio_device.clone(),
154-
bar.addr(),
155-
bar.size(),
156-
)?;
149+
vm.common
150+
.mmio_bus
151+
.insert(virtio_device.clone(), bar.addr(), bar.size())?;
157152
}
158153
}
159154
}
@@ -223,11 +218,8 @@ impl PciDevices {
223218
bar.size()
224219
);
225220
#[cfg(target_arch = "x86_64")]
226-
vm.common.resource_allocator.pio_bus.insert(
227-
virtio_device.clone(),
228-
bar.addr(),
229-
bar.size(),
230-
)?;
221+
vm.pio_bus
222+
.insert(virtio_device.clone(), bar.addr(), bar.size())?;
231223
#[cfg(target_arch = "aarch64")]
232224
log::error!("pci: We do not support I/O region allocation")
233225
}
@@ -237,11 +229,9 @@ impl PciDevices {
237229
bar.addr(),
238230
bar.size()
239231
);
240-
vm.common.resource_allocator.mmio_bus.insert(
241-
virtio_device.clone(),
242-
bar.addr(),
243-
bar.size(),
244-
)?;
232+
vm.common
233+
.mmio_bus
234+
.insert(virtio_device.clone(), bar.addr(), bar.size())?;
245235
}
246236
}
247237
}

‎src/vmm/src/device_manager/persist.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -389,7 +389,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
389389
if state.type_ == DeviceType::Rtc {
390390
let rtc = Arc::new(Mutex::new(RTCDevice::new()));
391391
dev_manager.register_mmio_rtc(
392-
&constructor_args.vm.common.resource_allocator,
392+
constructor_args.vm,
393393
rtc,
394394
Some(state.device_info),
395395
)?;

‎src/vmm/src/devices/pci/pci_segment.rs

Lines changed: 28 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use uuid::Uuid;
2121
use vm_allocator::AddressAllocator;
2222
use vm_device::{BusDeviceSync, BusError};
2323

24-
use crate::arch::{PCI_MMCONFIG_START, PCI_MMIO_CONFIG_SIZE_PER_SEGMENT};
24+
use crate::arch::{ArchVm as Vm, PCI_MMCONFIG_START, PCI_MMIO_CONFIG_SIZE_PER_SEGMENT};
2525
use crate::vstate::resources::ResourceAllocator;
2626

2727
pub struct PciSegment {
@@ -67,28 +67,21 @@ impl std::fmt::Debug for PciSegment {
6767
}
6868

6969
impl PciSegment {
70-
fn build(
71-
id: u16,
72-
resource_allocator: &Arc<ResourceAllocator>,
73-
pci_irq_slots: &[u8; 32],
74-
) -> Result<PciSegment, BusError> {
70+
fn build(id: u16, vm: &Arc<Vm>, pci_irq_slots: &[u8; 32]) -> Result<PciSegment, BusError> {
7571
let pci_root = PciRoot::new(None);
76-
let pci_bus = Arc::new(Mutex::new(PciBus::new(
77-
pci_root,
78-
resource_allocator.clone(),
79-
)));
72+
let pci_bus = Arc::new(Mutex::new(PciBus::new(pci_root, vm.clone())));
8073

8174
let pci_config_mmio = Arc::new(Mutex::new(PciConfigMmio::new(Arc::clone(&pci_bus))));
8275
let mmio_config_address = PCI_MMCONFIG_START + PCI_MMIO_CONFIG_SIZE_PER_SEGMENT * id as u64;
8376

84-
resource_allocator.mmio_bus.insert(
77+
vm.common.mmio_bus.insert(
8578
Arc::clone(&pci_config_mmio) as Arc<dyn BusDeviceSync>,
8679
mmio_config_address,
8780
PCI_MMIO_CONFIG_SIZE_PER_SEGMENT,
8881
)?;
8982

90-
let mem32_allocator = resource_allocator.mmio32_memory.clone();
91-
let mem64_allocator = resource_allocator.mmio64_memory.clone();
83+
let mem32_allocator = vm.common.resource_allocator.mmio32_memory.clone();
84+
let mem64_allocator = vm.common.resource_allocator.mmio64_memory.clone();
9285

9386
let start_of_mem32_area = mem32_allocator.lock().unwrap().base();
9487
let end_of_mem32_area = mem32_allocator.lock().unwrap().end();
@@ -119,13 +112,15 @@ impl PciSegment {
119112
#[cfg(target_arch = "x86_64")]
120113
pub(crate) fn new(
121114
id: u16,
122-
resource_allocator: &Arc<ResourceAllocator>,
115+
vm: &Arc<Vm>,
123116
pci_irq_slots: &[u8; 32],
124117
) -> Result<PciSegment, BusError> {
125-
let mut segment = Self::build(id, resource_allocator, pci_irq_slots)?;
118+
use crate::Vm;
119+
120+
let mut segment = Self::build(id, vm, pci_irq_slots)?;
126121
let pci_config_io = Arc::new(Mutex::new(PciConfigIo::new(Arc::clone(&segment.pci_bus))));
127122

128-
resource_allocator.pio_bus.insert(
123+
vm.pio_bus.insert(
129124
pci_config_io.clone(),
130125
PCI_CONFIG_IO_PORT,
131126
PCI_CONFIG_IO_PORT_SIZE,
@@ -151,10 +146,10 @@ impl PciSegment {
151146
#[cfg(target_arch = "aarch64")]
152147
pub(crate) fn new(
153148
id: u16,
154-
resource_allocator: &Arc<ResourceAllocator>,
149+
vm: &Arc<Vm>,
155150
pci_irq_slots: &[u8; 32],
156151
) -> Result<PciSegment, BusError> {
157-
let segment = Self::build(id, resource_allocator, pci_irq_slots)?;
152+
let segment = Self::build(id, vm, pci_irq_slots)?;
158153
info!(
159154
"pci: adding PCI segment: id={:#x}, PCI MMIO config address: {:#x}, mem32 area: \
160155
[{:#x}-{:#x}], mem64 area: [{:#x}-{:#x}]",
@@ -468,13 +463,14 @@ mod tests {
468463

469464
use super::*;
470465
use crate::arch;
466+
use crate::builder::tests::default_vmm;
471467
use crate::utils::u64_to_usize;
472468

473469
#[test]
474470
fn test_pci_segment_build() {
475-
let resource_allocator = Arc::new(ResourceAllocator::new().unwrap());
471+
let vmm = default_vmm();
476472
let pci_irq_slots = &[0u8; 32];
477-
let pci_segment = PciSegment::new(0, &resource_allocator, pci_irq_slots).unwrap();
473+
let pci_segment = PciSegment::new(0, &vmm.vm, pci_irq_slots).unwrap();
478474

479475
assert_eq!(pci_segment.id, 0);
480476
assert_eq!(
@@ -503,35 +499,34 @@ mod tests {
503499
#[cfg(target_arch = "x86_64")]
504500
#[test]
505501
fn test_io_bus() {
506-
let resource_allocator = Arc::new(ResourceAllocator::new().unwrap());
502+
let vmm = default_vmm();
507503
let pci_irq_slots = &[0u8; 32];
508-
let pci_segment = PciSegment::new(0, &resource_allocator, pci_irq_slots).unwrap();
504+
let pci_segment = PciSegment::new(0, &vmm.vm, pci_irq_slots).unwrap();
509505

510506
let mut data = [0u8; u64_to_usize(PCI_CONFIG_IO_PORT_SIZE)];
511-
resource_allocator
512-
.pio_bus
513-
.read(PCI_CONFIG_IO_PORT, &mut data)
514-
.unwrap();
507+
vmm.vm.pio_bus.read(PCI_CONFIG_IO_PORT, &mut data).unwrap();
515508

516-
resource_allocator
509+
vmm.vm
517510
.pio_bus
518511
.read(PCI_CONFIG_IO_PORT + PCI_CONFIG_IO_PORT_SIZE, &mut data)
519512
.unwrap_err();
520513
}
521514

522515
#[test]
523516
fn test_mmio_bus() {
524-
let resource_allocator = Arc::new(ResourceAllocator::new().unwrap());
517+
let vmm = default_vmm();
525518
let pci_irq_slots = &[0u8; 32];
526-
let pci_segment = PciSegment::new(0, &resource_allocator, pci_irq_slots).unwrap();
519+
let pci_segment = PciSegment::new(0, &vmm.vm, pci_irq_slots).unwrap();
527520

528521
let mut data = [0u8; u64_to_usize(PCI_MMIO_CONFIG_SIZE_PER_SEGMENT)];
529522

530-
resource_allocator
523+
vmm.vm
524+
.common
531525
.mmio_bus
532526
.read(pci_segment.mmio_config_address, &mut data)
533527
.unwrap();
534-
resource_allocator
528+
vmm.vm
529+
.common
535530
.mmio_bus
536531
.read(
537532
pci_segment.mmio_config_address + PCI_MMIO_CONFIG_SIZE_PER_SEGMENT,
@@ -542,9 +537,9 @@ mod tests {
542537

543538
#[test]
544539
fn test_next_device_bdf() {
545-
let resource_allocator = Arc::new(ResourceAllocator::new().unwrap());
540+
let vmm = default_vmm();
546541
let pci_irq_slots = &[0u8; 32];
547-
let pci_segment = PciSegment::new(0, &resource_allocator, pci_irq_slots).unwrap();
542+
let pci_segment = PciSegment::new(0, &vmm.vm, pci_irq_slots).unwrap();
548543

549544
// Start checking from device id 1, since 0 is allocated to the Root port.
550545
for dev_id in 1..32 {

‎src/vmm/src/lib.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -373,10 +373,9 @@ impl Vmm {
373373
self.vcpus_handles.reserve(vcpu_count);
374374

375375
for mut vcpu in vcpus.drain(..) {
376-
vcpu.set_mmio_bus(self.vm.common.resource_allocator.mmio_bus.clone());
376+
vcpu.set_mmio_bus(self.vm.common.mmio_bus.clone());
377377
#[cfg(target_arch = "x86_64")]
378-
vcpu.kvm_vcpu
379-
.set_pio_bus(self.vm.common.resource_allocator.pio_bus.clone());
378+
vcpu.kvm_vcpu.set_pio_bus(self.vm.pio_bus.clone());
380379

381380
self.vcpus_handles
382381
.push(vcpu.start_threaded(vcpu_seccomp_filter.clone(), barrier.clone())?);

0 commit comments

Comments
 (0)