Skip to content

Commit 683b3a5

Browse files
committed
Configure KVM vCPUS and resume new threads after creation
Properly configure KVM vCPUs so that the configuration is normalized and matches that of already existing KVM vCPUs. Signed-off-by: James Curtis <[email protected]>
1 parent 8dbce31 commit 683b3a5

File tree

5 files changed

+108
-32
lines changed

5 files changed

+108
-32
lines changed

src/vmm/src/builder.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,8 @@ fn create_vmm_and_vcpus(
271271
vcpus_handles: Vec::new(),
272272
vcpus_exit_evt,
273273
#[cfg(target_arch = "x86_64")]
274+
vcpu_config: None,
275+
#[cfg(target_arch = "x86_64")]
274276
seccomp_filters,
275277
resource_allocator,
276278
mmio_device_manager,
@@ -883,6 +885,9 @@ pub fn configure_system_for_boot(
883885
cpu_config,
884886
};
885887

888+
#[cfg(target_arch = "x86_64")]
889+
vmm.attach_vcpu_config(vcpu_config.clone());
890+
886891
// Configure vCPUs with normalizing and setting the generated CPU configuration.
887892
for vcpu in vcpus.iter_mut() {
888893
vcpu.kvm_vcpu
@@ -1279,6 +1284,8 @@ pub mod tests {
12791284
vcpus_handles: Vec::new(),
12801285
vcpus_exit_evt,
12811286
#[cfg(target_arch = "x86_64")]
1287+
vcpu_config: None,
1288+
#[cfg(target_arch = "x86_64")]
12821289
seccomp_filters: crate::seccomp_filters::get_empty_filters(),
12831290
resource_allocator,
12841291
mmio_device_manager,

src/vmm/src/lib.rs

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,9 @@ pub struct Vmm {
323323
vcpus_handles: Vec<VcpuHandle>,
324324
// Used by Vcpus and devices to initiate teardown; Vmm should never write here.
325325
vcpus_exit_evt: EventFd,
326+
// Used to configure kvm vcpus during hotplugging.
327+
#[cfg(target_arch = "x86_64")]
328+
vcpu_config: Option<VcpuConfig>,
326329
// seccomp_filters are only needed in VMM for hotplugging vCPUS.
327330
#[cfg(target_arch = "x86_64")]
328331
seccomp_filters: BpfThreadMap,
@@ -415,23 +418,32 @@ impl Vmm {
415418
pub fn resume_vm(&mut self) -> Result<(), VmmError> {
416419
self.mmio_device_manager.kick_devices();
417420

418-
// Send the events.
419-
self.vcpus_handles
421+
self.resume_vcpu_threads(0)?;
422+
423+
self.instance_info.state = VmState::Running;
424+
Ok(())
425+
}
426+
427+
/// Resume vCPU threads
428+
fn resume_vcpu_threads(&mut self, start_idx: usize) -> Result<(), VmmError> {
429+
if start_idx >= self.vcpus_handles.len() {
430+
return Err(VmmError::VcpuMessage);
431+
}
432+
433+
self.vcpus_handles[start_idx..]
420434
.iter()
421435
.try_for_each(|handle| handle.send_event(VcpuEvent::Resume))
422436
.map_err(|_| VmmError::VcpuMessage)?;
423437

424438
// Check the responses.
425-
if self
426-
.vcpus_handles
439+
if self.vcpus_handles[start_idx..]
427440
.iter()
428441
.map(|handle| handle.response_receiver().recv_timeout(RECV_TIMEOUT_SEC))
429442
.any(|response| !matches!(response, Ok(VcpuResponse::Resumed)))
430443
{
431444
return Err(VmmError::VcpuMessage);
432445
}
433446

434-
self.instance_info.state = VmState::Running;
435447
Ok(())
436448
}
437449

@@ -621,6 +633,9 @@ impl Vmm {
621633
return Err(HotplugVcpuError::VcpuCountTooHigh);
622634
}
623635

636+
if let Some(kvm_config) = self.vcpu_config.as_mut() {
637+
kvm_config.vcpu_count += config.add;
638+
}
624639
// Create and start new vcpus
625640
let mut vcpus = Vec::with_capacity(config.add.into());
626641

@@ -635,8 +650,13 @@ impl Vmm {
635650
.vcpus_exit_evt
636651
.try_clone()
637652
.map_err(HotplugVcpuError::EventFd)?;
638-
let vcpu =
653+
let mut vcpu =
639654
Vcpu::new(cpu_idx, &self.vm, exit_evt).map_err(HotplugVcpuError::VcpuCreate)?;
655+
if let Some(kvm_config) = self.vcpu_config.as_ref() {
656+
vcpu.kvm_vcpu.hotplug_configure(kvm_config)?;
657+
} else {
658+
return Err(HotplugVcpuError::RestoredFromSnapshot);
659+
}
640660
locked_container.cpu_devices[cpu_idx as usize].inserting = true;
641661
vcpus.push(vcpu);
642662
}
@@ -665,6 +685,8 @@ impl Vmm {
665685
huge_pages: None,
666686
};
667687

688+
self.resume_vcpu_threads(start_idx.into())?;
689+
668690
self.acpi_device_manager.notify_cpu_container()?;
669691

670692
Ok(new_machine_config)
@@ -877,6 +899,13 @@ impl Vmm {
877899
}
878900
}
879901

902+
/// Add the vcpu configuration used during boot to the VMM. This is required as part of the
903+
/// hotplugging process, to correctly configure KVM vCPUs.
904+
#[cfg(target_arch = "x86_64")]
905+
pub fn attach_vcpu_config(&mut self, vcpu_config: VcpuConfig) {
906+
self.vcpu_config = Some(vcpu_config)
907+
}
908+
880909
/// Signals Vmm to stop and exit.
881910
pub fn stop(&mut self, exit_code: FcExitCode) {
882911
// To avoid cycles, all teardown paths take the following route:

src/vmm/src/vmm_config/hotplug.rs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ use std::io;
55

66
use serde::{Deserialize, Serialize};
77

8-
use crate::vstate::vcpu::VcpuError;
9-
use crate::StartVcpusError;
8+
use crate::vstate::vcpu::{KvmVcpuConfigureError, VcpuError};
9+
use crate::{StartVcpusError, VmmError};
1010
/// Unifying enum for all types of hotplug request configs.
1111
/// Currently only Vcpus may be hotplugged.
1212
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -34,10 +34,14 @@ pub enum HotplugVcpuError {
3434
EventFd(#[from] io::Error),
3535
/// Error creating the vcpu: {0}
3636
VcpuCreate(VcpuError),
37+
/// Error configuring KVM vcpu: {0}
38+
VcpuConfigure(#[from] KvmVcpuConfigureError),
3739
/// Failed to start vCPUs
3840
VcpuStart(StartVcpusError),
3941
/// No seccomp filter for thread category: {0}
4042
MissingSeccompFilters(String),
43+
/// Error resuming VM: {0}
44+
VmResume(#[from] VmmError),
4145
/// Cannot hotplug vCPUs after restoring from snapshot
4246
RestoredFromSnapshot,
4347
}

src/vmm/src/vstate/vcpu/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ pub enum VcpuError {
6363
}
6464

6565
/// Encapsulates configuration parameters for the guest vCPUS.
66-
#[derive(Debug)]
66+
#[derive(Debug, Clone)]
6767
pub struct VcpuConfig {
6868
/// Number of guest VCPUs.
6969
pub vcpu_count: u8,

src/vmm/src/vstate/vcpu/x86_64.rs

Lines changed: 59 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ use std::collections::{HashMap, HashSet};
99
use std::fmt::Debug;
1010

1111
use kvm_bindings::{
12-
kvm_debugregs, kvm_lapic_state, kvm_mp_state, kvm_regs, kvm_sregs, kvm_vcpu_events, kvm_xcrs,
13-
kvm_xsave, CpuId, Msrs, KVM_MAX_CPUID_ENTRIES, KVM_MAX_MSR_ENTRIES,
12+
kvm_debugregs, kvm_lapic_state, kvm_mp_state, kvm_msr_entry, kvm_regs, kvm_sregs,
13+
kvm_vcpu_events, kvm_xcrs, kvm_xsave, CpuId, Msrs, KVM_MAX_CPUID_ENTRIES, KVM_MAX_MSR_ENTRIES,
1414
};
1515
use kvm_ioctls::{VcpuExit, VcpuFd};
1616
use log::{error, warn};
@@ -165,22 +165,17 @@ impl KvmVcpu {
165165
})
166166
}
167167

168-
/// Configures a x86_64 specific vcpu for booting Linux and should be called once per vcpu.
169-
///
168+
/// General configuration - common for both boot and hotplugged CPUs.
169+
/// Normalizes and sets the CPUID in KVM and creates KVM MSRs
170170
/// # Arguments
171-
///
172-
/// * `guest_mem` - The guest memory used by this microvm.
173-
/// * `kernel_start_addr` - Offset from `guest_mem` at which the kernel starts.
174-
/// * `vcpu_config` - The vCPU configuration.
175-
/// * `cpuid` - The capabilities exposed by this vCPU.
176-
pub fn configure(
171+
/// * vcpu_config - The configuration for the vCPUs.
172+
/// * msrs - The MSRs currently present.
173+
fn configure_common(
177174
&mut self,
178-
guest_mem: &GuestMemoryMmap,
179-
kernel_start_addr: GuestAddress,
180175
vcpu_config: &VcpuConfig,
181-
) -> Result<(), KvmVcpuConfigureError> {
176+
msrs: HashMap<u32, u64>,
177+
) -> Result<(Vec<kvm_msr_entry>, CpuId), KvmVcpuConfigureError> {
182178
let mut cpuid = vcpu_config.cpu_config.cpuid.clone();
183-
184179
// Apply machine specific changes to CPUID.
185180
cpuid.normalize(
186181
// The index of the current logical CPU in the range [0..cpu_count].
@@ -199,6 +194,35 @@ impl KvmVcpu {
199194
.set_cpuid2(&kvm_cpuid)
200195
.map_err(KvmVcpuConfigureError::SetCpuid)?;
201196

197+
// // Clone MSR entries that are modified by CPU template from `VcpuConfig`.
198+
199+
let kvm_msrs = msrs
200+
.clone()
201+
.into_iter()
202+
.map(|entry| kvm_bindings::kvm_msr_entry {
203+
index: entry.0,
204+
data: entry.1,
205+
..Default::default()
206+
})
207+
.collect::<Vec<_>>();
208+
209+
Ok((kvm_msrs, kvm_cpuid))
210+
}
211+
212+
/// Configures a x86_64 specific vcpu for booting Linux and should be called once per vcpu.
213+
///
214+
/// # Arguments
215+
///
216+
/// * `guest_mem` - The guest memory used by this microvm.
217+
/// * `kernel_start_addr` - Offset from `guest_mem` at which the kernel starts.
218+
/// * `vcpu_config` - The vCPU configuration.
219+
/// * `cpuid` - The capabilities exposed by this vCPU.
220+
pub fn configure(
221+
&mut self,
222+
guest_mem: &GuestMemoryMmap,
223+
kernel_start_addr: GuestAddress,
224+
vcpu_config: &VcpuConfig,
225+
) -> Result<(), KvmVcpuConfigureError> {
202226
// Clone MSR entries that are modified by CPU template from `VcpuConfig`.
203227
let mut msrs = vcpu_config.cpu_config.msrs.clone();
204228
self.msrs_to_save.extend(msrs.keys());
@@ -208,6 +232,8 @@ impl KvmVcpu {
208232
msrs.insert(entry.index, entry.data);
209233
});
210234

235+
let (kvm_msrs, kvm_cpuid) = self.configure_common(vcpu_config, msrs)?;
236+
211237
// TODO - Add/amend MSRs for vCPUs based on cpu_config
212238
// By this point the Guest CPUID is established. Some CPU features require MSRs
213239
// to configure and interact with those features. If a MSR is writable from
@@ -226,15 +252,6 @@ impl KvmVcpu {
226252
// save is `architectural MSRs` + `MSRs inferred through CPUID` + `other
227253
// MSRs defined by the template`
228254

229-
let kvm_msrs = msrs
230-
.into_iter()
231-
.map(|entry| kvm_bindings::kvm_msr_entry {
232-
index: entry.0,
233-
data: entry.1,
234-
..Default::default()
235-
})
236-
.collect::<Vec<_>>();
237-
238255
crate::arch::x86_64::msr::set_msrs(&self.fd, &kvm_msrs)?;
239256
crate::arch::x86_64::regs::setup_regs(&self.fd, kernel_start_addr.raw_value())?;
240257
crate::arch::x86_64::regs::setup_fpu(&self.fd)?;
@@ -244,6 +261,25 @@ impl KvmVcpu {
244261
Ok(())
245262
}
246263

264+
/// Configures an x86_64 cpu that has been hotplugged post-boot. Called once per hotplugged
265+
/// vcpu.
266+
///
267+
/// # Arguments
268+
///
269+
/// * `vcpu_config` - The config to be applied to the vCPU, defined at boot time.
270+
pub fn hotplug_configure(
271+
&mut self,
272+
vcpu_config: &VcpuConfig,
273+
) -> Result<(), KvmVcpuConfigureError> {
274+
let msrs = vcpu_config.cpu_config.msrs.clone();
275+
let (kvm_msrs, _) = self.configure_common(vcpu_config, msrs)?;
276+
277+
crate::arch::x86_64::msr::set_msrs(&self.fd, &kvm_msrs)?;
278+
crate::arch::x86_64::interrupts::set_lint(&self.fd)?;
279+
280+
Ok(())
281+
}
282+
247283
/// Sets a Port Mapped IO bus for this vcpu.
248284
pub fn set_pio_bus(&mut self, pio_bus: crate::devices::Bus) {
249285
self.peripherals.pio_bus = Some(pio_bus);

0 commit comments

Comments
 (0)