Implement infrastructure to manage interrupts by KVM

Implement infrastructure to manage interrupt sources based on Linux KVM
kernel module.

Signed-off-by: Liu Jiang <[email protected]>
Signed-off-by: Bin Zha <[email protected]>

Author: jiangliu

Cargo.toml

@@ -8,8 +8,11 @@ edition = "2018"
 
 [dependencies]
 libc = ">=0.2.39"
+kvm-bindings = { version = "~0", optional = true }
+kvm-ioctls = { version = "~0", optional = true }
 vmm-sys-util = "~0"
 
 [features]
 legacy-irq = []
 msi-irq = []
+kvm-irq = ["kvm-ioctls", "kvm-bindings"]

src/interrupt/kvm/mod.rs

@@ -0,0 +1,169 @@
+// Copyright (C) 2019 Alibaba Cloud. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Manage virtual device's interrupts based on the Linux KVM framework.
+//!
+//! When updaing KVM IRQ routing by ioctl(KVM_SET_GSI_ROUTING), all interrupts of the virtual
+//! machine must be updated all together. The [KvmIrqRouting](struct.KvmIrqRouting.html)
+//! structure is to maintain the global interrupt routing table.
+
+use std::collections::HashMap;
+use std::io::{Error, ErrorKind};
+use std::sync::{Arc, Mutex};
+
+use kvm_bindings::{kvm_irq_routing, kvm_irq_routing_entry};
+use kvm_ioctls::VmFd;
+
+use super::*;
+
+/// Structure to manage interrupt sources for a virtual machine based on the Linux KVM framework.
+///
+/// The KVM framework provides methods to inject interrupts into the target virtual machines,
+/// which uses irqfd to notity the KVM kernel module for injecting interrupts. When the interrupt
+/// source, usually a virtual device backend in userspace, writes to the irqfd file descriptor,
+/// the KVM kernel module will inject a corresponding interrupt into the target VM according to
+/// the IRQ routing configuration.
+pub struct KvmIrqManager {
+    mgr: Mutex<KvmIrqManagerObj>,
+}
+
+impl KvmIrqManager {
+    /// Create a new interrupt manager based on the Linux KVM framework.
+    ///
+    /// # Arguments
+    /// * `vmfd`: The KVM VM file descriptor, which will be used to access the KVM subsystem.
+    pub fn new(vmfd: Arc<VmFd>) -> Self {
+        let vmfd2 = vmfd.clone();
+        KvmIrqManager {
+            mgr: Mutex::new(KvmIrqManagerObj {
+                vmfd,
+                groups: HashMap::new(),
+                routes: Arc::new(KvmIrqRouting::new(vmfd2)),
+            }),
+        }
+    }
+
+    /// Prepare the interrupt manager for generating interrupts into the target VM.
+    pub fn initialize(&self) -> Result<()> {
+        // Safe to unwrap because there's no legal way to break the mutex.
+        let mgr = self.mgr.lock().unwrap();
+        mgr.initialize()
+    }
+}
+
+impl InterruptManager for KvmIrqManager {
+    fn create_group(
+        &self,
+        ty: InterruptSourceType,
+        base: InterruptIndex,
+        count: u32,
+    ) -> Result<Arc<Box<dyn InterruptSourceGroup>>> {
+        // Safe to unwrap because there's no legal way to break the mutex.
+        let mut mgr = self.mgr.lock().unwrap();
+        mgr.create_group(ty, base, count)
+    }
+
+    fn destroy_group(&self, group: Arc<Box<dyn InterruptSourceGroup>>) -> Result<()> {
+        // Safe to unwrap because there's no legal way to break the mutex.
+        let mut mgr = self.mgr.lock().unwrap();
+        mgr.destroy_group(group)
+    }
+}
+
+struct KvmIrqManagerObj {
+    vmfd: Arc<VmFd>,
+    routes: Arc<KvmIrqRouting>,
+    groups: HashMap<InterruptIndex, Arc<Box<dyn InterruptSourceGroup>>>,
+}
+
+impl KvmIrqManagerObj {
+    fn initialize(&self) -> Result<()> {
+        self.routes.initialize()?;
+        Ok(())
+    }
+
+    fn create_group(
+        &mut self,
+        ty: InterruptSourceType,
+        base: InterruptIndex,
+        count: u32,
+    ) -> Result<Arc<Box<dyn InterruptSourceGroup>>> {
+        #[allow(unreachable_patterns)]
+        let group: Arc<Box<dyn InterruptSourceGroup>> = match ty {
+            _ => return Err(Error::from(ErrorKind::InvalidInput)),
+        };
+
+        self.groups.insert(base, group.clone());
+
+        Ok(group)
+    }
+
+    fn destroy_group(&mut self, group: Arc<Box<dyn InterruptSourceGroup>>) -> Result<()> {
+        self.groups.remove(&group.base());
+        Ok(())
+    }
+}
+
+// Use (entry.type, entry.gsi) as the hash key because entry.gsi can't uniquely identify an
+// interrupt source on x86 platforms. The PIC and IOAPIC may share the same GSI on x86 platforms.
+fn hash_key(entry: &kvm_irq_routing_entry) -> u64 {
+    let type1 = match entry.type_ {
+        #[cfg(feature = "kvm-legacy-irq")]
+        kvm_bindings::KVM_IRQ_ROUTING_IRQCHIP => unsafe { entry.u.irqchip.irqchip },
+        _ => 0u32,
+    };
+    (u64::from(type1) << 48 | u64::from(entry.type_) << 32) | u64::from(entry.gsi)
+}
+
+pub(super) struct KvmIrqRouting {
+    vm_fd: Arc<VmFd>,
+    routes: Mutex<HashMap<u64, kvm_irq_routing_entry>>,
+}
+
+impl KvmIrqRouting {
+    pub(super) fn new(vm_fd: Arc<VmFd>) -> Self {
+        KvmIrqRouting {
+            vm_fd,
+            routes: Mutex::new(HashMap::new()),
+        }
+    }
+
+    pub(super) fn initialize(&self) -> Result<()> {
+        // Safe to unwrap because there's no legal way to break the mutex.
+        #[allow(unused_mut)]
+        let mut routes = self.routes.lock().unwrap();
+
+        self.set_routing(&*routes)
+    }
+
+    fn set_routing(&self, routes: &HashMap<u64, kvm_irq_routing_entry>) -> Result<()> {
+        // Allocate enough buffer memory.
+        let elem_sz = std::mem::size_of::<kvm_irq_routing>();
+        let total_sz = std::mem::size_of::<kvm_irq_routing_entry>() * routes.len() + elem_sz;
+        let elem_cnt = (total_sz + elem_sz - 1) / elem_sz;
+        let mut irq_routings = Vec::<kvm_irq_routing>::with_capacity(elem_cnt);
+        irq_routings.resize_with(elem_cnt, Default::default);
+
+        // Prepare the irq_routing header.
+        let mut irq_routing = &mut irq_routings[0];
+        irq_routing.nr = routes.len() as u32;
+        irq_routing.flags = 0;
+
+        // Safe because we have just allocated enough memory above.
+        let irq_routing_entries = unsafe { irq_routing.entries.as_mut_slice(routes.len()) };
+        for (idx, entry) in routes.values().enumerate() {
+            irq_routing_entries[idx] = *entry;
+        }
+
+        self.vm_fd
+            .set_gsi_routing(irq_routing)
+            .map_err(from_sys_util_errno)?;
+
+        Ok(())
+    }
+}
+
+/// Helper function convert from vmm_sys_util::errno::Error to std::io::Error.
+pub fn from_sys_util_errno(e: vmm_sys_util::errno::Error) -> std::io::Error {
+    std::io::Error::from_raw_os_error(e.errno())
+}

src/interrupt/mod.rs

@@ -203,3 +203,8 @@ pub trait InterruptSourceGroup: Send + Sync {
         Ok(())
     }
 }
+
+#[cfg(feature = "kvm-irq")]
+mod kvm;
+#[cfg(feature = "kvm-irq")]
+pub use self::kvm::KvmIrqManager;