Support using Box64 for x86 emulation

Add the "emu" command line argument to let users choose the emulator to
be used for running x86 binaries. Add support for installing Box64 in
binfmt_misc. If the argument is not present in the command line, try to
use FEX first and if it can't be set up, try again with Box64.

Supersedes: #140
Co-authored-by: ptitSeb <[email protected]>
Co-authored-by: Alex Arnold <[email protected]>
Signed-off-by: Sergio Lopez <[email protected]>

crates/muvm/src/bin/muvm.rs

@@ -394,6 +394,7 @@ fn main() -> Result<ExitCode> {
         gid: getgid().as_raw(),
         host_display: display,
         merged_rootfs: options.merged_rootfs,
+        emulator: options.emulator,
     };
     let mut muvm_config_file = NamedTempFile::new()
         .context("Failed to create a temporary file to store the muvm guest config")?;

crates/muvm/src/cli_options.rs

@@ -5,6 +5,7 @@ use anyhow::{anyhow, Context};
 use bpaf::{any, construct, long, positional, OptionParser, Parser};
 
 use crate::types::MiB;
+use crate::utils::launch::Emulator;
 
 #[derive(Clone, Debug)]
 pub struct Options {
@@ -19,6 +20,7 @@ pub struct Options {
     pub tty: bool,
     pub privileged: bool,
     pub publish_ports: Vec<String>,
+    pub emulator: Option<Emulator>,
     pub command: PathBuf,
     pub command_args: Vec<String>,
 }
@@ -60,6 +62,15 @@ pub fn options() -> OptionParser<Options> {
             None => Ok((s, None)),
         })
         .many();
+    let emulator = long("emu")
+        .help(
+            "Which emulator to use for running x86_64 binaries.
+             Valid options are \"box\" and \"fex\". If this argument is not
+             present, muvm will try to use FEX, falling back to Box if it
+             can't be found.",
+        )
+        .argument::<Emulator>("EMU")
+        .optional();
     let mem = long("mem")
         .help(
             "The amount of RAM, in MiB, that will be available to this microVM.
@@ -140,6 +151,7 @@ pub fn options() -> OptionParser<Options> {
         tty,
         privileged,
         publish_ports,
+        emulator,
         // positionals
         command,
         command_args,

crates/muvm/src/guest/bin/muvm-guest.rs

@@ -6,6 +6,7 @@ use std::process::Command;
 use std::{cmp, env, fs, thread};
 
 use anyhow::{Context, Result};
+use muvm::guest::box64::setup_box;
 use muvm::guest::fex::setup_fex;
 use muvm::guest::hidpipe::start_hidpipe;
 use muvm::guest::mount::mount_filesystems;
@@ -15,7 +16,7 @@ use muvm::guest::socket::setup_socket_proxy;
 use muvm::guest::user::setup_user;
 use muvm::guest::x11::setup_x11_forwarding;
 use muvm::guest::x11bridge::start_x11bridge;
-use muvm::utils::launch::{GuestConfiguration, PULSE_SOCKET};
+use muvm::utils::launch::{Emulator, GuestConfiguration, PULSE_SOCKET};
 use nix::unistd::{Gid, Uid};
 use rustix::process::{getrlimit, setrlimit, Resource};
 
@@ -77,7 +78,20 @@ fn main() -> Result<()> {
 
     Command::new("/usr/lib/systemd/systemd-udevd").spawn()?;
 
-    setup_fex()?;
+    if let Some(emulator) = options.emulator {
+        match emulator {
+            Emulator::Box => setup_box()?,
+            Emulator::Fex => setup_fex()?,
+        };
+    } else if let Err(err) = setup_fex() {
+        eprintln!("Error setting up FEX in binfmt_misc: {err}");
+        eprintln!("Failed to find or configure FEX, falling back to Box");
+
+        if let Err(err) = setup_box() {
+            eprintln!("Error setting up Box in binfmt_misc: {err}");
+            eprintln!("No emulators were configured, x86 emulation may not work");
+        }
+    }
 
     configure_network()?;
 

crates/muvm/src/guest/box64.rs

@@ -0,0 +1,43 @@
+use std::fs::File;
+use std::io::Write;
+
+use anyhow::{anyhow, Context, Result};
+
+use crate::utils::env::find_in_path;
+
+const BOX32_BINFMT_MISC_RULE: &str = ":BOX32:M:0:\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\\
+                                        x00\\x00\\x00\\x00\\x00\\x02\\x00\\x03\\x00:\\xff\\xff\\\
+                                        xff\\xff\\xff\\xfe\\xfe\\x00\\x00\\x00\\x00\\xff\\xff\\\
+                                        xff\\xff\\xff\\xfe\\xff\\xff\\xff:${BOX64}:POCF";
+const BOX64_BINFMT_MISC_RULE: &str =
+    ":BOX64:M:0:\\x7fELF\\x02\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\\
+     x00\\x3e\\x00:\\xff\\xff\\xff\\xff\\xff\\xfe\\xfe\\x00\\x00\\x00\\x00\\xff\\xff\\xff\\xff\\\
+     xff\\xfe\\xff\\xff\\xff:${BOX64}:POCF";
+
+pub fn setup_box() -> Result<()> {
+    let box64_path = find_in_path("box64").context("Failed to check existence of `box64`")?;
+    let Some(box64_path) = box64_path else {
+        return Err(anyhow!("Failed to find `box64` in PATH"));
+    };
+    let box64_path = box64_path
+        .to_str()
+        .context("Failed to process `box64` path as it contains invalid UTF-8")?;
+
+    let mut file = File::options()
+        .write(true)
+        .open("/proc/sys/fs/binfmt_misc/register")
+        .context("Failed to open binfmt_misc/register for writing")?;
+
+    {
+        let rule = BOX32_BINFMT_MISC_RULE.replace("${BOX64}", box64_path);
+        file.write_all(rule.as_bytes())
+            .context("Failed to register `Box32` binfmt_misc rule")?;
+    }
+    {
+        let rule = BOX64_BINFMT_MISC_RULE.replace("${BOX64}", box64_path);
+        file.write_all(rule.as_bytes())
+            .context("Failed to register `Box64` binfmt_misc rule")?;
+    }
+
+    Ok(())
+}

crates/muvm/src/guest/fex.rs

@@ -1,7 +1,7 @@
 use std::fs::File;
 use std::io::Write;
 
-use anyhow::{Context, Result};
+use anyhow::{anyhow, Context, Result};
 
 use crate::utils::env::find_in_path;
 
@@ -18,7 +18,7 @@ pub fn setup_fex() -> Result<()> {
     let fex_interpreter_path =
         find_in_path("FEXInterpreter").context("Failed to check existence of `FEXInterpreter`")?;
     let Some(fex_interpreter_path) = fex_interpreter_path else {
-        return Ok(());
+        return Err(anyhow!("Failed to find `FEXInterpreter` in PATH"));
     };
     let fex_interpreter_path = fex_interpreter_path
         .to_str()

crates/muvm/src/guest/mod.rs

@@ -1,3 +1,4 @@
+pub mod box64;
 pub mod fex;
 pub mod hidpipe;
 pub mod mount;

crates/muvm/src/utils/launch.rs

@@ -1,6 +1,8 @@
+use anyhow::anyhow;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::path::PathBuf;
+use std::str::FromStr;
 
 #[derive(Clone, Eq, PartialEq, Debug, Deserialize, Serialize)]
 pub struct Launch {
@@ -12,6 +14,27 @@ pub struct Launch {
     pub privileged: bool,
 }
 
+#[derive(Clone, Eq, PartialEq, Debug, Deserialize, Serialize)]
+pub enum Emulator {
+    Box,
+    Fex,
+}
+
+impl FromStr for Emulator {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let v = s.to_lowercase();
+        if v.starts_with("box") {
+            Ok(Emulator::Box)
+        } else if v.starts_with("fex") {
+            Ok(Emulator::Fex)
+        } else {
+            Err(anyhow!("Invalid or unsupported emulator"))
+        }
+    }
+}
+
 #[derive(Clone, Eq, PartialEq, Debug, Deserialize, Serialize)]
 pub struct GuestConfiguration {
     pub command: Launch,
@@ -20,6 +43,7 @@ pub struct GuestConfiguration {
     pub gid: u32,
     pub host_display: Option<String>,
     pub merged_rootfs: bool,
+    pub emulator: Option<Emulator>,
 }
 
 pub const PULSE_SOCKET: u32 = 3333;