Consider all other exit codes besides 'test-success-exit-code' as failures

.github/workflows/build.yml

@@ -116,6 +116,10 @@ jobs:
       working-directory: example-kernels/runner-doctest
       name: 'Run `cargo test -Z doctest-xcompile` for "runner-doctest" kernel'
 
+    - run: cargo test
+      working-directory: example-kernels/runner-fail-reboot
+      name: 'Run `cargo test` for "runner-fail-reboot" kernel'
+
   check_formatting:
     name: "Check Formatting"
     runs-on: ubuntu-latest

Readme.md

@@ -82,6 +82,9 @@ test-success-exit-code = {integer}
 
 # The timeout for running a test through `bootimage test` or `bootimage runner` (in seconds)
 test-timeout = 300
+
+# Whether the `-no-reboot` flag should be passed to test executables
+test-no-reboot = true
 ```
 
 ## License

example-kernels/Cargo.toml

@@ -3,5 +3,6 @@ members = [
     "basic",
     "runner",
     "runner-doctest",
+    "runner-fail-reboot",
     "runner-test",
 ]

example-kernels/runner-fail-reboot/.cargo/config

@@ -0,0 +1,5 @@
+[build]
+target = "../x86_64-bootimage-example-kernels.json"
+
+[target.'cfg(target_os = "none")']
+runner = "bootimage runner"

example-kernels/runner-fail-reboot/.gitignore

@@ -0,0 +1,2 @@
+/target/
+**/*.rs.bk

example-kernels/runner-fail-reboot/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "runner-fail-reboot"
+version = "0.1.0"
+authors = ["Philipp Oppermann <[email protected]>"]
+edition = "2018"
+
+[dependencies]
+bootloader = "0.9.7"
+x86_64 = "0.11.0"
+rlibc = "1.0.0"
+
+[package.metadata.bootimage]
+test-success-exit-code = 0 # this will test for the reboot
+test-args = ["-device", "isa-debug-exit,iobase=0xf4,iosize=0x04", "-display", "none"]

example-kernels/runner-fail-reboot/src/lib.rs

@@ -0,0 +1,71 @@
+#![no_std]
+#![cfg_attr(test, no_main)]
+#![feature(custom_test_frameworks)]
+#![test_runner(crate::test_runner)]
+#![reexport_test_harness_main = "test_main"]
+
+extern crate rlibc;
+
+pub fn test_runner(tests: &[&dyn Fn()]) {
+    for test in tests.iter() {
+        test();
+    }
+
+    unsafe {
+        exit_qemu(ExitCode::Success);
+    }
+}
+
+#[test_case]
+fn should_reboot() {
+    // this overflows the stack which leads to a triple fault
+    // the as-if rule might allow this to get optimized away on release builds
+    #[allow(unconditional_recursion)]
+    fn stack_overflow() {
+        stack_overflow()
+    }
+    stack_overflow()
+}
+
+pub enum ExitCode {
+    Success,
+    Failed,
+}
+
+impl ExitCode {
+    fn code(&self) -> u32 {
+        match self {
+            ExitCode::Success => 0x10,
+            ExitCode::Failed => 0x11,
+        }
+    }
+}
+
+/// exit QEMU (see https://os.phil-opp.com/integration-tests/#shutting-down-qemu)
+pub unsafe fn exit_qemu(exit_code: ExitCode) {
+    use x86_64::instructions::port::Port;
+
+    let mut port = Port::<u32>::new(0xf4);
+    port.write(exit_code.code());
+}
+
+#[cfg(test)]
+#[no_mangle]
+pub extern "C" fn _start() -> ! {
+    test_main();
+
+    unsafe {
+        exit_qemu(ExitCode::Failed);
+    }
+
+    loop {}
+}
+
+#[cfg(test)]
+#[panic_handler]
+fn panic(_info: &core::panic::PanicInfo) -> ! {
+    unsafe {
+        exit_qemu(ExitCode::Failed);
+    }
+    loop {}
+}

src/config.rs

@@ -33,6 +33,10 @@ pub struct Config {
     /// An exit code that should be considered as success for test executables (applies to
     /// `bootimage runner`)
     pub test_success_exit_code: Option<i32>,
+    /// Whether the `-no-reboot` flag should be passed to test executables
+    ///
+    /// Defaults to `true`
+    pub test_no_reboot: bool,
 }
 
 /// Reads the configuration from a `package.metadata.bootimage` in the given Cargo.toml.
@@ -91,6 +95,9 @@ fn read_config_inner(manifest_path: &Path) -> Result<Config> {
             ("test-args", Value::Array(array)) => {
                 config.test_args = Some(parse_string_array(array, "test-args")?);
             }
+            ("test-no-reboot", Value::Boolean(no_reboot)) => {
+                config.test_no_reboot = Some(no_reboot);
+            }
             (key, value) => {
                 return Err(anyhow!(
                     "unexpected `package.metadata.bootimage` \
@@ -123,6 +130,7 @@ struct ConfigBuilder {
     test_args: Option<Vec<String>>,
     test_timeout: Option<u32>,
     test_success_exit_code: Option<i32>,
+    test_no_reboot: Option<bool>,
 }
 
 impl Into<Config> for ConfigBuilder {
@@ -140,6 +148,7 @@ impl Into<Config> for ConfigBuilder {
             test_args: self.test_args,
             test_timeout: self.test_timeout.unwrap_or(60 * 5),
             test_success_exit_code: self.test_success_exit_code,
+            test_no_reboot: self.test_no_reboot.unwrap_or(true),
         }
     }
 }

src/run.rs

@@ -23,6 +23,9 @@ pub fn run(
         .map(|arg| arg.replace("{}", &format!("{}", image_path.display())))
         .collect();
     if is_test {
+        if config.test_no_reboot {
+            run_command.push("-no-reboot".to_owned());
+        }
         if let Some(args) = config.test_args {
             run_command.extend(args);
         }
@@ -69,6 +72,7 @@ pub fn run(
                 let qemu_exit_code = exit_status.code().ok_or(RunError::NoQemuExitCode)?;
                 match config.test_success_exit_code {
                     Some(code) if qemu_exit_code == code => 0,
+                    Some(_) if qemu_exit_code == 0 => 1,
                     _ => qemu_exit_code,
                 }
             }