diff --git a/Makefile b/Makefile index 4e629165e..d23229475 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,11 @@ bin/kind: test/k8s/Dockerfile test/k8s/_out/img: test/k8s/Dockerfile Cargo.toml Cargo.lock $(shell find . -type f -name '*.rs') mkdir -p $(@D) && $(DOCKER_BUILD) -f test/k8s/Dockerfile --iidfile=$(@) --load . +.PHONY: test/nginx +test/nginx: + docker pull docker.io/nginx:latest + mkdir -p $@/out && docker save -o $@/out/img.tar docker.io/nginx:latest + .PHONY: test/k8s/cluster test/k8s/cluster: target/wasm32-wasi/$(TARGET)/img.tar bin/kind test/k8s/_out/img bin/kind bin/kind create cluster --name $(KIND_CLUSTER_NAME) --image="$(shell cat test/k8s/_out/img)" && \ diff --git a/crates/containerd-shim-wasm/src/lib.rs b/crates/containerd-shim-wasm/src/lib.rs index b4a324107..86973ce8c 100644 --- a/crates/containerd-shim-wasm/src/lib.rs +++ b/crates/containerd-shim-wasm/src/lib.rs @@ -6,3 +6,6 @@ pub mod sandbox; pub mod services; + +#[cfg(feature = "libcontainer")] +pub mod libcontainer_instance; diff --git a/crates/containerd-shim-wasm/src/libcontainer_instance/container_executor.rs b/crates/containerd-shim-wasm/src/libcontainer_instance/container_executor.rs new file mode 100644 index 000000000..4da20c797 --- /dev/null +++ b/crates/containerd-shim-wasm/src/libcontainer_instance/container_executor.rs @@ -0,0 +1,133 @@ +use crate::sandbox::oci; +use libcontainer::workload::default::DefaultExecutor; +use libcontainer::workload::{Executor, ExecutorError}; +use nix::unistd::{dup, dup2}; + +use libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; +use oci_spec::runtime::Spec; +use std::io::Read; +use std::{fs::OpenOptions, os::fd::RawFd, path::PathBuf}; + +#[derive(Default)] +pub struct LinuxContainerExecutor { + stdin: Option, + stdout: Option, + stderr: Option, + default_executor: DefaultExecutor, +} + +impl LinuxContainerExecutor { + pub fn new(stdin: Option, stdout: Option, stderr: Option) -> Self { + Self { + stdin, + stdout, + stderr, + ..Default::default() + } + } +} + +impl Executor for LinuxContainerExecutor { + fn exec(&self, spec: &Spec) -> Result<(), ExecutorError> { + redirect_io(self.stdin, self.stdout, self.stderr).map_err(|err| { + log::error!("failed to redirect io: {}", err); + ExecutorError::Other(format!("failed to redirect io: {}", err)) + })?; + self.default_executor.exec(spec) + } + + fn can_handle(&self, spec: &Spec) -> bool { + let args = oci::get_args(spec); + + if args.is_empty() { + return false; + } + + let executable = args[0].as_str(); + + // mostly follows youki's verify_binary implementation + // https://github.com/containers/youki/blob/2d6fd7650bb0f22a78fb5fa982b5628f61fe25af/crates/libcontainer/src/process/container_init_process.rs#L106 + let path = if executable.contains('/') { + PathBuf::from(executable) + } else { + let path = std::env::var("PATH").unwrap_or_default(); + // check each path in $PATH + let mut found = false; + let mut found_path = PathBuf::default(); + for p in path.split(':') { + let path = PathBuf::from(p).join(executable); + if path.exists() { + found = true; + found_path = path; + break; + } + } + if !found { + return false; + } + found_path + }; + + // check execute permission + use std::os::unix::fs::PermissionsExt; + let metadata = path.metadata(); + if metadata.is_err() { + log::info!("failed to get metadata of {:?}", path); + return false; + } + let metadata = metadata.unwrap(); + let permissions = metadata.permissions(); + if !metadata.is_file() || permissions.mode() & 0o001 == 0 { + log::info!("{} is not a file or has no execute permission", executable); + return false; + } + + // check the shebang and ELF magic number + // https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header + let mut buffer = [0; 4]; + + let file = OpenOptions::new().read(true).open(path); + if file.is_err() { + log::info!("failed to open {}", executable); + return false; + } + let mut file = file.unwrap(); + match file.read_exact(&mut buffer) { + Ok(_) => {} + Err(err) => { + log::info!("failed to read shebang of {}: {}", executable, err); + return false; + } + } + match buffer { + // ELF magic number + [0x7f, 0x45, 0x4c, 0x46] => true, + // shebang + [0x23, 0x21, ..] => true, + _ => { + log::info!("{} is not a valid script or elf file", executable); + false + } + } + } + + fn name(&self) -> &'static str { + self.default_executor.name() + } +} + +fn redirect_io(stdin: Option, stdout: Option, stderr: Option) -> anyhow::Result<()> { + if let Some(stdin) = stdin { + dup(STDIN_FILENO)?; + dup2(stdin, STDIN_FILENO)?; + } + if let Some(stdout) = stdout { + dup(STDOUT_FILENO)?; + dup2(stdout, STDOUT_FILENO)?; + } + if let Some(stderr) = stderr { + dup(STDERR_FILENO)?; + dup2(stderr, STDERR_FILENO)?; + } + Ok(()) +} diff --git a/crates/containerd-shim-wasm/src/sandbox/libcontainer_instance.rs b/crates/containerd-shim-wasm/src/libcontainer_instance/instance.rs similarity index 99% rename from crates/containerd-shim-wasm/src/sandbox/libcontainer_instance.rs rename to crates/containerd-shim-wasm/src/libcontainer_instance/instance.rs index 28dc99cba..3b2125590 100644 --- a/crates/containerd-shim-wasm/src/sandbox/libcontainer_instance.rs +++ b/crates/containerd-shim-wasm/src/libcontainer_instance/instance.rs @@ -21,7 +21,7 @@ use crate::sandbox::{ use crate::sandbox::InstanceConfig; use std::{path::PathBuf, thread}; -use super::{error::Error, instance::Wait, Instance}; +use crate::sandbox::{error::Error, instance::Wait, Instance}; /// LibcontainerInstance is a trait that gets implemented by a WASI runtime that /// uses youki's libcontainer library as the container runtime. diff --git a/crates/containerd-shim-wasm/src/libcontainer_instance/mod.rs b/crates/containerd-shim-wasm/src/libcontainer_instance/mod.rs new file mode 100644 index 000000000..7d0b07faa --- /dev/null +++ b/crates/containerd-shim-wasm/src/libcontainer_instance/mod.rs @@ -0,0 +1,5 @@ +pub mod container_executor; +pub mod instance; + +pub use container_executor::LinuxContainerExecutor; +pub use instance::LibcontainerInstance; diff --git a/crates/containerd-shim-wasm/src/sandbox/mod.rs b/crates/containerd-shim-wasm/src/sandbox/mod.rs index 5f683d776..0abbac89e 100644 --- a/crates/containerd-shim-wasm/src/sandbox/mod.rs +++ b/crates/containerd-shim-wasm/src/sandbox/mod.rs @@ -2,17 +2,11 @@ use crate::services::sandbox; -// pub mod cgroups; pub mod error; -// pub mod exec; pub mod instance; pub mod instance_utils; -#[cfg(feature = "libcontainer")] -pub mod libcontainer_instance; pub mod manager; pub mod shim; -#[cfg(feature = "libcontainer")] -pub use libcontainer_instance::LibcontainerInstance; pub use error::{Error, Result}; pub use instance::{EngineGetter, Instance, InstanceConfig}; diff --git a/crates/containerd-shim-wasmedge/src/executor.rs b/crates/containerd-shim-wasmedge/src/executor.rs index 37425eaa6..0660cb11f 100644 --- a/crates/containerd-shim-wasmedge/src/executor.rs +++ b/crates/containerd-shim-wasmedge/src/executor.rs @@ -6,7 +6,7 @@ use oci_spec::runtime::Spec; use libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; use libcontainer::workload::{Executor, ExecutorError}; use log::debug; -use std::os::unix::io::RawFd; +use std::{os::unix::io::RawFd, path::PathBuf}; use wasmedge_sdk::{ config::{CommonConfigOptions, ConfigBuilder, HostRegistrationConfigOptions}, @@ -16,9 +16,19 @@ use wasmedge_sdk::{ const EXECUTOR_NAME: &str = "wasmedge"; pub struct WasmEdgeExecutor { - pub stdin: Option, - pub stdout: Option, - pub stderr: Option, + stdin: Option, + stdout: Option, + stderr: Option, +} + +impl WasmEdgeExecutor { + pub fn new(stdin: Option, stdout: Option, stderr: Option) -> Self { + Self { + stdin, + stdout, + stderr, + } + } } impl Executor for WasmEdgeExecutor { @@ -43,8 +53,21 @@ impl Executor for WasmEdgeExecutor { }; } - fn can_handle(&self, _spec: &Spec) -> bool { - true + fn can_handle(&self, spec: &Spec) -> bool { + // check if the entrypoint of the spec is a wasm binary. + let args = oci::get_args(spec); + if args.is_empty() { + return false; + } + + let start = args[0].clone(); + let mut iterator = start.split('#'); + let cmd = iterator.next().unwrap().to_string(); + let path = PathBuf::from(cmd); + + path.extension() + .map(|ext| ext.to_ascii_lowercase()) + .is_some_and(|ext| ext == "wasm" || ext == "wat") } fn name(&self) -> &'static str { diff --git a/crates/containerd-shim-wasmedge/src/instance.rs b/crates/containerd-shim-wasmedge/src/instance.rs index ae78a20ef..8523fe2b3 100644 --- a/crates/containerd-shim-wasmedge/src/instance.rs +++ b/crates/containerd-shim-wasmedge/src/instance.rs @@ -6,10 +6,12 @@ use std::sync::{Arc, Condvar, Mutex}; use anyhow::Context; use anyhow::Result; +use containerd_shim_wasm::libcontainer_instance::LibcontainerInstance; +use containerd_shim_wasm::libcontainer_instance::LinuxContainerExecutor; use containerd_shim_wasm::sandbox::error::Error; use containerd_shim_wasm::sandbox::instance::ExitCode; use containerd_shim_wasm::sandbox::instance_utils::maybe_open_stdio; -use containerd_shim_wasm::sandbox::{EngineGetter, InstanceConfig, LibcontainerInstance}; +use containerd_shim_wasm::sandbox::{EngineGetter, InstanceConfig}; use libc::{dup2, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; use nix::unistd::close; use serde::{Deserialize, Serialize}; @@ -125,12 +127,11 @@ impl LibcontainerInstance for Wasi { let syscall = create_syscall(); let err_others = |err| Error::Others(format!("failed to create container: {}", err)); + let default_executor = Box::new(LinuxContainerExecutor::new(stdin, stdout, stderr)); + let wasmedge_executor = Box::new(WasmEdgeExecutor::new(stdin, stdout, stderr)); + let container = ContainerBuilder::new(self.id.clone(), syscall.as_ref()) - .with_executor(vec![Box::new(WasmEdgeExecutor { - stdin, - stdout, - stderr, - })]) + .with_executor(vec![default_executor, wasmedge_executor]) .map_err(err_others)? .with_root_path(self.rootdir.clone()) .map_err(err_others)? diff --git a/crates/containerd-shim-wasmtime/src/executor.rs b/crates/containerd-shim-wasmtime/src/executor.rs index e992ce5ee..1f935afb0 100644 --- a/crates/containerd-shim-wasmtime/src/executor.rs +++ b/crates/containerd-shim-wasmtime/src/executor.rs @@ -1,5 +1,5 @@ use nix::unistd::{dup, dup2}; -use std::{fs::OpenOptions, os::fd::RawFd}; +use std::{fs::OpenOptions, os::fd::RawFd, path::PathBuf}; use anyhow::{anyhow, Result}; use containerd_shim_wasm::sandbox::oci; @@ -15,10 +15,26 @@ use crate::oci_wasmtime::{self, wasi_dir}; const EXECUTOR_NAME: &str = "wasmtime"; pub struct WasmtimeExecutor { - pub stdin: Option, - pub stdout: Option, - pub stderr: Option, - pub engine: Engine, + stdin: Option, + stdout: Option, + stderr: Option, + engine: Engine, +} + +impl WasmtimeExecutor { + pub fn new( + stdin: Option, + stdout: Option, + stderr: Option, + engine: Engine, + ) -> Self { + Self { + stdin, + stdout, + stderr, + engine, + } + } } impl Executor for WasmtimeExecutor { @@ -39,8 +55,27 @@ impl Executor for WasmtimeExecutor { }; } - fn can_handle(&self, _spec: &Spec) -> bool { - true + fn can_handle(&self, spec: &Spec) -> bool { + // check if the entrypoint of the spec is a wasm binary. + let args = oci::get_args(spec); + if args.is_empty() { + return false; + } + + let start = args[0].clone(); + let mut iterator = start.split('#'); + let cmd = iterator.next().unwrap().to_string(); + let path = PathBuf::from(cmd); + + // TODO: do we need to validate the wasm binary? + // ```rust + // let bytes = std::fs::read(path).unwrap(); + // wasmparser::validate(&bytes).is_ok() + // ``` + + path.extension() + .map(|ext| ext.to_ascii_lowercase()) + .is_some_and(|ext| ext == "wasm" || ext == "wat") } fn name(&self) -> &'static str { diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index 7e745b673..714f6aa01 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -5,28 +5,22 @@ use nix::unistd::close; use serde::{Deserialize, Serialize}; use std::fs::File; use std::io::{ErrorKind, Read}; -use std::os::fd::RawFd; use std::path::{Path, PathBuf}; use std::sync::{Arc, Condvar, Mutex}; use anyhow::Context; +use containerd_shim_wasm::libcontainer_instance::{LibcontainerInstance, LinuxContainerExecutor}; use containerd_shim_wasm::sandbox::error::Error; use containerd_shim_wasm::sandbox::instance::ExitCode; use containerd_shim_wasm::sandbox::instance_utils::maybe_open_stdio; -use containerd_shim_wasm::sandbox::{EngineGetter, InstanceConfig, LibcontainerInstance}; -use libc::{dup2, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; +use containerd_shim_wasm::sandbox::{EngineGetter, InstanceConfig}; use libcontainer::syscall::syscall::create_syscall; use wasmtime::Engine; use crate::executor::WasmtimeExecutor; - static DEFAULT_CONTAINER_ROOT_DIR: &str = "/run/containerd/wasmtime"; -static mut STDIN_FD: Option = None; -static mut STDOUT_FD: Option = None; -static mut STDERR_FD: Option = None; - pub struct Wasi { exit_code: ExitCode, engine: wasmtime::Engine, @@ -38,20 +32,6 @@ pub struct Wasi { id: String, } -pub fn reset_stdio() { - unsafe { - if STDIN_FD.is_some() { - dup2(STDIN_FD.unwrap(), STDIN_FILENO); - } - if STDOUT_FD.is_some() { - dup2(STDOUT_FD.unwrap(), STDOUT_FILENO); - } - if STDERR_FD.is_some() { - dup2(STDERR_FD.unwrap(), STDERR_FILENO); - } - } -} - #[derive(Serialize, Deserialize)] struct Options { root: Option, @@ -123,13 +103,12 @@ impl LibcontainerInstance for Wasi { let stdout = maybe_open_stdio(&self.stdout).context("could not open stdout")?; let stderr = maybe_open_stdio(&self.stderr).context("could not open stderr")?; let err_others = |err| Error::Others(format!("failed to create container: {}", err)); + + let wasmtime_executor = Box::new(WasmtimeExecutor::new(stdin, stdout, stderr, engine)); + let default_executor = Box::new(LinuxContainerExecutor::new(stdin, stdout, stderr)); + let container = ContainerBuilder::new(self.id.clone(), syscall.as_ref()) - .with_executor(vec![Box::new(WasmtimeExecutor { - stdin, - stdout, - stderr, - engine, - })]) + .with_executor(vec![default_executor, wasmtime_executor]) .map_err(err_others)? .with_root_path(self.rootdir.clone()) .map_err(err_others)? @@ -160,18 +139,26 @@ mod wasitest { use std::borrow::Cow; use std::fs::{create_dir, read_to_string, File, OpenOptions}; use std::io::prelude::*; + use std::os::fd::RawFd; use std::os::unix::prelude::OpenOptionsExt; use std::sync::mpsc::channel; use std::time::Duration; + use chrono::{DateTime, Utc}; use containerd_shim_wasm::function; use containerd_shim_wasm::sandbox::instance::Wait; use containerd_shim_wasm::sandbox::testutil::{has_cap_sys_admin, run_test_with_sudo}; use containerd_shim_wasm::sandbox::Instance; use libc::SIGKILL; + use libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; + use nix::unistd::dup2; use oci_spec::runtime::{ProcessBuilder, RootBuilder, SpecBuilder}; use tempfile::{tempdir, TempDir}; + static mut STDIN_FD: Option = None; + static mut STDOUT_FD: Option = None; + static mut STDERR_FD: Option = None; + use super::*; // This is taken from https://github.com/bytecodealliance/wasmtime/blob/6a60e8363f50b936e4c4fc958cb9742314ff09f3/docs/WASI-tutorial.md?plain=1#L270-L298 @@ -350,4 +337,18 @@ mod wasitest { wasi.delete()?; res } + + fn reset_stdio() { + unsafe { + if STDIN_FD.is_some() { + let _ = dup2(STDIN_FD.unwrap(), STDIN_FILENO); + } + if STDOUT_FD.is_some() { + let _ = dup2(STDOUT_FD.unwrap(), STDOUT_FILENO); + } + if STDERR_FD.is_some() { + let _ = dup2(STDERR_FD.unwrap(), STDERR_FILENO); + } + } + } } diff --git a/test/k8s/deploy.yaml b/test/k8s/deploy.yaml index 91f59edbf..64d0de157 100644 --- a/test/k8s/deploy.yaml +++ b/test/k8s/deploy.yaml @@ -24,4 +24,8 @@ spec: containers: - name: demo image: ghcr.io/containerd/runwasi/wasi-demo-app:latest - imagePullPolicy: Never \ No newline at end of file + imagePullPolicy: Never + - name: nginx + image: docker.io/nginx:latest + ports: + - containerPort: 80 \ No newline at end of file