overlay: refactor first implementation

This commit refactor a lot to previous one, including:

* Modified layer trait
* Lots of bugfixes to make xfstests happy.
* Performance improvement.
* Whiteout logics
* Remove plugin & Direct layer logics.
* A better inode allocator with inode consistency guarantee.

Signed-off-by: Wei Zhang <[email protected]>

Author: WeiZhang555

Cargo.toml

@@ -27,6 +27,7 @@ libc = "0.2.68"
 log = "0.4.6"
 mio = { version = "0.8", features = ["os-poll", "os-ext"] }
 nix = "0.24"
+radix_trie = "0.2.1"
 tokio = { version = "1", optional = true }
 tokio-uring = { version = "0.4.0", optional = true }
 vmm-sys-util = { version = "0.11", optional = true }

src/api/filesystem/mod.rs

@@ -30,6 +30,11 @@ pub use async_io::{AsyncFileSystem, AsyncZeroCopyReader, AsyncZeroCopyWriter};
 mod sync_io;
 pub use sync_io::FileSystem;
 
+#[cfg(all(any(feature = "fusedev", feature = "virtiofs"), target_os = "linux"))]
+mod overlay;
+#[cfg(all(any(feature = "fusedev", feature = "virtiofs"), target_os = "linux"))]
+pub use overlay::Layer;
+
 /// Information about a path in the filesystem.
 #[derive(Copy, Clone, Debug)]
 pub struct Entry {

src/api/filesystem/overlay.rs

@@ -0,0 +1,205 @@
+// Copyright (C) 2023 Ant Group. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE-BSD-3-Clause file.
+
+#![allow(missing_docs)]
+
+use std::ffi::{CStr, CString};
+use std::io::{Error, ErrorKind, Result};
+
+use super::{Context, Entry, FileSystem, GetxattrReply};
+use crate::abi::fuse_abi::stat64;
+
+pub const OPAQUE_XATTR_LEN: u32 = 16;
+pub const OPAQUE_XATTR: &str = "user.fuseoverlayfs.opaque";
+pub const UNPRIVILEGED_OPAQUE_XATTR: &str = "user.overlay.opaque";
+pub const PRIVILEGED_OPAQUE_XATTR: &str = "trusted.overlay.opaque";
+
+/// A filesystem must implement Layer trait, or it cannot be used as an OverlayFS layer.
+pub trait Layer: FileSystem {
+    /// Return the root inode number
+    fn root_inode(&self) -> Self::Inode;
+
+    /// Create whiteout file with name <name>.
+    ///
+    /// If this call is successful then the lookup count of the `Inode` associated with the returned
+    /// `Entry` must be increased by 1.
+    fn create_whiteout(&self, ctx: &Context, parent: Self::Inode, name: &CStr) -> Result<Entry> {
+        // Use temp value to avoid moved 'parent'.
+        let ino: u64 = parent.into();
+        match self.lookup(ctx, ino.into(), name) {
+            Ok(v) => {
+                // Find whiteout char dev.
+                if is_whiteout(v.attr) {
+                    return Ok(v);
+                }
+                // Non-negative entry with inode larger than 0 indicates file exists.
+                if v.inode != 0 {
+                    // Decrease the refcount.
+                    self.forget(ctx, v.inode.into(), 1);
+                    // File exists with same name, create whiteout file is not allowed.
+                    return Err(Error::from_raw_os_error(libc::EEXIST));
+                }
+            }
+            Err(e) => match e.raw_os_error() {
+                Some(raw_error) => {
+                    // We expect ENOENT error.
+                    if raw_error != libc::ENOENT {
+                        return Err(e);
+                    }
+                }
+                None => return Err(e),
+            },
+        }
+
+        // Try to create whiteout char device with 0/0 device number.
+        let dev = libc::makedev(0, 0);
+        let mode = libc::S_IFCHR | 0o777;
+        self.mknod(ctx, ino.into(), name, mode, dev as u32, 0)
+    }
+
+    /// Delete whiteout file with name <name>.
+    fn delete_whiteout(&self, ctx: &Context, parent: Self::Inode, name: &CStr) -> Result<()> {
+        // Use temp value to avoid moved 'parent'.
+        let ino: u64 = parent.into();
+        match self.lookup(ctx, ino.into(), name) {
+            Ok(v) => {
+                if v.inode != 0 {
+                    // Decrease the refcount since we make a lookup call.
+                    self.forget(ctx, v.inode.into(), 1);
+                }
+
+                // Find whiteout so we can safely delete it.
+                if is_whiteout(v.attr) {
+                    return self.unlink(ctx, ino.into(), name);
+                }
+                //  Non-negative entry with inode larger than 0 indicates file exists.
+                if v.inode != 0 {
+                    // File exists but not whiteout file.
+                    return Err(Error::from_raw_os_error(libc::EINVAL));
+                }
+            }
+            Err(e) => match e.raw_os_error() {
+                Some(raw_error) => {
+                    // ENOENT is acceptable.
+                    if raw_error != libc::ENOENT {
+                        return Err(e);
+                    }
+                }
+                None => return Err(e),
+            },
+        }
+        Ok(())
+    }
+
+    /// Check if the Inode is a whiteout file
+    fn is_whiteout(&self, ctx: &Context, inode: Self::Inode) -> Result<bool> {
+        let (st, _) = self.getattr(ctx, inode, None)?;
+
+        // Check attributes of the inode to see if it's a whiteout char device.
+        Ok(is_whiteout(st))
+    }
+
+    /// Set the directory to opaque.
+    fn set_opaque(&self, ctx: &Context, inode: Self::Inode) -> Result<()> {
+        // Use temp value to avoid moved 'parent'.
+        let ino: u64 = inode.into();
+
+        // Get attributes and check if it's directory.
+        let (st, _d) = self.getattr(ctx, ino.into(), None)?;
+        if !is_dir(st) {
+            // Only directory can be set to opaque.
+            return Err(Error::from_raw_os_error(libc::ENOTDIR));
+        }
+        // A directory is made opaque by setting the xattr "trusted.overlay.opaque" to "y".
+        // See ref: https://docs.kernel.org/filesystems/overlayfs.html#whiteouts-and-opaque-directories
+        self.setxattr(
+            ctx,
+            ino.into(),
+            to_cstring(OPAQUE_XATTR)?.as_c_str(),
+            b"y",
+            0,
+        )
+    }
+
+    /// Check if the directory is opaque.
+    fn is_opaque(&self, ctx: &Context, inode: Self::Inode) -> Result<bool> {
+        // Use temp value to avoid moved 'parent'.
+        let ino: u64 = inode.into();
+
+        // Get attributes of the directory.
+        let (st, _d) = self.getattr(ctx, ino.into(), None)?;
+        if !is_dir(st) {
+            return Err(Error::from_raw_os_error(libc::ENOTDIR));
+        }
+
+        // Return Result<is_opaque>.
+        let check_attr = |inode: Self::Inode, attr_name: &str, attr_size: u32| -> Result<bool> {
+            let cname = CString::new(attr_name)?;
+            match self.getxattr(ctx, inode, cname.as_c_str(), attr_size) {
+                Ok(v) => {
+                    // xattr name exists and we get value.
+                    if let GetxattrReply::Value(buf) = v {
+                        if buf.len() == 1 && buf[0].to_ascii_lowercase() == b'y' {
+                            return Ok(true);
+                        }
+                    }
+                    // No value found, go on to next check.
+                    Ok(false)
+                }
+                Err(e) => {
+                    if let Some(raw_error) = e.raw_os_error() {
+                        if raw_error == libc::ENODATA {
+                            return Ok(false);
+                        }
+                    }
+
+                    Err(e)
+                }
+            }
+        };
+
+        // A directory is made opaque by setting some specific xattr to "y".
+        // See ref: https://docs.kernel.org/filesystems/overlayfs.html#whiteouts-and-opaque-directories
+
+        // Check our customized version of the xattr "user.fuseoverlayfs.opaque".
+        let is_opaque = check_attr(ino.into(), OPAQUE_XATTR, OPAQUE_XATTR_LEN)?;
+        if is_opaque {
+            return Ok(true);
+        }
+
+        // Also check for the unprivileged version of the xattr "trusted.overlay.opaque".
+        let is_opaque = check_attr(ino.into(), PRIVILEGED_OPAQUE_XATTR, OPAQUE_XATTR_LEN)?;
+        if is_opaque {
+            return Ok(true);
+        }
+
+        // Also check for the unprivileged version of the xattr "user.overlay.opaque".
+        let is_opaque = check_attr(ino.into(), UNPRIVILEGED_OPAQUE_XATTR, OPAQUE_XATTR_LEN)?;
+        if is_opaque {
+            return Ok(true);
+        }
+
+        Ok(false)
+    }
+}
+
+pub(crate) fn is_dir(st: stat64) -> bool {
+    st.st_mode & libc::S_IFMT == libc::S_IFDIR
+}
+
+pub(crate) fn is_chardev(st: stat64) -> bool {
+    st.st_mode & libc::S_IFMT == libc::S_IFCHR
+}
+
+pub(crate) fn is_whiteout(st: stat64) -> bool {
+    // A whiteout is created as a character device with 0/0 device number.
+    // See ref: https://docs.kernel.org/filesystems/overlayfs.html#whiteouts-and-opaque-directories
+    let major = unsafe { libc::major(st.st_rdev) };
+    let minor = unsafe { libc::minor(st.st_rdev) };
+    is_chardev(st) && major == 0 && minor == 0
+}
+
+pub(crate) fn to_cstring(name: &str) -> Result<CString> {
+    CString::new(name).map_err(|e| Error::new(ErrorKind::InvalidData, e))
+}

src/lib.rs

@@ -117,10 +117,11 @@ pub type Result<T> = ::std::result::Result<T, Error>;
 pub mod abi;
 pub mod api;
 
+#[cfg(all(any(feature = "fusedev", feature = "virtiofs"), target_os = "linux"))]
+pub mod overlayfs;
 #[cfg(all(any(feature = "fusedev", feature = "virtiofs"), target_os = "linux"))]
 pub mod passthrough;
 pub mod transport;
-pub mod overlayfs;
 
 pub mod common;
 pub use self::common::*;

src/overlayfs/config.rs

@@ -1,53 +1,45 @@
+// Copyright (C) 2023 Ant Group. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
 
-use std::time::Duration;
 use self::super::CachePolicy;
 use std::fmt;
+use std::time::Duration;
 
 #[derive(Default, Clone, Debug)]
 pub struct Config {
-	pub upper: String,
-	pub lower: Vec<String>,
-	pub work: String,
-	pub mountpoint: String,
-	pub do_import: bool,
-	pub writeback: bool,
-	pub no_open: bool,
-	pub no_opendir: bool,
-	pub killpriv_v2: bool,
-	pub no_readdir: bool,
-	pub xattr: bool,
-	pub xattr_permissions: bool,
-	pub perfile_dax: bool,
-	pub cache_policy: CachePolicy,
-	pub attr_timeout: Duration,
-	pub entry_timeout: Duration,
-}
-
-impl Default for CachePolicy {
-	fn default() -> Self {
-		CachePolicy::Auto
-	}
+    pub mountpoint: String,
+    pub work: String,
+    pub do_import: bool,
+    // Filesystem options.
+    pub writeback: bool,
+    pub no_open: bool,
+    pub no_opendir: bool,
+    pub killpriv_v2: bool,
+    pub no_readdir: bool,
+    pub perfile_dax: bool,
+    pub cache_policy: CachePolicy,
+    pub attr_timeout: Duration,
+    pub entry_timeout: Duration,
 }
 
 impl Clone for CachePolicy {
-	fn clone(&self) -> Self {
-		match *self {
-			CachePolicy::Never => CachePolicy::Never,
-			CachePolicy::Always => CachePolicy::Always,
-			CachePolicy::Auto => CachePolicy::Auto,
-		}
-	}
+    fn clone(&self) -> Self {
+        match *self {
+            CachePolicy::Never => CachePolicy::Never,
+            CachePolicy::Always => CachePolicy::Always,
+            CachePolicy::Auto => CachePolicy::Auto,
+        }
+    }
 }
 
 impl fmt::Debug for CachePolicy {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		let policy = 
-		match *self {
-			CachePolicy::Never => "Never",
-			CachePolicy::Always => "Always",
-			CachePolicy::Auto => "Auto",
-		};
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let policy = match *self {
+            CachePolicy::Never => "Never",
+            CachePolicy::Always => "Always",
+            CachePolicy::Auto => "Auto",
+        };
 
-		write!(f, "CachePolicy: {}", policy)
-	}
+        write!(f, "CachePolicy: {}", policy)
+    }
 }