fuse: Ensure fd has same flags as read/write

We found when running minio with recent versions of nydus that it
was no longer able to boot properly due to Invalid Argument when
writing to a temporary file.
Tracing the high-level flow showed that the file was originally
opened with O_DIRECT, then later on if the buffer size was not
correctly aligned, fcntl with F_SETFL would be used to remove
the O_DIRECT flag.
In virtiofs there is no independant call for fcntl, so the
underlying file descriptor opened by nydus still had the
O_DIRECT flag set. Meaning that during a write to this file
it would error with -EINVAL as the buffer size was not aligned.

This patch aims to fix this by first recording the open flags for
a file in the `HandleData` struct, then on read/write checking
these flags match the flags provided by virtiofs in the read/write
message. If the flags do not match, the current flags of the file
descriptor are set with F_SETFL and updated on the HandleData entry.

Signed-off-by: Champ-Goblem <[email protected]>

Author: Champ-Goblem

src/passthrough/mod.rs

@@ -23,7 +23,7 @@ use std::os::unix::ffi::OsStringExt;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::path::PathBuf;
 use std::str::FromStr;
-use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicU8, Ordering};
 use std::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockWriteGuard};
 use std::time::Duration;
 
@@ -333,14 +333,16 @@ struct HandleData {
     inode: Inode,
     file: File,
     lock: Mutex<()>,
+    open_flags: AtomicU32,
 }
 
 impl HandleData {
-    fn new(inode: Inode, file: File) -> Self {
+    fn new(inode: Inode, file: File, flags: u32) -> Self {
         HandleData {
             inode,
             file,
             lock: Mutex::new(()),
+            open_flags: AtomicU32::new(flags),
         }
     }
 
@@ -355,6 +357,14 @@ impl HandleData {
     fn get_handle_raw_fd(&self) -> RawFd {
         self.file.as_raw_fd()
     }
+
+    fn get_flags(&self) -> u32 {
+        self.open_flags.load(Ordering::Relaxed)
+    }
+
+    fn set_flags(&self, flags: u32) {
+        self.open_flags.store(flags, Ordering::Relaxed);
+    }
 }
 
 struct HandleMap {

src/passthrough/sync_io.rs

@@ -36,6 +36,22 @@ impl<S: BitmapSlice + Send + Sync> PassthroughFs<S> {
         Self::open_proc_file(&self.proc_self_fd, file.as_raw_fd(), new_flags, data.mode)
     }
 
+    /// Check the HandleData flags against the flags from the current request
+    /// if these do not match update the file descriptor flags and store the new
+    /// result in the HandleData entry
+    #[inline(always)]
+    fn check_fd_flags(&self, data: Arc<HandleData>, fd: RawFd, flags: u32) -> io::Result<()> {
+        let open_flags = data.get_flags();
+        if open_flags != flags {
+            let ret = unsafe { libc::fcntl(fd, libc::F_SETFL, flags) };
+            if ret != 0 {
+                return Err(io::Error::last_os_error());
+            }
+            data.set_flags(flags);
+        }
+        Ok(())
+    }
+
     fn do_readdir(
         &self,
         inode: Inode,
@@ -173,7 +189,7 @@ impl<S: BitmapSlice + Send + Sync> PassthroughFs<S> {
         let file = self.open_inode(inode, flags as i32)?;
         drop(killpriv);
 
-        let data = HandleData::new(inode, file);
+        let data = HandleData::new(inode, file, flags);
         let handle = self.next_handle.fetch_add(1, Ordering::Relaxed);
         self.handle_map.insert(handle, data);
 
@@ -258,7 +274,7 @@ impl<S: BitmapSlice + Send + Sync> PassthroughFs<S> {
             self.handle_map.get(handle, inode)
         } else {
             let file = self.open_inode(inode, flags | libc::O_DIRECTORY)?;
-            Ok(Arc::new(HandleData::new(inode, file)))
+            Ok(Arc::new(HandleData::new(inode, file, flags as u32)))
         }
     }
 
@@ -273,7 +289,7 @@ impl<S: BitmapSlice + Send + Sync> PassthroughFs<S> {
             self.handle_map.get(handle, inode)
         } else {
             let file = self.open_inode(inode, flags)?;
-            Ok(Arc::new(HandleData::new(inode, file)))
+            Ok(Arc::new(HandleData::new(inode, file, flags as u32)))
         }
     }
 }
@@ -573,7 +589,7 @@ impl<S: BitmapSlice + Send + Sync> FileSystem for PassthroughFs<S> {
 
         let ret_handle = if !self.no_open.load(Ordering::Relaxed) {
             let handle = self.next_handle.fetch_add(1, Ordering::Relaxed);
-            let data = HandleData::new(entry.inode, file);
+            let data = HandleData::new(entry.inode, file, args.flags);
 
             self.handle_map.insert(handle, data);
             Some(handle)
@@ -643,14 +659,17 @@ impl<S: BitmapSlice + Send + Sync> FileSystem for PassthroughFs<S> {
         size: u32,
         offset: u64,
         _lock_owner: Option<u64>,
-        _flags: u32,
+        flags: u32,
     ) -> io::Result<usize> {
         let data = self.get_data(handle, inode, libc::O_RDONLY)?;
 
         // Manually implement File::try_clone() by borrowing fd of data.file instead of dup().
         // It's safe because the `data` variable's lifetime spans the whole function,
         // so data.file won't be closed.
         let f = unsafe { File::from_raw_fd(data.get_handle_raw_fd()) };
+
+        self.check_fd_flags(data, f.as_raw_fd(), flags)?;
+
         let mut f = ManuallyDrop::new(f);
 
         w.write_from(&mut *f, size as usize, offset)
@@ -666,7 +685,7 @@ impl<S: BitmapSlice + Send + Sync> FileSystem for PassthroughFs<S> {
         offset: u64,
         _lock_owner: Option<u64>,
         _delayed_write: bool,
-        _flags: u32,
+        flags: u32,
         fuse_flags: u32,
     ) -> io::Result<usize> {
         let data = self.get_data(handle, inode, libc::O_RDWR)?;
@@ -676,6 +695,8 @@ impl<S: BitmapSlice + Send + Sync> FileSystem for PassthroughFs<S> {
         // so data.file won't be closed.
         let f = unsafe { File::from_raw_fd(data.get_handle_raw_fd()) };
 
+        self.check_fd_flags(data, f.as_raw_fd(), flags)?;
+
         if self.seal_size.load(Ordering::Relaxed) {
             let st = Self::stat_fd(f.as_raw_fd(), None)?;
             self.seal_size_check(Opcode::Write, st.st_size as u64, offset, size as u64, 0)?;