Skip to content

Commit deb6321

Browse files
committed
fusedev: test async lookups, notifications and takeover on tokio
Add tests that serve a mount with FuseDevTask on a current-thread tokio runtime and a file system whose lookups wait in the file system: - A lookup that waits doesn't hold up a lookup in the same directory, and notify_inval_entry() from another thread works while the task serves. - A second server takes over the connection from one that stopped with a lookup in flight. It restores the INIT state, sends notify_resend(), and answers the lookup that the kernel sends again. The file system enables FUSE_PARALLEL_DIROPS, without which the kernel serializes lookups in a directory. Signed-off-by: Ilan Joselevich <personal@ilanjoselevich.com>
1 parent 33d1c8c commit deb6321

1 file changed

Lines changed: 342 additions & 7 deletions

File tree

‎crates/fuse-backend-fusedev/tests/tokio_runtime.rs‎

Lines changed: 342 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,28 @@
66
77
#[cfg(all(target_os = "linux", feature = "async-io"))]
88
mod tokio_runtime {
9-
use std::sync::atomic::AtomicBool;
9+
use std::ffi::CStr;
10+
use std::fs::File;
11+
use std::io;
12+
use std::os::unix::io::AsRawFd;
13+
use std::path::Path;
14+
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
1015
use std::sync::mpsc::channel;
1116
use std::sync::Arc;
1217
use std::thread;
1318
use std::time::Duration;
1419

20+
use async_trait::async_trait;
21+
use fuse_backend_core::abi::fuse_abi::{stat64, CreateIn, FsOptions, OpenOptions, ROOT_ID};
22+
use fuse_backend_core::api::filesystem::{
23+
AsyncFileSystem, AsyncZeroCopyReader, AsyncZeroCopyWriter, Context, Entry, FileSystem,
24+
SetattrValid,
25+
};
1526
use fuse_backend_core::api::server::Server;
1627
use fuse_backend_core::async_runtime::{self, RuntimeType};
17-
use fuse_backend_fusedev::{FuseDevTask, FuseSession};
28+
use fuse_backend_fusedev::{FuseDevTask, FuseDevWriter, FuseSession};
1829
use fuse_backend_passthrough::{Config, PassthroughFs};
30+
use tokio::sync::{oneshot, Semaphore};
1931
use vmm_sys_util::tempdir::TempDir;
2032

2133
/// CPU time used by the calling thread.
@@ -45,12 +57,9 @@ mod tokio_runtime {
4557
let server = Arc::new(Server::new(fs));
4658

4759
let mnt = TempDir::new().unwrap();
48-
let mut se = FuseSession::new(mnt.as_path(), "tokio_runtime_test", "", false).unwrap();
49-
se.set_allow_other(false);
50-
if let Err(e) = se.mount() {
51-
eprintln!("can't mount ({}), skipping", e);
60+
let Some(mut se) = mount(mnt.as_path()) else {
5261
return;
53-
}
62+
};
5463
let file = se.clone_fuse_file().unwrap();
5564
let buf_size = se.bufsize();
5665

@@ -83,4 +92,330 @@ mod tokio_runtime {
8392
cpu
8493
);
8594
}
95+
96+
fn mount(mountpoint: &Path) -> Option<FuseSession> {
97+
let mut se = FuseSession::new(mountpoint, "tokio_runtime_test", "", false).unwrap();
98+
se.set_allow_other(false);
99+
match se.mount() {
100+
Ok(()) => Some(se),
101+
Err(e) => {
102+
eprintln!("can't mount ({}), skipping", e);
103+
None
104+
}
105+
}
106+
}
107+
108+
/// Stat `path` in a thread of its own and send the inode number.
109+
fn stat_in_thread(path: std::path::PathBuf) -> std::sync::mpsc::Receiver<u64> {
110+
let (tx, rx) = channel();
111+
thread::spawn(move || {
112+
use std::os::unix::fs::MetadataExt;
113+
let _ = tx.send(std::fs::metadata(path).unwrap().ino());
114+
});
115+
rx
116+
}
117+
118+
const FAST: u64 = 2;
119+
const SLOW: u64 = 3;
120+
121+
/// A root directory with the files `fast` and `slow`. Looking up `slow` waits for a permit
122+
/// from `gate`, or forever with `stall`.
123+
#[derive(Default)]
124+
struct TestFs {
125+
gate: Option<Arc<Semaphore>>,
126+
stall: bool,
127+
fast_lookups: AtomicUsize,
128+
has_resend: AtomicBool,
129+
}
130+
131+
fn stat(inode: u64) -> stat64 {
132+
let mut st: stat64 = unsafe { std::mem::zeroed() };
133+
st.st_ino = inode;
134+
st.st_nlink = 1;
135+
st.st_mode = if inode == ROOT_ID {
136+
libc::S_IFDIR | 0o755
137+
} else {
138+
libc::S_IFREG | 0o444
139+
};
140+
st
141+
}
142+
143+
fn entry(inode: u64) -> Entry {
144+
Entry {
145+
inode,
146+
generation: 0,
147+
attr: stat(inode),
148+
attr_flags: 0,
149+
attr_timeout: Duration::from_secs(60),
150+
entry_timeout: Duration::from_secs(60),
151+
}
152+
}
153+
154+
fn enosys<T>() -> io::Result<T> {
155+
Err(io::Error::from_raw_os_error(libc::ENOSYS))
156+
}
157+
158+
impl FileSystem for TestFs {
159+
type Inode = u64;
160+
type Handle = u64;
161+
162+
fn init(&self, capable: FsOptions) -> io::Result<FsOptions> {
163+
self.has_resend
164+
.store(capable.contains(FsOptions::HAS_RESEND), Ordering::Relaxed);
165+
// Without PARALLEL_DIROPS, the kernel serializes lookups in a directory.
166+
Ok(capable & (FsOptions::HAS_RESEND | FsOptions::PARALLEL_DIROPS))
167+
}
168+
}
169+
170+
#[allow(unused_variables)]
171+
#[async_trait]
172+
impl AsyncFileSystem for TestFs {
173+
async fn async_lookup(&self, ctx: &Context, parent: u64, name: &CStr) -> io::Result<Entry> {
174+
match name.to_bytes() {
175+
b"fast" => {
176+
self.fast_lookups.fetch_add(1, Ordering::Relaxed);
177+
Ok(entry(FAST))
178+
}
179+
b"slow" => {
180+
if self.stall {
181+
std::future::pending::<()>().await;
182+
}
183+
if let Some(gate) = &self.gate {
184+
gate.acquire().await.unwrap().forget();
185+
}
186+
Ok(entry(SLOW))
187+
}
188+
_ => Err(io::Error::from_raw_os_error(libc::ENOENT)),
189+
}
190+
}
191+
192+
async fn async_getattr(
193+
&self,
194+
ctx: &Context,
195+
inode: u64,
196+
handle: Option<u64>,
197+
) -> io::Result<(stat64, Duration)> {
198+
Ok((stat(inode), Duration::from_secs(60)))
199+
}
200+
201+
async fn async_setattr(
202+
&self,
203+
ctx: &Context,
204+
inode: u64,
205+
attr: stat64,
206+
handle: Option<u64>,
207+
valid: SetattrValid,
208+
) -> io::Result<(stat64, Duration)> {
209+
enosys()
210+
}
211+
212+
async fn async_open(
213+
&self,
214+
ctx: &Context,
215+
inode: u64,
216+
flags: u32,
217+
fuse_flags: u32,
218+
) -> io::Result<(Option<u64>, OpenOptions)> {
219+
enosys()
220+
}
221+
222+
async fn async_create(
223+
&self,
224+
ctx: &Context,
225+
parent: u64,
226+
name: &CStr,
227+
args: CreateIn,
228+
) -> io::Result<(Entry, Option<u64>, OpenOptions)> {
229+
enosys()
230+
}
231+
232+
async fn async_read(
233+
&self,
234+
ctx: &Context,
235+
inode: u64,
236+
handle: u64,
237+
w: &mut (dyn AsyncZeroCopyWriter + Send),
238+
size: u32,
239+
offset: u64,
240+
lock_owner: Option<u64>,
241+
flags: u32,
242+
) -> io::Result<usize> {
243+
enosys()
244+
}
245+
246+
async fn async_write(
247+
&self,
248+
ctx: &Context,
249+
inode: u64,
250+
handle: u64,
251+
r: &mut (dyn AsyncZeroCopyReader + Send),
252+
size: u32,
253+
offset: u64,
254+
lock_owner: Option<u64>,
255+
delayed_write: bool,
256+
flags: u32,
257+
fuse_flags: u32,
258+
) -> io::Result<usize> {
259+
enosys()
260+
}
261+
262+
async fn async_fsync(
263+
&self,
264+
ctx: &Context,
265+
inode: u64,
266+
datasync: bool,
267+
handle: u64,
268+
) -> io::Result<()> {
269+
enosys()
270+
}
271+
272+
async fn async_fallocate(
273+
&self,
274+
ctx: &Context,
275+
inode: u64,
276+
handle: u64,
277+
mode: u32,
278+
offset: u64,
279+
length: u64,
280+
) -> io::Result<()> {
281+
enosys()
282+
}
283+
284+
async fn async_fsyncdir(
285+
&self,
286+
ctx: &Context,
287+
inode: u64,
288+
datasync: bool,
289+
handle: u64,
290+
) -> io::Result<()> {
291+
enosys()
292+
}
293+
}
294+
295+
/// Serve `file` with `server` on a tokio runtime of its own until the task ends or `stop`
296+
/// fires, which drops the requests in flight.
297+
fn serve(
298+
file: File,
299+
buf_size: usize,
300+
server: Arc<Server<Arc<TestFs>>>,
301+
stop: oneshot::Receiver<()>,
302+
) -> thread::JoinHandle<()> {
303+
thread::spawn(move || {
304+
let rt = tokio::runtime::Builder::new_current_thread()
305+
.enable_all()
306+
.build()
307+
.unwrap();
308+
let state = Arc::new(AtomicBool::new(false));
309+
let mut task = FuseDevTask::new_with_max_inflight(buf_size, file, server, state, 4);
310+
rt.block_on(async {
311+
tokio::select! {
312+
_ = task.poll_handler() => {}
313+
_ = stop => {}
314+
}
315+
});
316+
})
317+
}
318+
319+
#[test]
320+
fn test_fuse_dev_task_concurrent_lookups_and_notify() {
321+
async_runtime::set_runtime_type(RuntimeType::Tokio).unwrap();
322+
323+
let mnt = TempDir::new().unwrap();
324+
let Some(mut se) = mount(mnt.as_path()) else {
325+
return;
326+
};
327+
let gate = Arc::new(Semaphore::new(0));
328+
let fs = Arc::new(TestFs {
329+
gate: Some(gate.clone()),
330+
..Default::default()
331+
});
332+
let server = Arc::new(Server::new(fs.clone()));
333+
let (_stop_tx, stop) = oneshot::channel();
334+
let handle = serve(
335+
se.clone_fuse_file().unwrap(),
336+
se.bufsize(),
337+
server.clone(),
338+
stop,
339+
);
340+
341+
let slow = stat_in_thread(mnt.as_path().join("slow"));
342+
thread::sleep(Duration::from_millis(100));
343+
344+
// A lookup waiting in the file system doesn't hold up the others.
345+
use std::os::unix::fs::MetadataExt;
346+
let fast = std::fs::metadata(mnt.as_path().join("fast")).unwrap();
347+
assert_eq!(fast.ino(), FAST);
348+
assert!(slow.try_recv().is_err());
349+
gate.add_permits(1);
350+
assert_eq!(slow.recv_timeout(Duration::from_secs(5)).unwrap(), SLOW);
351+
352+
// Invalidate the cached entry while the task serves, so the next stat looks it up again.
353+
assert_eq!(fs.fast_lookups.load(Ordering::Relaxed), 1);
354+
let notify_file = se.clone_fuse_file().unwrap();
355+
let mut buf = vec![0u8; 0x1000];
356+
let writer = FuseDevWriter::<()>::new(notify_file.as_raw_fd(), &mut buf).unwrap();
357+
server
358+
.notify_inval_entry(
359+
writer,
360+
ROOT_ID,
361+
CStr::from_bytes_with_nul(b"fast\0").unwrap(),
362+
)
363+
.unwrap();
364+
std::fs::metadata(mnt.as_path().join("fast")).unwrap();
365+
assert_eq!(fs.fast_lookups.load(Ordering::Relaxed), 2);
366+
367+
se.umount().unwrap();
368+
handle.join().unwrap();
369+
}
370+
371+
#[test]
372+
fn test_fuse_dev_task_takeover() {
373+
async_runtime::set_runtime_type(RuntimeType::Tokio).unwrap();
374+
375+
let mnt = TempDir::new().unwrap();
376+
let Some(mut se) = mount(mnt.as_path()) else {
377+
return;
378+
};
379+
let buf_size = se.bufsize();
380+
381+
// The old server stalls on `slow` and stops with the lookup in flight.
382+
let old_fs = Arc::new(TestFs {
383+
stall: true,
384+
..Default::default()
385+
});
386+
let old = Arc::new(Server::new(old_fs.clone()));
387+
// The kernel ends the requests read from a file description when it is closed, so the
388+
// test keeps the one the old server reads from, as a takeover passes it to the new one.
389+
let file = se.clone_fuse_file().unwrap();
390+
let (stop_tx, stop) = oneshot::channel();
391+
let old_handle = serve(file.try_clone().unwrap(), buf_size, old.clone(), stop);
392+
393+
std::fs::metadata(mnt.as_path().join("fast")).unwrap();
394+
if !old_fs.has_resend.load(Ordering::Relaxed) {
395+
eprintln!("kernel without FUSE_NOTIFY_RESEND, skipping");
396+
se.umount().unwrap();
397+
drop(stop_tx);
398+
old_handle.join().unwrap();
399+
return;
400+
}
401+
let slow = stat_in_thread(mnt.as_path().join("slow"));
402+
thread::sleep(Duration::from_millis(100));
403+
stop_tx.send(()).unwrap();
404+
old_handle.join().unwrap();
405+
let (version, options) = old.init_state();
406+
assert!(options.contains(FsOptions::HAS_RESEND));
407+
408+
// The new server serves the same connection without INIT and gets the lookup again.
409+
let new = Arc::new(Server::new(Arc::new(TestFs::default())));
410+
new.restore_init(version, options);
411+
let (_stop_tx, stop) = oneshot::channel();
412+
let new_handle = serve(file.try_clone().unwrap(), buf_size, new.clone(), stop);
413+
let mut buf = vec![0u8; 0x1000];
414+
let writer = FuseDevWriter::<()>::new(file.as_raw_fd(), &mut buf).unwrap();
415+
new.notify_resend(writer).unwrap();
416+
assert_eq!(slow.recv_timeout(Duration::from_secs(5)).unwrap(), SLOW);
417+
418+
se.umount().unwrap();
419+
new_handle.join().unwrap();
420+
}
86421
}

0 commit comments

Comments
 (0)