Use a S3FIFO cache

.gitignore

@@ -18,8 +18,7 @@ compose-dev.yaml
 .LSOverride
 
 # Icon must end with two \r
-Icon
-
+Icon
 
 # Thumbnails
 ._*

storage/Cargo.toml

@@ -24,6 +24,7 @@ bitfield = "0.18.1"
 fastrace = { version = "0.7.4" }
 io-uring = { version = "0.7.4", optional = true }
 triomphe = "0.1.14"
+s3-fifo = { git = "https://github.com/rkuris/s3-fifo", branch = "main" }
 
 [dev-dependencies]
 rand = "0.9.0"

storage/src/linear/filebacked.rs

@@ -11,10 +11,11 @@ use std::io::{Error, Read};
 use std::num::NonZero;
 use std::os::unix::fs::FileExt;
 use std::path::PathBuf;
-use std::sync::Mutex;
+use std::sync::{Mutex, RwLock};
 
 use lru::LruCache;
 use metrics::counter;
+use s3_fifo::S3FIFO;
 
 use crate::{CacheReadStrategy, LinearAddress, SharedNode};
 
@@ -23,7 +24,7 @@ use super::{ReadableStorage, WritableStorage};
 /// A [ReadableStorage] backed by a file
 pub struct FileBacked {
     fd: File,
-    cache: Mutex<LruCache<LinearAddress, SharedNode>>,
+    cache: RwLock<S3FIFO<LinearAddress, SharedNode>>,
     free_list_cache: Mutex<LruCache<LinearAddress, Option<LinearAddress>>>,
     cache_read_strategy: CacheReadStrategy,
     #[cfg(feature = "io-uring")]
@@ -95,7 +96,7 @@ impl FileBacked {
 
         Ok(Self {
             fd,
-            cache: Mutex::new(LruCache::new(node_cache_size)),
+            cache: RwLock::new(S3FIFO::new(node_cache_size.into())),
             free_list_cache: Mutex::new(LruCache::new(free_list_cache_size)),
             cache_read_strategy,
             #[cfg(feature = "io-uring")]
@@ -114,7 +115,7 @@ impl ReadableStorage for FileBacked {
     }
 
     fn read_cached_node(&self, addr: LinearAddress, mode: &'static str) -> Option<SharedNode> {
-        let mut guard = self.cache.lock().expect("poisoned lock");
+        let guard = self.cache.read().expect("poisoned lock");
         let cached = guard.get(&addr).cloned();
         counter!("firewood.cache.node", "mode" => mode, "type" => if cached.is_some() { "hit" } else { "miss" })
             .increment(1);
@@ -138,12 +139,12 @@ impl ReadableStorage for FileBacked {
                 // we don't cache reads
             }
             CacheReadStrategy::All => {
-                let mut guard = self.cache.lock().expect("poisoned lock");
+                let mut guard = self.cache.write().expect("poisoned lock");
                 guard.put(addr, node);
             }
             CacheReadStrategy::BranchReads => {
                 if !node.is_leaf() {
-                    let mut guard = self.cache.lock().expect("poisoned lock");
+                    let mut guard = self.cache.write().expect("poisoned lock");
                     guard.put(addr, node);
                 }
             }
@@ -160,18 +161,15 @@ impl WritableStorage for FileBacked {
         &self,
         nodes: impl Iterator<Item = (&'a std::num::NonZero<u64>, &'a SharedNode)>,
     ) -> Result<(), Error> {
-        let mut guard = self.cache.lock().expect("poisoned lock");
+        let mut guard = self.cache.write().expect("poisoned lock");
         for (addr, node) in nodes {
             guard.put(*addr, node.clone());
         }
         Ok(())
     }
 
-    fn invalidate_cached_nodes<'a>(&self, addresses: impl Iterator<Item = &'a LinearAddress>) {
-        let mut guard = self.cache.lock().expect("poisoned lock");
-        for addr in addresses {
-            guard.pop(addr);
-        }
+    fn invalidate_cached_nodes<'a>(&self, _addresses: impl Iterator<Item = &'a LinearAddress>) {
+        // TODO: We might be able to implement this, but currently S3FIFO doesn't support it
     }
 
     fn add_to_free_list_cache(&self, addr: LinearAddress, next: Option<LinearAddress>) {