From 483c8f27574ed63678c3113763bfc8bf48438650 Mon Sep 17 00:00:00 2001 From: Prince Kumar Date: Thu, 9 May 2024 09:07:46 +0000 Subject: [PATCH 1/5] ReadDir response cache POC --- conversions.go | 8 ++++++++ fuseops/ops.go | 6 ++++++ internal/fusekernel/fuse_kernel.go | 2 ++ 3 files changed, 16 insertions(+) diff --git a/conversions.go b/conversions.go index 422d5481..f1bdbe8e 100644 --- a/conversions.go +++ b/conversions.go @@ -853,6 +853,14 @@ func (c *Connection) kernelResponseForOp( out := (*fusekernel.OpenOut)(m.Grow(int(unsafe.Sizeof(fusekernel.OpenOut{})))) out.Fh = uint64(o.Handle) + if o.KeepDirectoryContentsPageCache { + out.OpenFlags |= uint32(fusekernel.OpenCacheDir) + } + + if o.KeepDirectoryContentsPageCache { + out.OpenFlags |= uint32(fusekernel.OpenKeepCache) + } + case *fuseops.ReadDirOp: // convertInMessage already set up the destination buffer to be at the end // of the out message. We need only shrink to the right size based on how diff --git a/fuseops/ops.go b/fuseops/ops.go index 0cbb202c..11a0820c 100644 --- a/fuseops/ops.go +++ b/fuseops/ops.go @@ -500,6 +500,12 @@ type OpenDirOp struct { // a later call to ReleaseDirHandle. Handle HandleID OpContext OpContext + + // Convey the kernel to cache the directory contents in the page cache. + CacheDirContentsAsPageCache bool + + // Either to keep previous directory contents page cache or not. + KeepDirectoryContentsPageCache bool } // Read entries from a directory previously opened with OpenDir. diff --git a/internal/fusekernel/fuse_kernel.go b/internal/fusekernel/fuse_kernel.go index c1aded97..3c922f57 100644 --- a/internal/fusekernel/fuse_kernel.go +++ b/internal/fusekernel/fuse_kernel.go @@ -227,6 +227,7 @@ const ( OpenDirectIO OpenResponseFlags = 1 << 0 // bypass page cache for this open file OpenKeepCache OpenResponseFlags = 1 << 1 // don't invalidate the data cache on open OpenNonSeekable OpenResponseFlags = 1 << 2 // mark the file as non-seekable (not supported on OS X) + OpenCacheDir OpenResponseFlags = 1 << 3 // allow caching this directory OpenPurgeAttr OpenResponseFlags = 1 << 30 // OS X OpenPurgeUBC OpenResponseFlags = 1 << 31 // OS X @@ -240,6 +241,7 @@ var openResponseFlagNames = []flagName{ {uint32(OpenDirectIO), "OpenDirectIO"}, {uint32(OpenKeepCache), "OpenKeepCache"}, {uint32(OpenNonSeekable), "OpenNonSeekable"}, + {uint32(OpenCacheDir), "OpenCacheDir"}, {uint32(OpenPurgeAttr), "OpenPurgeAttr"}, {uint32(OpenPurgeUBC), "OpenPurgeUBC"}, } From 626390a0a78958d4386939bcfeb416402622b032 Mon Sep 17 00:00:00 2001 From: Prince Kumar Date: Thu, 9 May 2024 10:49:37 +0000 Subject: [PATCH 2/5] Renaming the variable --- conversions.go | 4 ++-- fuseops/ops.go | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/conversions.go b/conversions.go index f1bdbe8e..22c03416 100644 --- a/conversions.go +++ b/conversions.go @@ -853,11 +853,11 @@ func (c *Connection) kernelResponseForOp( out := (*fusekernel.OpenOut)(m.Grow(int(unsafe.Sizeof(fusekernel.OpenOut{})))) out.Fh = uint64(o.Handle) - if o.KeepDirectoryContentsPageCache { + if o.CacheDirContentAsPageCache { out.OpenFlags |= uint32(fusekernel.OpenCacheDir) } - if o.KeepDirectoryContentsPageCache { + if o.KeepDirContentPageCache { out.OpenFlags |= uint32(fusekernel.OpenKeepCache) } diff --git a/fuseops/ops.go b/fuseops/ops.go index 11a0820c..ba28668a 100644 --- a/fuseops/ops.go +++ b/fuseops/ops.go @@ -501,11 +501,13 @@ type OpenDirOp struct { Handle HandleID OpContext OpContext - // Convey the kernel to cache the directory contents in the page cache. - CacheDirContentsAsPageCache bool + // CacheDirContentAsPageCache conveys the kernel to cache the response of next + // ReadDirOp as page cache. Once cached, listing on that directory will be + // served from the kernel until invalidated. + CacheDirContentAsPageCache bool - // Either to keep previous directory contents page cache or not. - KeepDirectoryContentsPageCache bool + // KeepDirContentPageCache used to invalidate the cached directory content. + KeepDirContentPageCache bool } // Read entries from a directory previously opened with OpenDir. From 22d48706c06af2896111e40968ac4651e40288d8 Mon Sep 17 00:00:00 2001 From: Prince Kumar Date: Mon, 20 May 2024 16:43:25 +0000 Subject: [PATCH 3/5] Review comments --- conversions.go | 4 +- fuseops/ops.go | 8 +- samples/cachingfs/caching_fs.go | 89 ++++++++++ samples/cachingfs/caching_fs_test.go | 252 +++++++++++++++++++++++++++ 4 files changed, 347 insertions(+), 6 deletions(-) diff --git a/conversions.go b/conversions.go index 22c03416..ff4c0d69 100644 --- a/conversions.go +++ b/conversions.go @@ -853,11 +853,11 @@ func (c *Connection) kernelResponseForOp( out := (*fusekernel.OpenOut)(m.Grow(int(unsafe.Sizeof(fusekernel.OpenOut{})))) out.Fh = uint64(o.Handle) - if o.CacheDirContentAsPageCache { + if o.CacheDir { out.OpenFlags |= uint32(fusekernel.OpenCacheDir) } - if o.KeepDirContentPageCache { + if o.KeepCache { out.OpenFlags |= uint32(fusekernel.OpenKeepCache) } diff --git a/fuseops/ops.go b/fuseops/ops.go index ba28668a..474ad921 100644 --- a/fuseops/ops.go +++ b/fuseops/ops.go @@ -501,13 +501,13 @@ type OpenDirOp struct { Handle HandleID OpContext OpContext - // CacheDirContentAsPageCache conveys the kernel to cache the response of next + // CacheDir conveys the kernel to cache the response of next // ReadDirOp as page cache. Once cached, listing on that directory will be // served from the kernel until invalidated. - CacheDirContentAsPageCache bool + CacheDir bool - // KeepDirContentPageCache used to invalidate the cached directory content. - KeepDirContentPageCache bool + // KeepCache instructs the kernel to not invalidate the data cache on open calls. + KeepCache bool } // Read entries from a directory previously opened with OpenDir. diff --git a/samples/cachingfs/caching_fs.go b/samples/cachingfs/caching_fs.go index 4a43d70c..f5d18f8f 100644 --- a/samples/cachingfs/caching_fs.go +++ b/samples/cachingfs/caching_fs.go @@ -48,6 +48,11 @@ const ( // Each file responds to reads with random contents. SetKeepCache can be used // to control whether the response to OpenFileOp tells the kernel to keep the // file's data in the page cache or not. +// +// Each directory responds to readdir with random entries (different names). +// SetCacheDir and SetKeepDirCache can be used to control whether the response +// to OpenDirOp tells the kernel to cache the next response of ReadDirOp +// in cache, or invalidate the existing cached entry in page cache. type CachingFS interface { fuseutil.FileSystem @@ -66,6 +71,14 @@ type CachingFS interface { // Instruct the file system whether or not to reply to OpenFileOp with // FOPEN_KEEP_CACHE set. SetKeepCache(keep bool) + + // Instruct the file system whether or not to reply to OpenDirOp with + // FOPEN_KEEP_CACHE set. + SetKeepDirCache(keep bool) + + // Instruct the file system whether or not to reply to OpenDirOp with + // FOPEN_CACHE_DIR set. + SetCacheDir(cacheDir bool) } // Create a file system that issues cacheable responses according to the @@ -126,6 +139,10 @@ type cachingFS struct { // GUARDED_BY(mu) keepPageCache bool + // GUARDED_BY(mu) + keepDirCache bool + cacheDir bool + // The current ID of the lowest numbered non-root inode. // // INVARIANT: baseID > fuseops.RootInodeID @@ -202,6 +219,17 @@ func (fs *cachingFS) barAttrs() fuseops.InodeAttributes { } } +func randomString(len int) string { + bytes := make([]byte, len) + nn, err := rand.Read(bytes) + + if err != nil || len != nn { + return "" + } + + return string(bytes) +} + //////////////////////////////////////////////////////////////////////// // Public interface //////////////////////////////////////////////////////////////////////// @@ -254,6 +282,22 @@ func (fs *cachingFS) SetKeepCache(keep bool) { fs.keepPageCache = keep } +// LOCKS_EXCLUDED(fs.mu) +func (fs *cachingFS) SetKeepDirCache(keep bool) { + fs.mu.Lock() + defer fs.mu.Unlock() + + fs.keepDirCache = keep +} + +// LOCKS_EXCLUDED(fs.mu) +func (fs *cachingFS) SetCacheDir(cacheDir bool) { + fs.mu.Lock() + defer fs.mu.Unlock() + + fs.cacheDir = cacheDir +} + //////////////////////////////////////////////////////////////////////// // FileSystem methods //////////////////////////////////////////////////////////////////////// @@ -349,6 +393,51 @@ func (fs *cachingFS) GetInodeAttributes( func (fs *cachingFS) OpenDir( ctx context.Context, op *fuseops.OpenDirOp) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + op.CacheDir = fs.cacheDir + op.KeepCache = fs.keepDirCache + + return nil +} + +func (fs *cachingFS) ReadDir( + ctx context.Context, + op *fuseops.ReadDirOp) error { + + entries := []fuseutil.Dirent{ + { + Offset: 1, + Inode: 101, + Name: "rdir" + randomString(4), + Type: fuseutil.DT_Directory, + }, + { + Offset: 2, + Inode: 102, + Name: "rfoo" + randomString(4), + Type: fuseutil.DT_File, + }, + } + + // Grab the range of interest. + if op.Offset > fuseops.DirOffset(len(entries)) { + return fuse.EIO + } + + entries = entries[op.Offset:] + + // Resume at the specified offset into the array. + for _, e := range entries { + n := fuseutil.WriteDirent(op.Dst[op.BytesRead:], e) + if n == 0 { + break + } + + op.BytesRead += n + } + return nil } diff --git a/samples/cachingfs/caching_fs_test.go b/samples/cachingfs/caching_fs_test.go index abd8c462..4c0855ed 100644 --- a/samples/cachingfs/caching_fs_test.go +++ b/samples/cachingfs/caching_fs_test.go @@ -16,6 +16,7 @@ package cachingfs_test import ( "bytes" + "fmt" "io/ioutil" "os" "path" @@ -682,6 +683,7 @@ func (t *PageCacheTest) TwoFileHandles_NoKeepCache() { func (t *PageCacheTest) TwoFileHandles_KeepCache() { t.fs.SetKeepCache(true) + fmt.Print(t.Dir) // Open the file. f1, err := os.Open(path.Join(t.Dir, "foo")) @@ -723,3 +725,253 @@ func (t *PageCacheTest) TwoFileHandles_KeepCache() { ExpectTrue(bytes.Equal(c1, c3)) } + +//////////////////////////////////////////////////////////////////////// +// Dir cache +//////////////////////////////////////////////////////////////////////// + +type DirCacheTest struct { + cachingFSTest +} + +var _ SetUpInterface = &DirCacheTest{} + +func init() { RegisterTestSuite(&DirCacheTest{}) } + +func (t *DirCacheTest) SetUp(ti *TestInfo) { + const ( + lookupEntryTimeout = 0 + getattrTimeout = 0 + ) + + t.cachingFSTest.setUp(ti, lookupEntryTimeout, getattrTimeout) +} + +func (t *DirCacheTest) CacheDirAndKeepDirCache() { + t.fs.SetCacheDir(true) + t.fs.SetKeepDirCache(true) + + // First read, kernel will cache the dir response. + f, err := os.Open(path.Join(t.Dir, "dir")) + defer f.Close() // Make sure to close specially required in failure scenario. + AssertEq(nil, err) + + names1, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names1)) + + err = f.Close() + AssertEq(nil, err) + + // Second read, will be served from cache. + f, err = os.Open(path.Join(t.Dir, "dir")) + AssertEq(nil, err) + + names2, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names2)) + + err = f.Close() + AssertEq(nil, err) + + AssertEq(names1[0], names2[0]) + AssertEq(names1[1], names2[1]) +} + +func (t *DirCacheTest) NoCacheDirAndKeepDirCache() { + t.fs.SetCacheDir(false) + t.fs.SetKeepDirCache(true) + + // First read, no caching since NoCacheDir. + f, err := os.Open(path.Join(t.Dir, "dir")) + defer f.Close() // Make sure to close specially required in failure scenario. + AssertEq(nil, err) + + names1, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names1)) + + err = f.Close() + AssertEq(nil, err) + + // Second read, will be served from filesystem, hence different name. + f, err = os.Open(path.Join(t.Dir, "dir")) + AssertEq(nil, err) + + names2, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names2)) + + err = f.Close() + AssertEq(nil, err) + + AssertNe(names1[0], names2[0]) + AssertNe(names1[1], names2[1]) +} + +func (t *DirCacheTest) CacheDirAndNoKeepDirCache() { + t.fs.SetCacheDir(true) + t.fs.SetKeepDirCache(false) + + // First read, kernel will cache the dir response. + f, err := os.Open(path.Join(t.Dir, "dir")) + defer f.Close() // Make sure to close specially required in failure scenario. + AssertEq(nil, err) + + names1, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names1)) + + err = f.Close() + AssertEq(nil, err) + + // Second read, cached response will be invalidated since NoKeepDirCache. + // Hence, different names. + f, err = os.Open(path.Join(t.Dir, "dir")) + AssertEq(nil, err) + + names2, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names2)) + + err = f.Close() + AssertEq(nil, err) + + AssertNe(names1[0], names2[0]) + AssertNe(names1[1], names2[1]) +} + +func (t *DirCacheTest) NoCacheDirAndNoKeepDirCache() { + t.fs.SetCacheDir(false) + t.fs.SetKeepDirCache(false) + + // First read, no caching since NoCacheDir. + f, err := os.Open(path.Join(t.Dir, "dir")) + defer f.Close() // Make sure to close specially required in failure scenario. + AssertEq(nil, err) + + names1, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names1)) + + err = f.Close() + AssertEq(nil, err) + + // Second read, will be served from filesystem. + // Since NoCacheDir also NoKeepDirCache. Hence, different names. + f, err = os.Open(path.Join(t.Dir, "dir")) + AssertEq(nil, err) + + names2, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names2)) + + err = f.Close() + AssertEq(nil, err) + + AssertNe(names1[0], names2[0]) + AssertNe(names1[1], names2[1]) +} + +func (t *DirCacheTest) CacheDirWithChangingKeepCacheDir() { + t.fs.SetCacheDir(true) + t.fs.SetKeepDirCache(false) + + // First read, kernel will cache the dir response. + f, err := os.Open(path.Join(t.Dir, "dir")) + defer f.Close() // Make sure to close specially required in failure scenario. + AssertEq(nil, err) + + names1, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names1)) + + err = f.Close() + AssertEq(nil, err) + + // Cached response will not be invalidated. Hence, served from kernel cache. + t.fs.SetKeepDirCache(true) + // Second read, will be served from cache + f, err = os.Open(path.Join(t.Dir, "dir")) + AssertEq(nil, err) + + names2, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names2)) + + err = f.Close() + AssertEq(nil, err) + + AssertEq(names1[0], names2[0]) + AssertEq(names1[1], names2[1]) + + // Kernel has cached but invalidated due to NoKeepDirCache. + // Hence, third read will be served from filesystem. + t.fs.SetKeepDirCache(false) + + // Third read, will be served from filesystem. So, names will be different. + f, err = os.Open(path.Join(t.Dir, "dir")) + AssertEq(nil, err) + + names3, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names3)) + + err = f.Close() + AssertEq(nil, err) + + AssertNe(names2[0], names3[0]) + AssertNe(names2[1], names3[1]) +} + +func (t *DirCacheTest) ChangingCacheDirWithKeepCacheDir() { + t.fs.SetCacheDir(true) + t.fs.SetKeepDirCache(true) + + // First read, kernel will cache the dir response. + f, err := os.Open(path.Join(t.Dir, "dir")) + defer f.Close() // Make sure to close specially required in failure scenario. + AssertEq(nil, err) + + names1, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names1)) + + err = f.Close() + AssertEq(nil, err) + + // Cached response will not be invalidated, but also not be served from cache. + // Since NoCacheDir so names will be different. + t.fs.SetCacheDir(false) + // Second read, will be served from filesystem. + f, err = os.Open(path.Join(t.Dir, "dir")) + AssertEq(nil, err) + + names2, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names2)) + + err = f.Close() + AssertEq(nil, err) + + AssertNe(names1[0], names2[0]) + AssertNe(names1[1], names2[1]) + + // Third read will be served from cache. + // But first read response is cached, since KeepCacheDir. + t.fs.SetCacheDir(true) + + // Third read, will be served from filesystem. So, names will be different. + f, err = os.Open(path.Join(t.Dir, "dir")) + AssertEq(nil, err) + + names3, err := f.Readdirnames(-1) + AssertEq(nil, err) + AssertEq(2, len(names3)) + + err = f.Close() + AssertEq(nil, err) + + AssertEq(names1[0], names3[0]) + AssertEq(names1[1], names3[1]) +} From eac919bb043465a1c7ec141378dfc85a22345129 Mon Sep 17 00:00:00 2001 From: Prince Kumar Date: Tue, 21 May 2024 05:36:09 +0000 Subject: [PATCH 4/5] minor rename --- samples/cachingfs/caching_fs_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/cachingfs/caching_fs_test.go b/samples/cachingfs/caching_fs_test.go index 4c0855ed..fea785cc 100644 --- a/samples/cachingfs/caching_fs_test.go +++ b/samples/cachingfs/caching_fs_test.go @@ -873,7 +873,7 @@ func (t *DirCacheTest) NoCacheDirAndNoKeepDirCache() { AssertNe(names1[1], names2[1]) } -func (t *DirCacheTest) CacheDirWithChangingKeepCacheDir() { +func (t *DirCacheTest) CacheDirWithChangingKeepDirCache() { t.fs.SetCacheDir(true) t.fs.SetKeepDirCache(false) @@ -924,7 +924,7 @@ func (t *DirCacheTest) CacheDirWithChangingKeepCacheDir() { AssertNe(names2[1], names3[1]) } -func (t *DirCacheTest) ChangingCacheDirWithKeepCacheDir() { +func (t *DirCacheTest) ChangingCacheDirWithKeepDirCache() { t.fs.SetCacheDir(true) t.fs.SetKeepDirCache(true) @@ -958,7 +958,7 @@ func (t *DirCacheTest) ChangingCacheDirWithKeepCacheDir() { AssertNe(names1[1], names2[1]) // Third read will be served from cache. - // But first read response is cached, since KeepCacheDir. + // But first read response is cached, since KeepDirCache. t.fs.SetCacheDir(true) // Third read, will be served from filesystem. So, names will be different. From ce29edb8c2f5190799db48db9b1ec47192866671 Mon Sep 17 00:00:00 2001 From: Prince Kumar Date: Wed, 22 May 2024 09:02:47 +0000 Subject: [PATCH 5/5] incorporating review comments --- fuseops/ops.go | 2 +- samples/cachingfs/caching_fs.go | 2 +- samples/cachingfs/caching_fs_test.go | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/fuseops/ops.go b/fuseops/ops.go index 474ad921..bcfb409d 100644 --- a/fuseops/ops.go +++ b/fuseops/ops.go @@ -501,7 +501,7 @@ type OpenDirOp struct { Handle HandleID OpContext OpContext - // CacheDir conveys the kernel to cache the response of next + // CacheDir conveys to the kernel to cache the response of next // ReadDirOp as page cache. Once cached, listing on that directory will be // served from the kernel until invalidated. CacheDir bool diff --git a/samples/cachingfs/caching_fs.go b/samples/cachingfs/caching_fs.go index f5d18f8f..15cfa174 100644 --- a/samples/cachingfs/caching_fs.go +++ b/samples/cachingfs/caching_fs.go @@ -50,7 +50,7 @@ const ( // file's data in the page cache or not. // // Each directory responds to readdir with random entries (different names). -// SetCacheDir and SetKeepDirCache can be used to control whether the response +// SetCacheDir and SetKeepDirCache can be used to control whether the response // to OpenDirOp tells the kernel to cache the next response of ReadDirOp // in cache, or invalidate the existing cached entry in page cache. type CachingFS interface { diff --git a/samples/cachingfs/caching_fs_test.go b/samples/cachingfs/caching_fs_test.go index fea785cc..414ba813 100644 --- a/samples/cachingfs/caching_fs_test.go +++ b/samples/cachingfs/caching_fs_test.go @@ -16,7 +16,6 @@ package cachingfs_test import ( "bytes" - "fmt" "io/ioutil" "os" "path" @@ -683,7 +682,6 @@ func (t *PageCacheTest) TwoFileHandles_NoKeepCache() { func (t *PageCacheTest) TwoFileHandles_KeepCache() { t.fs.SetKeepCache(true) - fmt.Print(t.Dir) // Open the file. f1, err := os.Open(path.Join(t.Dir, "foo"))