objstorage: cache local object paths

Local object operations repeatedly reconstructed the same filesystem path through base.MakeFilepath because object metadata stored only the file number, file type, and storage tier rather than the complete path. In a focused benchmark, this path construction took approximately 247–256 ns/op and allocated 112 bytes across four allocations per call.

Store the eagerly computed local path in ObjectMetadata.Local.Path. Populate the path when existing hot- and cold-tier objects are discovered at startup, when a new local object is created, and when an object is created through LinkOrCopyFromLocal.

Local path lookup, opening, sizing, and removal now reuse the stored path. Provider.Path retains a fallback for metadata without a populated path.

A focused post-change benchmark runs at approximately 4.13–4.27 ns/op with 0 B/op and 0 allocs/op.

Tests cover newly created hot- and cold-tier objects, startup-discovered objects, reuse of the cached path across local operations, and the fallback path.

Testing:
- go test ./...
- go test -race ./objstorage/objstorageprovider

Fixes #5340.

Author: srinathshrestha

objstorage/objstorage.go

@@ -142,6 +142,10 @@ type ObjectMetadata struct {
 	// The fields below are only set if the object is on local storage.
 	Local struct {
 		Tier base.StorageTier
+		// Path is the object's path within the filesystem for Tier (the hot
+		// tier FS or the cold tier FS). It is populated by the provider for all
+		// local objects it knows about; callers should use Provider.Path.
+		Path string
 	}
 
 	// The fields below are only set if the object is on remote storage.
@@ -200,6 +204,9 @@ func (meta *ObjectMetadata) Placement() base.Placement {
 // AssertValid checks that the metadata is sane.
 func (meta *ObjectMetadata) AssertValid() {
 	if !meta.IsRemote() {
+		if meta.Local.Path == "" {
+			panic(errors.AssertionFailedf("meta.Local.Path not set"))
+		}
 		// Verify all Remote fields are empty.
 		if meta.Remote != (ObjectMetadata{}).Remote {
 			panic(errors.AssertionFailedf("meta.Remote not empty: %#v", errors.Safe(meta.Remote)))

objstorage/objstorageprovider/path_test.go

@@ -0,0 +1,114 @@
+// Copyright 2026 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"testing"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+	"github.com/stretchr/testify/require"
+)
+
+type pathJoinCountingFS struct {
+	vfs.FS
+	pathJoins int
+}
+
+func (f *pathJoinCountingFS) PathJoin(elem ...string) string {
+	f.pathJoins++
+	return f.FS.PathJoin(elem...)
+}
+
+func TestLocalObjectPath(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("created", func(t *testing.T) {
+		mem := vfs.NewMem()
+		fs := &pathJoinCountingFS{FS: mem}
+		require.NoError(t, fs.MkdirAll("hot", 0755))
+		require.NoError(t, fs.MkdirAll("cold", 0755))
+
+		settings := DefaultSettings(fs, "hot")
+		settings.Local.ColdTier.FS = fs
+		settings.Local.ColdTier.FSDirName = "cold"
+		p, err := Open(settings)
+		require.NoError(t, err)
+		defer func() { require.NoError(t, p.Close()) }()
+
+		for _, tc := range []struct {
+			fileType base.FileType
+			fileNum  base.DiskFileNum
+			tier     base.StorageTier
+			dir      string
+		}{
+			{fileType: base.FileTypeTable, fileNum: 1, tier: base.HotTier, dir: "hot"},
+			{fileType: base.FileTypeBlob, fileNum: 2, tier: base.ColdTier, dir: "cold"},
+		} {
+			w, meta, err := p.Create(ctx, tc.fileType, tc.fileNum, objstorage.CreateOptions{Tier: tc.tier})
+			require.NoError(t, err)
+			require.NoError(t, w.Finish())
+			require.Equal(t, base.MakeFilepath(mem, tc.dir, tc.fileType, tc.fileNum), meta.Local.Path)
+
+			fs.pathJoins = 0
+			require.Equal(t, meta.Local.Path, p.Path(meta))
+			_, err = p.Size(meta)
+			require.NoError(t, err)
+			r, err := p.OpenForReading(ctx, tc.fileType, tc.fileNum, objstorage.OpenOptions{})
+			require.NoError(t, err)
+			require.NoError(t, r.Close())
+			require.NoError(t, p.Remove(tc.fileType, tc.fileNum))
+			require.Zero(t, fs.pathJoins)
+		}
+	})
+
+	t.Run("discovered", func(t *testing.T) {
+		mem := vfs.NewMem()
+		require.NoError(t, mem.MkdirAll("db", 0755))
+		path := base.MakeFilepath(mem, "db", base.FileTypeTable, 1)
+		file, err := mem.Create(path, vfs.WriteCategoryUnspecified)
+		require.NoError(t, err)
+		require.NoError(t, file.Close())
+
+		fs := &pathJoinCountingFS{FS: mem}
+		p, err := Open(DefaultSettings(fs, "db"))
+		require.NoError(t, err)
+		defer func() { require.NoError(t, p.Close()) }()
+
+		meta, err := p.Lookup(base.FileTypeTable, 1)
+		require.NoError(t, err)
+		require.Equal(t, path, meta.Local.Path)
+
+		fs.pathJoins = 0
+		require.Equal(t, path, p.Path(meta))
+		_, err = p.Size(meta)
+		require.NoError(t, err)
+		r, err := p.OpenForReading(ctx, base.FileTypeTable, 1, objstorage.OpenOptions{})
+		require.NoError(t, err)
+		require.NoError(t, r.Close())
+		require.NoError(t, p.Remove(base.FileTypeTable, 1))
+		require.Zero(t, fs.pathJoins)
+	})
+
+	t.Run("fallback", func(t *testing.T) {
+		mem := vfs.NewMem()
+		fs := &pathJoinCountingFS{FS: mem}
+		require.NoError(t, fs.MkdirAll("db", 0755))
+		p, err := Open(DefaultSettings(fs, "db"))
+		require.NoError(t, err)
+		defer func() { require.NoError(t, p.Close()) }()
+
+		// reportCorruption passes bare metadata when a local object is missing.
+		meta := objstorage.ObjectMetadata{
+			DiskFileNum: 1,
+			FileType:    base.FileTypeTable,
+		}
+		fs.pathJoins = 0
+		require.Equal(t, base.MakeFilepath(mem, "db", base.FileTypeTable, 1), p.Path(meta))
+		require.NotZero(t, fs.pathJoins)
+	})
+}

objstorage/objstorageprovider/provider.go

@@ -286,7 +286,7 @@ func (p *provider) OpenForReading(
 
 	var r objstorage.Readable
 	if !meta.IsRemote() {
-		r, err = p.localOpenForReading(ctx, fileType, fileNum, meta.Local.Tier, opts)
+		r, err = p.localOpenForReading(ctx, meta, opts)
 	} else {
 		r, err = p.remoteOpenForReading(ctx, meta, opts)
 		if err != nil && p.isNotExistError(meta, err) {
@@ -354,7 +354,7 @@ func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) erro
 	}
 
 	if !meta.IsRemote() {
-		err = p.localRemove(fileType, fileNum, meta.Local.Tier)
+		err = p.localRemove(meta)
 	} else {
 		// TODO(radu): implement remote object removal (i.e. deref).
 		err = p.sharedUnref(meta)
@@ -436,6 +436,7 @@ func (p *provider) LinkOrCopyFromLocal(
 				FileType:    dstFileType,
 			}
 			meta.Local.Tier = tier
+			meta.Local.Path = dstPath
 			p.addMetadata(meta)
 			return meta, nil
 		}
@@ -501,6 +502,12 @@ func (p *provider) Lookup(
 // Path is part of the objstorage.Provider interface.
 func (p *provider) Path(meta objstorage.ObjectMetadata) string {
 	if !meta.IsRemote() {
+		// Local.Path is set for all objects known to the provider. Callers may
+		// pass bare metadata for an object that isn't known (e.g. reportCorruption
+		// for a missing local file); reconstruct the path in that case.
+		if meta.Local.Path != "" {
+			return meta.Local.Path
+		}
 		_, path := p.localPath(meta.FileType, meta.DiskFileNum, meta.Local.Tier)
 		return path
 	}
@@ -510,7 +517,7 @@ func (p *provider) Path(meta objstorage.ObjectMetadata) string {
 // Size returns the size of the object.
 func (p *provider) Size(meta objstorage.ObjectMetadata) (int64, error) {
 	if !meta.IsRemote() {
-		return p.localSize(meta.FileType, meta.DiskFileNum, meta.Local.Tier)
+		return p.localSize(meta)
 	}
 	return p.remoteSize(meta)
 }

objstorage/objstorageprovider/vfs.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/vfs"
 )
@@ -73,10 +74,24 @@ func (ls *localLockedState) objChanged(meta objstorage.ObjectMetadata) {
 func (p *provider) localPath(
 	fileType base.FileType, fileNum base.DiskFileNum, tier base.StorageTier,
 ) (vfs.FS, string) {
+	fs, dir := p.localTierFS(tier)
+	return fs, base.MakeFilepath(fs, dir, fileType, fileNum)
+}
+
+// localFS returns the filesystem backing the given tier. It falls back to the
+// hot tier when no cold tier filesystem is configured.
+func (p *provider) localFS(tier base.StorageTier) vfs.FS {
+	fs, _ := p.localTierFS(tier)
+	return fs
+}
+
+// localTierFS returns the filesystem and directory backing the given tier. It
+// falls back to the hot tier when no cold tier filesystem is configured.
+func (p *provider) localTierFS(tier base.StorageTier) (vfs.FS, string) {
 	if coldFS := p.st.Local.ColdTier.FS; tier == base.ColdTier && coldFS != nil {
-		return coldFS, base.MakeFilepath(coldFS, p.st.Local.ColdTier.FSDirName, fileType, fileNum)
+		return coldFS, p.st.Local.ColdTier.FSDirName
 	}
-	return p.st.Local.FS, base.MakeFilepath(p.st.Local.FS, p.st.Local.FSDirName, fileType, fileNum)
+	return p.st.Local.FS, p.st.Local.FSDirName
 }
 
 // metaFileType returns the file type for a file that contains only the metadata
@@ -119,17 +134,14 @@ func offsetFromMetaPath(filename string) (startOffset int64, ok bool) {
 }
 
 func (p *provider) localOpenForReading(
-	ctx context.Context,
-	fileType base.FileType,
-	fileNum base.DiskFileNum,
-	tier base.StorageTier,
-	opts objstorage.OpenOptions,
+	ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions,
 ) (objstorage.Readable, error) {
-	fs, filename := p.localPath(fileType, fileNum, tier)
+	fs := p.localFS(meta.Local.Tier)
+	filename := meta.Local.Path
 	file, err := fs.Open(filename, vfs.RandomReadsOption)
 	if err != nil {
 		if opts.MustExist && p.IsNotExistError(err) {
-			err = base.AddDetailsToNotExistError(p.st.Local.FS, filename, err)
+			err = base.AddDetailsToNotExistError(fs, filename, err)
 			err = base.MarkCorruptionError(err)
 		}
 		return nil, err
@@ -138,9 +150,11 @@ func (p *provider) localOpenForReading(
 	if err != nil {
 		return nil, err
 	}
-	if tier == base.ColdTier {
-		if startOffset, ok := p.getColdObjectMetaFile(fileType, fileNum); ok {
-			metaPath := p.metaPath(fileType, fileNum, startOffset)
+	if meta.Local.Tier == base.ColdTier {
+		if startOffset, ok := p.getColdObjectMetaFile(meta.FileType, meta.DiskFileNum); ok {
+			// TODO: Store this path in metaFileInfo to avoid reconstructing it on
+			// every open and removal of a cold-tier object.
+			metaPath := p.metaPath(meta.FileType, meta.DiskFileNum, startOffset)
 			return newColdReadableWithHotMeta(r, p.st.Local.FS, metaPath, startOffset), nil
 		}
 	}
@@ -172,22 +186,21 @@ func (p *provider) vfsCreate(
 		FileType:    fileType,
 	}
 	meta.Local.Tier = tier
+	meta.Local.Path = filename
 	w := objstorage.Writable(newFileBufferedWritable(file))
 	if tier == base.ColdTier {
 		w = newColdWritable(p, fileType, fileNum, w, category)
 	}
 	return w, meta, nil
 }
 
-func (p *provider) localRemove(
-	fileType base.FileType, fileNum base.DiskFileNum, tier base.StorageTier,
-) error {
-	fs, path := p.localPath(fileType, fileNum, tier)
-	err := p.st.Local.FSCleaner.Clean(fs, fileType, path)
-	if tier == base.ColdTier {
-		if startOffset, ok := p.popColdObjectMetaFile(fileType, fileNum); ok {
-			metaPath := p.metaPath(fileType, fileNum, startOffset)
-			metaFileType := metaFileType(fileType)
+func (p *provider) localRemove(meta objstorage.ObjectMetadata) error {
+	fs := p.localFS(meta.Local.Tier)
+	err := p.st.Local.FSCleaner.Clean(fs, meta.FileType, meta.Local.Path)
+	if meta.Local.Tier == base.ColdTier {
+		if startOffset, ok := p.popColdObjectMetaFile(meta.FileType, meta.DiskFileNum); ok {
+			metaPath := p.metaPath(meta.FileType, meta.DiskFileNum, startOffset)
+			metaFileType := metaFileType(meta.FileType)
 			err = firstError(err, p.st.Local.FSCleaner.Clean(p.st.Local.FS, metaFileType, metaPath))
 		}
 	}
@@ -219,6 +232,10 @@ func (p *provider) localInit() error {
 					DiskFileNum: fileNum,
 				}
 				o.Local.Tier = base.HotTier
+				_, o.Local.Path = p.localPath(o.FileType, o.DiskFileNum, o.Local.Tier)
+				if invariants.Enabled {
+					o.AssertValid()
+				}
 				p.mu.knownObjects[o.DiskFileNum] = o
 			}
 		}
@@ -246,6 +263,10 @@ func (p *provider) localInit() error {
 						DiskFileNum: fileNum,
 					}
 					o.Local.Tier = base.ColdTier
+					_, o.Local.Path = p.localPath(o.FileType, o.DiskFileNum, o.Local.Tier)
+					if invariants.Enabled {
+						o.AssertValid()
+					}
 					if _, exists := p.mu.knownObjects[o.DiskFileNum]; exists {
 						p.st.Logger.Errorf("object %s exists on both tiers; using hot tier version", o.DiskFileNum)
 					} else {
@@ -327,11 +348,9 @@ func (p *provider) localSync() error {
 	return nil
 }
 
-func (p *provider) localSize(
-	fileType base.FileType, fileNum base.DiskFileNum, tier base.StorageTier,
-) (int64, error) {
-	fs, filename := p.localPath(fileType, fileNum, tier)
-	stat, err := fs.Stat(filename)
+func (p *provider) localSize(meta objstorage.ObjectMetadata) (int64, error) {
+	fs := p.localFS(meta.Local.Tier)
+	stat, err := fs.Stat(meta.Local.Path)
 	if err != nil {
 		return 0, err
 	}