diff --git a/compaction.go b/compaction.go index b39ab15d1c..03798a1872 100644 --- a/compaction.go +++ b/compaction.go @@ -2320,7 +2320,7 @@ func (d *DB) cleanupVersionEdit(ve *versionEdit) { // Add this file to zombie tables as well, as the versionSet // asserts on whether every obsolete file was at one point // marked zombie. - d.mu.versions.zombieTables[obsoleteFiles[i].DiskFileNum] = tableInfo{ + d.mu.versions.zombieTables[obsoleteFiles[i].DiskFileNum] = objectInfo{ fileInfo: fileInfo{ FileNum: obsoleteFiles[i].DiskFileNum, FileSize: obsoleteFiles[i].Size, @@ -2936,7 +2936,7 @@ func (d *DB) runCompaction( // Add this file to zombie tables as well, as the versionSet // asserts on whether every obsolete file was at one point // marked zombie. - d.mu.versions.zombieTables[backing.DiskFileNum] = tableInfo{ + d.mu.versions.zombieTables[backing.DiskFileNum] = objectInfo{ fileInfo: fileInfo{ FileNum: backing.DiskFileNum, FileSize: backing.Size, diff --git a/internal/base/cleaner.go b/internal/base/cleaner.go index 1582ac80b0..e03b4e6597 100644 --- a/internal/base/cleaner.go +++ b/internal/base/cleaner.go @@ -40,7 +40,7 @@ var _ NeedsFileContents = ArchiveCleaner{} // also write to the secondary. We should consider archiving to the primary. func (ArchiveCleaner) Clean(fs vfs.FS, fileType FileType, path string) error { switch fileType { - case FileTypeLog, FileTypeManifest, FileTypeTable: + case FileTypeLog, FileTypeManifest, FileTypeTable, FileTypeBlob: destDir := fs.PathJoin(fs.PathDir(path), "archive") if err := fs.MkdirAll(destDir, 0755); err != nil { diff --git a/internal/base/filenames.go b/internal/base/filenames.go index 3b75d6c5d3..01051cc7bc 100644 --- a/internal/base/filenames.go +++ b/internal/base/filenames.go @@ -61,8 +61,44 @@ const ( FileTypeOptions FileTypeOldTemp FileTypeTemp + FileTypeBlob ) +var fileTypeStrings = [...]string{ + FileTypeLog: "log", + FileTypeLock: "lock", + FileTypeTable: "sstable", + FileTypeManifest: "manifest", + FileTypeOptions: "options", + FileTypeOldTemp: "old-temp", + FileTypeTemp: "temp", + FileTypeBlob: "blob", +} + +// FileTypeFromName parses a FileType from its string representation. +func FileTypeFromName(name string) FileType { + for i, s := range fileTypeStrings { + if s == name { + return FileType(i) + } + } + panic(fmt.Sprintf("unknown file type: %q", name)) +} + +// SafeFormat implements redact.SafeFormatter. +func (ft FileType) SafeFormat(w redact.SafePrinter, _ rune) { + if ft < 0 || int(ft) >= len(fileTypeStrings) { + w.Print(redact.SafeString("unknown")) + return + } + w.Print(redact.SafeString(fileTypeStrings[ft])) +} + +// String implements fmt.Stringer. +func (ft FileType) String() string { + return redact.StringWithoutMarkers(ft) +} + // MakeFilename builds a filename from components. func MakeFilename(fileType FileType, dfn DiskFileNum) string { switch fileType { @@ -80,6 +116,8 @@ func MakeFilename(fileType FileType, dfn DiskFileNum) string { return fmt.Sprintf("CURRENT.%s.dbtmp", dfn) case FileTypeTemp: return fmt.Sprintf("temporary.%s.dbtmp", dfn) + case FileTypeBlob: + return fmt.Sprintf("%s.blob", dfn) } panic("unreachable") } @@ -130,10 +168,11 @@ func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileN if !ok { break } - // TODO(sumeer): stop handling FileTypeLog in this function. switch filename[i+1:] { case "sst": return FileTypeTable, dfn, true + case "blob": + return FileTypeBlob, dfn, true } } return 0, dfn, false diff --git a/internal/base/filenames_test.go b/internal/base/filenames_test.go index df07bb8b3c..15ead6f67c 100644 --- a/internal/base/filenames_test.go +++ b/internal/base/filenames_test.go @@ -45,6 +45,10 @@ func TestParseFilename(t *testing.T) { "CURRENT.dbtmp": false, "CURRENT.123456.dbtmp": true, "temporary.123456.dbtmp": true, + "foo.blob": false, + "000000.blob": true, + "000001.blob": true, + "935203523.blob": true, } fs := vfs.NewMem() for tc, want := range testCases { @@ -65,6 +69,7 @@ func TestFilenameRoundTrip(t *testing.T) { FileTypeOptions: true, FileTypeOldTemp: true, FileTypeTemp: true, + FileTypeBlob: true, // NB: Log filenames are created and parsed elsewhere in the wal/ // package. // FileTypeLog: true, diff --git a/objstorage/objstorageprovider/provider.go b/objstorage/objstorageprovider/provider.go index 953a9d33dd..389ae247eb 100644 --- a/objstorage/objstorageprovider/provider.go +++ b/objstorage/objstorageprovider/provider.go @@ -488,13 +488,13 @@ func (p *provider) Lookup( if !ok { return objstorage.ObjectMetadata{}, errors.Wrapf( os.ErrNotExist, - "file %s (type %d) unknown to the objstorage provider", - fileNum, errors.Safe(fileType), + "file %s (type %s) unknown to the objstorage provider", + fileNum, fileType, ) } if meta.FileType != fileType { return objstorage.ObjectMetadata{}, base.AssertionFailedf( - "file %s type mismatch (known type %d, expected type %d)", + "file %s type mismatch (known type %s, expected type %s)", fileNum, errors.Safe(meta.FileType), errors.Safe(fileType), ) } @@ -549,8 +549,8 @@ func (p *provider) CheckpointState( if _, ok := p.mu.knownObjects[fileNums[i]]; !ok { return errors.Wrapf( os.ErrNotExist, - "file %s (type %d) unknown to the objstorage provider", - fileNums[i], errors.Safe(fileType), + "file %s (type %s) unknown to the objstorage provider", + fileNums[i], fileType, ) } // Prevent this object from deletion, at least for the life of this instance. diff --git a/objstorage/objstorageprovider/provider_test.go b/objstorage/objstorageprovider/provider_test.go index 0c8f9f0290..1c371b46bf 100644 --- a/objstorage/objstorageprovider/provider_test.go +++ b/objstorage/objstorageprovider/provider_test.go @@ -110,6 +110,11 @@ func TestProvider(t *testing.T) { opts := objstorage.CreateOptions{ SharedCleanupMethod: objstorage.SharedRefTracking, } + ft := base.FileTypeTable + if len(d.CmdArgs) > 0 && d.CmdArgs[0].Key == "file-type" { + ft = base.FileTypeFromName(d.CmdArgs[0].FirstVal(t)) + d.CmdArgs = d.CmdArgs[1:] + } if len(d.CmdArgs) == 5 && d.CmdArgs[4].Key == "no-ref-tracking" { d.CmdArgs = d.CmdArgs[:4] opts.SharedCleanupMethod = objstorage.SharedNoCleanup @@ -117,7 +122,7 @@ func TestProvider(t *testing.T) { var fileNum base.DiskFileNum var typ string var salt, size int - scanArgs(" [no-ref-tracking]", &fileNum, &typ, &salt, &size) + scanArgs("[file-type=sstable|blob] [no-ref-tracking]", &fileNum, &typ, &salt, &size) switch typ { case "local": case "shared": @@ -125,7 +130,7 @@ func TestProvider(t *testing.T) { default: d.Fatalf(t, "'%s' should be 'local' or 'shared'", typ) } - w, _, err := curProvider.Create(ctx, base.FileTypeTable, fileNum, opts) + w, _, err := curProvider.Create(ctx, ft, fileNum, opts) if err != nil { return err.Error() } @@ -141,6 +146,11 @@ func TestProvider(t *testing.T) { opts := objstorage.CreateOptions{ SharedCleanupMethod: objstorage.SharedRefTracking, } + ft := base.FileTypeTable + if len(d.CmdArgs) > 0 && d.CmdArgs[0].Key == "file-type" { + ft = base.FileTypeFromName(d.CmdArgs[0].FirstVal(t)) + d.CmdArgs = d.CmdArgs[1:] + } if len(d.CmdArgs) == 5 && d.CmdArgs[4].Key == "no-ref-tracking" { d.CmdArgs = d.CmdArgs[:4] opts.SharedCleanupMethod = objstorage.SharedNoCleanup @@ -148,7 +158,7 @@ func TestProvider(t *testing.T) { var fileNum base.DiskFileNum var typ string var salt, size int - scanArgs(" [no-ref-tracking]", &fileNum, &typ, &salt, &size) + scanArgs("[file-type=sstable|blob] [no-ref-tracking]", &fileNum, &typ, &salt, &size) switch typ { case "local": case "shared": @@ -168,9 +178,7 @@ func TestProvider(t *testing.T) { require.NoError(t, err) require.NoError(t, f.Close()) - _, err = curProvider.LinkOrCopyFromLocal( - ctx, fs, tmpFilename, base.FileTypeTable, fileNum, opts, - ) + _, err = curProvider.LinkOrCopyFromLocal(ctx, fs, tmpFilename, ft, fileNum, opts) require.NoError(t, err) return log.String() @@ -195,10 +203,15 @@ func TestProvider(t *testing.T) { } } + ft := base.FileTypeTable + if len(d.CmdArgs) > 0 && d.CmdArgs[0].Key == "file-type" { + ft = base.FileTypeFromName(d.CmdArgs[0].FirstVal(t)) + d.CmdArgs = d.CmdArgs[1:] + } d.CmdArgs = d.CmdArgs[:1] var fileNum base.DiskFileNum - scanArgs(" [for-compaction] [readahead|speculative-overhead=off|sys-readahead|fadvise-sequential]", &fileNum) - r, err := curProvider.OpenForReading(ctx, base.FileTypeTable, fileNum, objstorage.OpenOptions{}) + scanArgs("[file-type=sstable|blob] [for-compaction] [readahead|speculative-overhead=off|sys-readahead|fadvise-sequential]", &fileNum) + r, err := curProvider.OpenForReading(ctx, ft, fileNum, objstorage.OpenOptions{}) if err != nil { return err.Error() } @@ -231,9 +244,14 @@ func TestProvider(t *testing.T) { return log.String() case "remove": + ft := base.FileTypeTable + if len(d.CmdArgs) > 0 && d.CmdArgs[0].Key == "file-type" { + ft = base.FileTypeFromName(d.CmdArgs[0].FirstVal(t)) + d.CmdArgs = d.CmdArgs[1:] + } var fileNum base.DiskFileNum - scanArgs("", &fileNum) - if err := curProvider.Remove(base.FileTypeTable, fileNum); err != nil { + scanArgs("[file-type=sstable|blob] ", &fileNum) + if err := curProvider.Remove(ft, fileNum); err != nil { return err.Error() } return log.String() diff --git a/objstorage/objstorageprovider/testdata/provider/local b/objstorage/objstorageprovider/testdata/provider/local index c2bba671c9..3d7f189977 100644 --- a/objstorage/objstorageprovider/testdata/provider/local +++ b/objstorage/objstorageprovider/testdata/provider/local @@ -62,7 +62,7 @@ list read 1 ---- -file 000001 (type 2) unknown to the objstorage provider: file does not exist +file 000001 (type sstable) unknown to the objstorage provider: file does not exist link-or-copy 3 local 3 100 ---- @@ -94,6 +94,38 @@ size: 1234 0 1234: ok (salt 4) close: p0/000004.sst +create file-type=blob 000005 local 1 4096 +---- + create: p0/000005.blob + sync-data: p0/000005.blob + close: p0/000005.blob + +read file-type=blob 000005 +0 1024 +2048 1024 +---- + open: p0/000005.blob (options: *vfs.randomReadsOption) +size: 4096 + read-at(0, 1024): p0/000005.blob +0 1024: ok (salt 1) + read-at(2048, 1024): p0/000005.blob +2048 1024: ok (salt 1) + close: p0/000005.blob + +link-or-copy file-type=blob 000006 shared 6 1234 +---- + create: temp-file-3 + close: temp-file-3 + link: temp-file-3 -> p0/000006.blob + +list +---- +000002 -> p0/000002.sst +000003 -> p0/000003.sst +000004 -> p0/000004.sst +000005 -> p0/000005.blob +000006 -> p0/000006.blob + close ---- sync: p0 diff --git a/objstorage/objstorageprovider/vfs.go b/objstorage/objstorageprovider/vfs.go index 6e842dcef8..5728531529 100644 --- a/objstorage/objstorageprovider/vfs.go +++ b/objstorage/objstorageprovider/vfs.go @@ -73,12 +73,15 @@ func (p *provider) vfsInit() error { for _, filename := range listing { fileType, fileNum, ok := base.ParseFilename(p.st.FS, filename) - if ok && fileType == base.FileTypeTable { - o := objstorage.ObjectMetadata{ - FileType: fileType, - DiskFileNum: fileNum, + if ok { + switch fileType { + case base.FileTypeTable, base.FileTypeBlob: + o := objstorage.ObjectMetadata{ + FileType: fileType, + DiskFileNum: fileNum, + } + p.mu.knownObjects[o.DiskFileNum] = o } - p.mu.knownObjects[o.DiskFileNum] = o } } return nil diff --git a/obsolete_files.go b/obsolete_files.go index d1fdabee24..b53707841f 100644 --- a/obsolete_files.go +++ b/obsolete_files.go @@ -164,17 +164,20 @@ func (cm *cleanupManager) mainLoop() { for job := range cm.jobsCh { for _, of := range job.obsoleteFiles { switch of.fileType { - case fileTypeTable: + case base.FileTypeTable: cm.maybePace(&tb, of.fileType, of.nonLogFile.fileNum, of.nonLogFile.fileSize) cm.onTableDeleteFn(of.nonLogFile.fileSize, of.nonLogFile.isLocal) - cm.deleteObsoleteObject(fileTypeTable, job.jobID, of.nonLogFile.fileNum) - case fileTypeLog: + cm.deleteObsoleteObject(of.fileType, job.jobID, of.nonLogFile.fileNum) + case base.FileTypeBlob: + cm.maybePace(&tb, of.fileType, of.nonLogFile.fileNum, of.nonLogFile.fileSize) + cm.deleteObsoleteObject(of.fileType, job.jobID, of.nonLogFile.fileNum) + case base.FileTypeLog: cm.deleteObsoleteFile(of.logFile.FS, fileTypeLog, job.jobID, of.logFile.Path, - base.DiskFileNum(of.logFile.NumWAL), of.logFile.ApproxFileSize) + base.DiskFileNum(of.logFile.NumWAL)) default: path := base.MakeFilepath(cm.opts.FS, of.nonLogFile.dir, of.fileType, of.nonLogFile.fileNum) cm.deleteObsoleteFile( - cm.opts.FS, of.fileType, job.jobID, path, of.nonLogFile.fileNum, of.nonLogFile.fileSize) + cm.opts.FS, of.fileType, job.jobID, path, of.nonLogFile.fileNum) } } cm.mu.Lock() @@ -187,7 +190,7 @@ func (cm *cleanupManager) mainLoop() { // fileNumIfSST is read iff fileType is fileTypeTable. func (cm *cleanupManager) needsPacing(fileType base.FileType, fileNumIfSST base.DiskFileNum) bool { - if fileType != fileTypeTable { + if fileType != base.FileTypeTable && fileType != base.FileTypeBlob { return false } meta, err := cm.objProvider.Lookup(fileType, fileNumIfSST) @@ -229,7 +232,7 @@ func (cm *cleanupManager) maybePace( // deleteObsoleteFile deletes a (non-object) file that is no longer needed. func (cm *cleanupManager) deleteObsoleteFile( - fs vfs.FS, fileType fileType, jobID JobID, path string, fileNum base.DiskFileNum, fileSize uint64, + fs vfs.FS, fileType fileType, jobID JobID, path string, fileNum base.DiskFileNum, ) { // TODO(peter): need to handle this error, probably by re-adding the // file that couldn't be deleted to one of the obsolete slices map. @@ -239,21 +242,21 @@ func (cm *cleanupManager) deleteObsoleteFile( } switch fileType { - case fileTypeLog: + case base.FileTypeLog: cm.opts.EventListener.WALDeleted(WALDeleteInfo{ JobID: int(jobID), Path: path, FileNum: fileNum, Err: err, }) - case fileTypeManifest: + case base.FileTypeManifest: cm.opts.EventListener.ManifestDeleted(ManifestDeleteInfo{ JobID: int(jobID), Path: path, FileNum: fileNum, Err: err, }) - case fileTypeTable: + case base.FileTypeTable, base.FileTypeBlob: panic("invalid deletion of object file") } } @@ -261,7 +264,7 @@ func (cm *cleanupManager) deleteObsoleteFile( func (cm *cleanupManager) deleteObsoleteObject( fileType fileType, jobID JobID, fileNum base.DiskFileNum, ) { - if fileType != fileTypeTable { + if fileType != base.FileTypeTable && fileType != base.FileTypeBlob { panic("not an object") } @@ -278,13 +281,14 @@ func (cm *cleanupManager) deleteObsoleteObject( } switch fileType { - case fileTypeTable: + case base.FileTypeTable: cm.opts.EventListener.TableDeleted(TableDeleteInfo{ JobID: int(jobID), Path: path, FileNum: fileNum, Err: err, }) + // TODO(jackson): Add BlobFileDeleted event. } } @@ -367,7 +371,8 @@ func (d *DB) scanObsoleteFiles(list []string, flushableIngests []*ingestedFlusha manifestFileNum := d.mu.versions.manifestFileNum - var obsoleteTables []tableInfo + var obsoleteTables []objectInfo + var obsoleteBlobs []objectInfo var obsoleteManifests []fileInfo var obsoleteOptions []fileInfo @@ -377,7 +382,7 @@ func (d *DB) scanObsoleteFiles(list []string, flushableIngests []*ingestedFlusha continue } switch fileType { - case fileTypeManifest: + case base.FileTypeManifest: if diskFileNum >= manifestFileNum { continue } @@ -386,7 +391,7 @@ func (d *DB) scanObsoleteFiles(list []string, flushableIngests []*ingestedFlusha fi.FileSize = uint64(stat.Size()) } obsoleteManifests = append(obsoleteManifests, fi) - case fileTypeOptions: + case base.FileTypeOptions: if diskFileNum >= d.optionsFileNum { continue } @@ -395,7 +400,7 @@ func (d *DB) scanObsoleteFiles(list []string, flushableIngests []*ingestedFlusha fi.FileSize = uint64(stat.Size()) } obsoleteOptions = append(obsoleteOptions, fi) - case fileTypeTable: + case base.FileTypeTable, base.FileTypeBlob: // Objects are handled through the objstorage provider below. default: // Don't delete files we don't know about. @@ -404,28 +409,32 @@ func (d *DB) scanObsoleteFiles(list []string, flushableIngests []*ingestedFlusha objects := d.objProvider.List() for _, obj := range objects { - switch obj.FileType { - case fileTypeTable: - if _, ok := liveFileNums[obj.DiskFileNum]; ok { - continue - } - fileInfo := fileInfo{ - FileNum: obj.DiskFileNum, - } + if _, ok := liveFileNums[obj.DiskFileNum]; ok { + continue + } + makeObjectInfo := func() objectInfo { + fileInfo := fileInfo{FileNum: obj.DiskFileNum} if size, err := d.objProvider.Size(obj); err == nil { fileInfo.FileSize = uint64(size) } - obsoleteTables = append(obsoleteTables, tableInfo{ + return objectInfo{ fileInfo: fileInfo, isLocal: !obj.IsRemote(), - }) + } + } + switch obj.FileType { + case base.FileTypeTable: + obsoleteTables = append(obsoleteTables, makeObjectInfo()) + case base.FileTypeBlob: + obsoleteBlobs = append(obsoleteBlobs, makeObjectInfo()) default: // Ignore object types we don't know about. } } - d.mu.versions.obsoleteTables = mergeTableInfos(d.mu.versions.obsoleteTables, obsoleteTables) + d.mu.versions.obsoleteTables = mergeObjectInfos(d.mu.versions.obsoleteTables, obsoleteTables) + d.mu.versions.obsoleteBlobs = mergeObjectInfos(d.mu.versions.obsoleteBlobs, obsoleteBlobs) d.mu.versions.updateObsoleteTableMetricsLocked() d.mu.versions.obsoleteManifests = merge(d.mu.versions.obsoleteManifests, obsoleteManifests) d.mu.versions.obsoleteOptions = merge(d.mu.versions.obsoleteOptions, obsoleteOptions) @@ -482,8 +491,10 @@ func (d *DB) deleteObsoleteFiles(jobID JobID) { panic(err) } - obsoleteTables := append([]tableInfo(nil), d.mu.versions.obsoleteTables...) - d.mu.versions.obsoleteTables = nil + obsoleteTables := slices.Clone(d.mu.versions.obsoleteTables) + d.mu.versions.obsoleteTables = d.mu.versions.obsoleteTables[:0] + obsoleteBlobs := slices.Clone(d.mu.versions.obsoleteBlobs) + d.mu.versions.obsoleteBlobs = d.mu.versions.obsoleteBlobs[:0] for _, tbl := range obsoleteTables { delete(d.mu.versions.zombieTables, tbl.FileNum) @@ -519,7 +530,10 @@ func (d *DB) deleteObsoleteFiles(jobID JobID) { } // We sort to make the order of deletions deterministic, which is nice for // tests. - slices.SortFunc(obsoleteTables, func(a, b tableInfo) int { + slices.SortFunc(obsoleteTables, func(a, b objectInfo) int { + return cmp.Compare(a.FileNum, b.FileNum) + }) + slices.SortFunc(obsoleteBlobs, func(a, b objectInfo) int { return cmp.Compare(a.FileNum, b.FileNum) }) for _, f := range obsoleteTables { @@ -534,6 +548,19 @@ func (d *DB) deleteObsoleteFiles(jobID JobID) { }, }) } + for _, f := range obsoleteBlobs { + d.fileCache.evict(f.FileNum) + filesToDelete = append(filesToDelete, obsoleteFile{ + fileType: base.FileTypeBlob, + nonLogFile: deletableFile{ + dir: d.dirname, + fileNum: f.FileNum, + fileSize: f.FileSize, + isLocal: f.isLocal, + }, + }) + } + files := [2]struct { fileType fileType obsolete []fileInfo @@ -594,16 +621,16 @@ func merge(a, b []fileInfo) []fileInfo { }) } -func mergeTableInfos(a, b []tableInfo) []tableInfo { +func mergeObjectInfos(a, b []objectInfo) []objectInfo { if len(b) == 0 { return a } a = append(a, b...) - slices.SortFunc(a, func(a, b tableInfo) int { + slices.SortFunc(a, func(a, b objectInfo) int { return cmp.Compare(a.FileNum, b.FileNum) }) - return slices.CompactFunc(a, func(a, b tableInfo) bool { + return slices.CompactFunc(a, func(a, b objectInfo) bool { return a.FileNum == b.FileNum }) } diff --git a/open.go b/open.go index 29638c6c0d..ec89ffce01 100644 --- a/open.go +++ b/open.go @@ -403,7 +403,7 @@ func Open(dirname string, opts *Options) (db *DB, err error) { if manifestExists && !opts.DisableConsistencyCheck { curVersion := d.mu.versions.currentVersion() - if err := checkConsistency(curVersion, dirname, d.objProvider); err != nil { + if err := checkConsistency(curVersion, d.objProvider); err != nil { return nil, err } } @@ -1238,7 +1238,7 @@ func IsCorruptionError(err error) bool { return errors.Is(err, base.ErrCorruption) } -func checkConsistency(v *manifest.Version, dirname string, objProvider objstorage.Provider) error { +func checkConsistency(v *manifest.Version, objProvider objstorage.Provider) error { var errs []error dedup := make(map[base.DiskFileNum]struct{}) for level, files := range v.Levels { diff --git a/open_test.go b/open_test.go index 0e9dddec50..0f617451c2 100644 --- a/open_test.go +++ b/open_test.go @@ -1417,7 +1417,7 @@ func TestCheckConsistency(t *testing.T) { } v := manifest.NewVersion(base.DefaultComparer, 0, filesByLevel) - err := checkConsistency(v, dir, provider) + err := checkConsistency(v, provider) if err != nil { if redactErr { redacted := redact.Sprint(err).Redact() diff --git a/testdata/cleaner b/testdata/cleaner index 0351b989a7..89bb180708 100644 --- a/testdata/cleaner +++ b/testdata/cleaner @@ -206,6 +206,19 @@ create: db1/000456.sst sync: db1/000456.sst close: db1/000456.sst +create-bogus-file db1/000234.blob +---- +create: db1/000234.blob +sync: db1/000234.blob +close: db1/000234.blob + +create-bogus-file db1/000345.blob +---- +create: db1/000345.blob +sync: db1/000345.blob +close: db1/000345.blob + + open db1 ---- mkdir-all: db1 0755 @@ -249,6 +262,8 @@ rename: db1/temporary.000460.dbtmp -> db1/OPTIONS-000460 sync: db1 remove: db1/000123.sst remove: db1/000456.sst +remove: db1/000234.blob +remove: db1/000345.blob remove: db1/OPTIONS-000003 list db1 diff --git a/testdata/version_check_consistency b/testdata/version_check_consistency index a8ca9b3c27..d741149d88 100644 --- a/testdata/version_check_consistency +++ b/testdata/version_check_consistency @@ -13,25 +13,25 @@ check-consistency L0 000005:10 ---- -L0: 000005: file 000005 (type 2) unknown to the objstorage provider: file does not exist +L0: 000005: file 000005 (type sstable) unknown to the objstorage provider: file does not exist check-consistency L0 000001:10 ---- -L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist +L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist check-consistency L0 000001:11 ---- -L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist +L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist check-consistency redact L0 000001:11 ---- -L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist +L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist check-consistency L0 @@ -41,9 +41,9 @@ L1 L2 000003:30 ---- -L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist -L1: 000002: file 000002 (type 2) unknown to the objstorage provider: file does not exist -L2: 000003: file 000003 (type 2) unknown to the objstorage provider: file does not exist +L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist +L1: 000002: file 000002 (type sstable) unknown to the objstorage provider: file does not exist +L2: 000003: file 000003 (type sstable) unknown to the objstorage provider: file does not exist check-consistency L0 @@ -53,9 +53,9 @@ L1 L2 000003:33 ---- -L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist -L1: 000002: file 000002 (type 2) unknown to the objstorage provider: file does not exist -L2: 000003: file 000003 (type 2) unknown to the objstorage provider: file does not exist +L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist +L1: 000002: file 000002 (type sstable) unknown to the objstorage provider: file does not exist +L2: 000003: file 000003 (type sstable) unknown to the objstorage provider: file does not exist check-consistency redact L0 @@ -65,6 +65,6 @@ L1 L2 000004:30 ---- -L0: 000001: file 000001 (type 2) unknown to the objstorage provider: file does not exist -L1: 000002: file 000002 (type 2) unknown to the objstorage provider: file does not exist -L2: 000004: file 000004 (type 2) unknown to the objstorage provider: file does not exist +L0: 000001: file 000001 (type sstable) unknown to the objstorage provider: file does not exist +L1: 000002: file 000002 (type sstable) unknown to the objstorage provider: file does not exist +L2: 000004: file 000004 (type sstable) unknown to the objstorage provider: file does not exist diff --git a/tool/testdata/db_excise b/tool/testdata/db_excise index 4f903bb699..df6e39c2ab 100644 --- a/tool/testdata/db_excise +++ b/tool/testdata/db_excise @@ -52,7 +52,7 @@ scanned 2 records in 1.0s db scan testdata/broken-external-db ---- -L0: 000008: file 000008 (type 2) unknown to the objstorage provider: file does not exist +L0: 000008: file 000008 (type sstable) unknown to the objstorage provider: file does not exist # The database LSM is as follows: # L0.0: diff --git a/version_set.go b/version_set.go index 6e121a66fd..5a3dfdd056 100644 --- a/version_set.go +++ b/version_set.go @@ -78,13 +78,14 @@ type versionSet struct { // A pointer to versionSet.addObsoleteLocked. Avoids allocating a new closure // on the creation of every version. obsoleteFn func(obsolete []*fileBacking) - obsoleteTables []tableInfo + obsoleteTables []objectInfo + obsoleteBlobs []objectInfo obsoleteManifests []fileInfo obsoleteOptions []fileInfo // Zombie tables which have been removed from the current version but are // still referenced by an inuse iterator. - zombieTables map[base.DiskFileNum]tableInfo + zombieTables map[base.DiskFileNum]objectInfo // virtualBackings contains information about the FileBackings which support // virtual sstables in the latest version. It is mainly used to determine when @@ -129,7 +130,9 @@ type versionSet struct { rotationHelper record.RotationHelper } -type tableInfo struct { +// objectInfo describes an object in object storage (either a sstable or a blob +// file). +type objectInfo struct { fileInfo isLocal bool } @@ -152,7 +155,7 @@ func (vs *versionSet) init( vs.dynamicBaseLevel = true vs.versions.Init(mu) vs.obsoleteFn = vs.addObsoleteLocked - vs.zombieTables = make(map[base.DiskFileNum]tableInfo) + vs.zombieTables = make(map[base.DiskFileNum]objectInfo) vs.virtualBackings = manifest.MakeVirtualBackings() vs.nextFileNum.Store(1) vs.manifestMarker = marker @@ -635,7 +638,7 @@ func (vs *versionSet) logAndApply( // will unref the previous version which could result in addObsoleteLocked // being called. for _, b := range zombieBackings { - vs.zombieTables[b.backing.DiskFileNum] = tableInfo{ + vs.zombieTables[b.backing.DiskFileNum] = objectInfo{ fileInfo: fileInfo{ FileNum: b.backing.DiskFileNum, FileSize: b.backing.Size, @@ -1035,7 +1038,7 @@ func (vs *versionSet) addObsoleteLocked(obsolete []*fileBacking) { return } - obsoleteFileInfo := make([]tableInfo, len(obsolete)) + obsoleteFileInfo := make([]objectInfo, len(obsolete)) for i, bs := range obsolete { obsoleteFileInfo[i].FileNum = bs.DiskFileNum obsoleteFileInfo[i].FileSize = bs.Size