Drop BoltDB from write-cache (#3091)

Closes #3076.

Author: roman-khimov

CHANGELOG.md

@@ -22,6 +22,7 @@ Changelog for NeoFS Node
 
 ### Removed
 - Drop creating new eacl tables with public keys (#3096)
+- BoltDB from write-cache (#3091)
 
 ### Updated
 
@@ -31,6 +32,12 @@ since this realese it is not supported to create new eACL table with keys,
 use addresses instead. 
 For more information call `neofs-cli acl extended create -h`.
 
+`small_object_size`, `workers_number`, `max_batch_size` and `max_batch_delay`
+paramteters are removed from `writecache` config. These parameters are related
+to the BoltDB part of the write-cache, which is dropped from the code. 
+Also, because of this, there will be automatic migration from BoltDB by flushing 
+objects to the main storage and removing database file.
+
 ## [0.44.2] - 2024-12-20
 
 ### Fixed

cmd/neofs-lens/internal/printers.go

@@ -95,9 +95,8 @@ func PrintStorageObjectStatus(cmd *cobra.Command, status engine.ObjectStatus) {
 			if shard.Shard.Metabase.Error != nil {
 				cmd.Printf("\t\tMetabase object error:\t%v\n", shard.Shard.Metabase.Error)
 			}
-			if shard.Shard.Writecache.PathDB != "" || shard.Shard.Writecache.PathFSTree != "" {
+			if shard.Shard.Writecache.PathFSTree != "" {
 				cmd.Printf("\tWritecache\n")
-				cmd.Printf("\t\tWritecache DB path:\t%s\n", shard.Shard.Writecache.PathDB)
 				cmd.Printf("\t\tWritecache FSTree path:\t%s\n", shard.Shard.Writecache.PathFSTree)
 			}
 			cmd.Println()

cmd/neofs-lens/internal/storage/root.go

@@ -82,11 +82,7 @@ func openEngine() (*engine.StorageEngine, error) {
 
 			wc.Enabled = true
 			wc.Path = writeCacheCfg.Path()
-			wc.MaxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize()
-			wc.MaxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay()
 			wc.MaxObjSize = writeCacheCfg.MaxObjectSize()
-			wc.SmallObjectSize = writeCacheCfg.SmallObjectSize()
-			wc.FlushWorkerCount = writeCacheCfg.WorkersNumber()
 			wc.SizeLimit = writeCacheCfg.SizeLimit()
 			wc.NoSync = writeCacheCfg.NoSync()
 		}
@@ -166,11 +162,7 @@ func openEngine() (*engine.StorageEngine, error) {
 		if wcRead := shCfg.WritecacheCfg; wcRead.Enabled {
 			writeCacheOpts = append(writeCacheOpts,
 				writecache.WithPath(wcRead.Path),
-				writecache.WithMaxBatchSize(wcRead.MaxBatchSize),
-				writecache.WithMaxBatchDelay(wcRead.MaxBatchDelay),
 				writecache.WithMaxObjectSize(wcRead.MaxObjSize),
-				writecache.WithSmallObjectSize(wcRead.SmallObjectSize),
-				writecache.WithFlushWorkersCount(wcRead.FlushWorkerCount),
 				writecache.WithMaxCacheSize(wcRead.SizeLimit),
 				writecache.WithNoSync(wcRead.NoSync),
 			)

cmd/neofs-lens/internal/writecache/get.go

@@ -4,8 +4,7 @@ import (
 	"fmt"
 
 	common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
-	"github.com/nspcc-dev/neofs-sdk-go/object"
+	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"github.com/spf13/cobra"
 )
 
@@ -25,23 +24,23 @@ func init() {
 }
 
 func getFunc(cmd *cobra.Command, _ []string) error {
-	db, err := openWC()
+	wc, err := openWC()
 	if err != nil {
 		return err
 	}
-	defer db.Close()
+	defer wc.Close()
 
-	data, err := writecache.Get(db, []byte(vAddress))
+	addr, err := oid.DecodeAddressString(vAddress)
 	if err != nil {
-		return fmt.Errorf("could not fetch object: %w", err)
+		return fmt.Errorf("invalid address: %w", err)
 	}
 
-	var o object.Object
-	if err := o.Unmarshal(data); err != nil {
-		return fmt.Errorf("could not unmarshal object: %w", err)
+	obj, err := wc.Get(addr)
+	if err != nil {
+		return fmt.Errorf("could not fetch object: %w", err)
 	}
 
-	common.PrintObjectHeader(cmd, o)
+	common.PrintObjectHeader(cmd, *obj)
 
-	return common.WriteObjectToFile(cmd, vOut, data, vPayloadOnly)
+	return common.WriteObjectToFile(cmd, vOut, obj.Marshal(), vPayloadOnly)
 }

cmd/neofs-lens/internal/writecache/list.go

@@ -5,7 +5,6 @@ import (
 	"io"
 
 	common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"github.com/spf13/cobra"
 )
@@ -26,18 +25,18 @@ func listFunc(cmd *cobra.Command, _ []string) error {
 	// other targets can be supported
 	w := cmd.OutOrStderr()
 
-	wAddr := func(addr oid.Address) error {
+	wAddr := func(addr oid.Address, _ []byte) error {
 		_, err := io.WriteString(w, fmt.Sprintf("%s\n", addr))
 		return err
 	}
 
-	db, err := openWC()
+	wc, err := openWC()
 	if err != nil {
 		return err
 	}
-	defer db.Close()
+	defer wc.Close()
 
-	err = writecache.IterateDB(db, wAddr)
+	err = wc.Iterate(wAddr, true)
 	if err != nil {
 		return fmt.Errorf("write-cache iterator failure: %w", err)
 	}

cmd/neofs-lens/internal/writecache/root.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	"github.com/spf13/cobra"
-	"go.etcd.io/bbolt"
 )
 
 var (
@@ -26,11 +25,14 @@ func init() {
 	Root.AddCommand(getCMD)
 }
 
-func openWC() (*bbolt.DB, error) {
-	db, err := writecache.OpenDB(vPath, true)
+// openWC opens and returns read-only writecache.Cache located in vPath.
+func openWC() (writecache.Cache, error) {
+	wc := writecache.New(writecache.WithPath(vPath))
+
+	err := wc.Open(true)
 	if err != nil {
-		return nil, fmt.Errorf("could not open write-cache db: %w", err)
+		return nil, fmt.Errorf("could not open write-cache: %w", err)
 	}
 
-	return db, nil
+	return wc, nil
 }

cmd/neofs-node/config.go

@@ -195,11 +195,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error {
 
 			wc.Enabled = true
 			wc.Path = writeCacheCfg.Path()
-			wc.MaxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize()
-			wc.MaxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay()
 			wc.MaxObjSize = writeCacheCfg.MaxObjectSize()
-			wc.SmallObjectSize = writeCacheCfg.SmallObjectSize()
-			wc.FlushWorkerCount = writeCacheCfg.WorkersNumber()
 			wc.SizeLimit = writeCacheCfg.SizeLimit()
 			wc.NoSync = writeCacheCfg.NoSync()
 		}

cmd/neofs-node/config/engine/config_test.go

@@ -74,9 +74,7 @@ func TestEngineSection(t *testing.T) {
 				require.Equal(t, true, wc.NoSync())
 
 				require.Equal(t, "tmp/0/cache", wc.Path())
-				require.EqualValues(t, 16384, wc.SmallObjectSize())
 				require.EqualValues(t, 134217728, wc.MaxObjectSize())
-				require.EqualValues(t, 30, wc.WorkersNumber())
 				require.EqualValues(t, 3221225472, wc.SizeLimit())
 
 				require.Equal(t, "tmp/0/meta", meta.Path())
@@ -117,9 +115,7 @@ func TestEngineSection(t *testing.T) {
 				require.Equal(t, false, wc.NoSync())
 
 				require.Equal(t, "tmp/1/cache", wc.Path())
-				require.EqualValues(t, 16384, wc.SmallObjectSize())
 				require.EqualValues(t, 134217728, wc.MaxObjectSize())
-				require.EqualValues(t, 30, wc.WorkersNumber())
 				require.EqualValues(t, 4294967296, wc.SizeLimit())
 
 				require.Equal(t, "tmp/1/meta", meta.Path())

cmd/neofs-node/config/engine/shard/writecache/config.go

@@ -2,23 +2,16 @@ package writecacheconfig
 
 import (
 	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
-	boltdbconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/boltdb"
 )
 
 // Config is a wrapper over the config section
 // which provides access to WriteCache configurations.
 type Config config.Config
 
 const (
-	// SmallSizeDefault is a default size of small objects.
-	SmallSizeDefault = 32 << 10
-
 	// MaxSizeDefault is a default value of the object payload size limit.
 	MaxSizeDefault = 64 << 20
 
-	// WorkersNumberDefault is a default number of workers.
-	WorkersNumberDefault = 20
-
 	// SizeLimitDefault is a default write-cache size limit.
 	SizeLimitDefault = 1 << 30
 )
@@ -51,22 +44,6 @@ func (x *Config) Path() string {
 	return p
 }
 
-// SmallObjectSize returns the value of "small_object_size" config parameter.
-//
-// Returns SmallSizeDefault if the value is not a positive number.
-func (x *Config) SmallObjectSize() uint64 {
-	s := config.SizeInBytesSafe(
-		(*config.Config)(x),
-		"small_object_size",
-	)
-
-	if s > 0 {
-		return s
-	}
-
-	return SmallSizeDefault
-}
-
 // MaxObjectSize returns the value of "max_object_size" config parameter.
 //
 // Returns MaxSizeDefault if the value is not a positive number.
@@ -83,22 +60,6 @@ func (x *Config) MaxObjectSize() uint64 {
 	return MaxSizeDefault
 }
 
-// WorkersNumber returns the value of "workers_number" config parameter.
-//
-// Returns WorkersNumberDefault if the value is not a positive number.
-func (x *Config) WorkersNumber() int {
-	c := config.IntSafe(
-		(*config.Config)(x),
-		"workers_number",
-	)
-
-	if c > 0 {
-		return int(c)
-	}
-
-	return WorkersNumberDefault
-}
-
 // SizeLimit returns the value of "capacity" config parameter.
 //
 // Returns SizeLimitDefault if the value is not a positive number.
@@ -121,8 +82,3 @@ func (x *Config) SizeLimit() uint64 {
 func (x *Config) NoSync() bool {
 	return config.BoolSafe((*config.Config)(x), "no_sync")
 }
-
-// BoltDB returns config instance for querying bolt db specific parameters.
-func (x *Config) BoltDB() *boltdbconfig.Config {
-	return (*boltdbconfig.Config)(x)
-}

cmd/neofs-node/config/internal/validate/config.go

@@ -141,15 +141,11 @@ type shardDetails struct {
 	ResyncMetabase bool   `mapstructure:"resync_metabase"`
 
 	WriteCache struct {
-		Enabled         bool          `mapstructure:"enabled"`
-		Path            string        `mapstructure:"path"`
-		Capacity        string        `mapstructure:"capacity"`
-		NoSync          bool          `mapstructure:"no_sync"`
-		SmallObjectSize string        `mapstructure:"small_object_size"`
-		MaxObjectSize   string        `mapstructure:"max_object_size"`
-		WorkersNumber   int           `mapstructure:"workers_number"`
-		MaxBatchDelay   time.Duration `mapstructure:"max_batch_delay"`
-		MaxBatchSize    int           `mapstructure:"max_batch_size"`
+		Enabled       bool   `mapstructure:"enabled"`
+		Path          string `mapstructure:"path"`
+		Capacity      string `mapstructure:"capacity"`
+		NoSync        bool   `mapstructure:"no_sync"`
+		MaxObjectSize string `mapstructure:"max_object_size"`
 	} `mapstructure:"writecache"`
 
 	Metabase struct {