Skip to content

Commit 36f1d3e

Browse files
roman-khimovroman-khimov
authored
Drop BoltDB from write-cache (#3091)
Closes #3076.
2 parents 9285b4f + 3364ab5 commit 36f1d3e

File tree

36 files changed

+174
-793
lines changed

36 files changed

+174
-793
lines changed

‎CHANGELOG.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ Changelog for NeoFS Node
2222

2323
### Removed
2424
- Drop creating new eacl tables with public keys (#3096)
25+
- BoltDB from write-cache (#3091)
2526

2627
### Updated
2728

@@ -31,6 +32,12 @@ since this realese it is not supported to create new eACL table with keys,
3132
use addresses instead.
3233
For more information call `neofs-cli acl extended create -h`.
3334

35+
`small_object_size`, `workers_number`, `max_batch_size` and `max_batch_delay`
36+
paramteters are removed from `writecache` config. These parameters are related
37+
to the BoltDB part of the write-cache, which is dropped from the code.
38+
Also, because of this, there will be automatic migration from BoltDB by flushing
39+
objects to the main storage and removing database file.
40+
3441
## [0.44.2] - 2024-12-20
3542

3643
### Fixed

‎cmd/neofs-lens/internal/printers.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,8 @@ func PrintStorageObjectStatus(cmd *cobra.Command, status engine.ObjectStatus) {
9595
if shard.Shard.Metabase.Error != nil {
9696
cmd.Printf("\t\tMetabase object error:\t%v\n", shard.Shard.Metabase.Error)
9797
}
98-
if shard.Shard.Writecache.PathDB != "" || shard.Shard.Writecache.PathFSTree != "" {
98+
if shard.Shard.Writecache.PathFSTree != "" {
9999
cmd.Printf("\tWritecache\n")
100-
cmd.Printf("\t\tWritecache DB path:\t%s\n", shard.Shard.Writecache.PathDB)
101100
cmd.Printf("\t\tWritecache FSTree path:\t%s\n", shard.Shard.Writecache.PathFSTree)
102101
}
103102
cmd.Println()

‎cmd/neofs-lens/internal/storage/root.go

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,7 @@ func openEngine() (*engine.StorageEngine, error) {
8282

8383
wc.Enabled = true
8484
wc.Path = writeCacheCfg.Path()
85-
wc.MaxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize()
86-
wc.MaxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay()
8785
wc.MaxObjSize = writeCacheCfg.MaxObjectSize()
88-
wc.SmallObjectSize = writeCacheCfg.SmallObjectSize()
89-
wc.FlushWorkerCount = writeCacheCfg.WorkersNumber()
9086
wc.SizeLimit = writeCacheCfg.SizeLimit()
9187
wc.NoSync = writeCacheCfg.NoSync()
9288
}
@@ -166,11 +162,7 @@ func openEngine() (*engine.StorageEngine, error) {
166162
if wcRead := shCfg.WritecacheCfg; wcRead.Enabled {
167163
writeCacheOpts = append(writeCacheOpts,
168164
writecache.WithPath(wcRead.Path),
169-
writecache.WithMaxBatchSize(wcRead.MaxBatchSize),
170-
writecache.WithMaxBatchDelay(wcRead.MaxBatchDelay),
171165
writecache.WithMaxObjectSize(wcRead.MaxObjSize),
172-
writecache.WithSmallObjectSize(wcRead.SmallObjectSize),
173-
writecache.WithFlushWorkersCount(wcRead.FlushWorkerCount),
174166
writecache.WithMaxCacheSize(wcRead.SizeLimit),
175167
writecache.WithNoSync(wcRead.NoSync),
176168
)

‎cmd/neofs-lens/internal/writecache/get.go

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ import (
44
"fmt"
55

66
common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
7-
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
8-
"github.com/nspcc-dev/neofs-sdk-go/object"
7+
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
98
"github.com/spf13/cobra"
109
)
1110

@@ -25,23 +24,23 @@ func init() {
2524
}
2625

2726
func getFunc(cmd *cobra.Command, _ []string) error {
28-
db, err := openWC()
27+
wc, err := openWC()
2928
if err != nil {
3029
return err
3130
}
32-
defer db.Close()
31+
defer wc.Close()
3332

34-
data, err := writecache.Get(db, []byte(vAddress))
33+
addr, err := oid.DecodeAddressString(vAddress)
3534
if err != nil {
36-
return fmt.Errorf("could not fetch object: %w", err)
35+
return fmt.Errorf("invalid address: %w", err)
3736
}
3837

39-
var o object.Object
40-
if err := o.Unmarshal(data); err != nil {
41-
return fmt.Errorf("could not unmarshal object: %w", err)
38+
obj, err := wc.Get(addr)
39+
if err != nil {
40+
return fmt.Errorf("could not fetch object: %w", err)
4241
}
4342

44-
common.PrintObjectHeader(cmd, o)
43+
common.PrintObjectHeader(cmd, *obj)
4544

46-
return common.WriteObjectToFile(cmd, vOut, data, vPayloadOnly)
45+
return common.WriteObjectToFile(cmd, vOut, obj.Marshal(), vPayloadOnly)
4746
}

‎cmd/neofs-lens/internal/writecache/list.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"io"
66

77
common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
8-
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
98
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
109
"github.com/spf13/cobra"
1110
)
@@ -26,18 +25,18 @@ func listFunc(cmd *cobra.Command, _ []string) error {
2625
// other targets can be supported
2726
w := cmd.OutOrStderr()
2827

29-
wAddr := func(addr oid.Address) error {
28+
wAddr := func(addr oid.Address, _ []byte) error {
3029
_, err := io.WriteString(w, fmt.Sprintf("%s\n", addr))
3130
return err
3231
}
3332

34-
db, err := openWC()
33+
wc, err := openWC()
3534
if err != nil {
3635
return err
3736
}
38-
defer db.Close()
37+
defer wc.Close()
3938

40-
err = writecache.IterateDB(db, wAddr)
39+
err = wc.Iterate(wAddr, true)
4140
if err != nil {
4241
return fmt.Errorf("write-cache iterator failure: %w", err)
4342
}

‎cmd/neofs-lens/internal/writecache/root.go

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55

66
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
77
"github.com/spf13/cobra"
8-
"go.etcd.io/bbolt"
98
)
109

1110
var (
@@ -26,11 +25,14 @@ func init() {
2625
Root.AddCommand(getCMD)
2726
}
2827

29-
func openWC() (*bbolt.DB, error) {
30-
db, err := writecache.OpenDB(vPath, true)
28+
// openWC opens and returns read-only writecache.Cache located in vPath.
29+
func openWC() (writecache.Cache, error) {
30+
wc := writecache.New(writecache.WithPath(vPath))
31+
32+
err := wc.Open(true)
3133
if err != nil {
32-
return nil, fmt.Errorf("could not open write-cache db: %w", err)
34+
return nil, fmt.Errorf("could not open write-cache: %w", err)
3335
}
3436

35-
return db, nil
37+
return wc, nil
3638
}

‎cmd/neofs-node/config.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -195,11 +195,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error {
195195

196196
wc.Enabled = true
197197
wc.Path = writeCacheCfg.Path()
198-
wc.MaxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize()
199-
wc.MaxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay()
200198
wc.MaxObjSize = writeCacheCfg.MaxObjectSize()
201-
wc.SmallObjectSize = writeCacheCfg.SmallObjectSize()
202-
wc.FlushWorkerCount = writeCacheCfg.WorkersNumber()
203199
wc.SizeLimit = writeCacheCfg.SizeLimit()
204200
wc.NoSync = writeCacheCfg.NoSync()
205201
}

‎cmd/neofs-node/config/engine/config_test.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,7 @@ func TestEngineSection(t *testing.T) {
7474
require.Equal(t, true, wc.NoSync())
7575

7676
require.Equal(t, "tmp/0/cache", wc.Path())
77-
require.EqualValues(t, 16384, wc.SmallObjectSize())
7877
require.EqualValues(t, 134217728, wc.MaxObjectSize())
79-
require.EqualValues(t, 30, wc.WorkersNumber())
8078
require.EqualValues(t, 3221225472, wc.SizeLimit())
8179

8280
require.Equal(t, "tmp/0/meta", meta.Path())
@@ -117,9 +115,7 @@ func TestEngineSection(t *testing.T) {
117115
require.Equal(t, false, wc.NoSync())
118116

119117
require.Equal(t, "tmp/1/cache", wc.Path())
120-
require.EqualValues(t, 16384, wc.SmallObjectSize())
121118
require.EqualValues(t, 134217728, wc.MaxObjectSize())
122-
require.EqualValues(t, 30, wc.WorkersNumber())
123119
require.EqualValues(t, 4294967296, wc.SizeLimit())
124120

125121
require.Equal(t, "tmp/1/meta", meta.Path())

‎cmd/neofs-node/config/engine/shard/writecache/config.go

Lines changed: 0 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,16 @@ package writecacheconfig
22

33
import (
44
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
5-
boltdbconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/boltdb"
65
)
76

87
// Config is a wrapper over the config section
98
// which provides access to WriteCache configurations.
109
type Config config.Config
1110

1211
const (
13-
// SmallSizeDefault is a default size of small objects.
14-
SmallSizeDefault = 32 << 10
15-
1612
// MaxSizeDefault is a default value of the object payload size limit.
1713
MaxSizeDefault = 64 << 20
1814

19-
// WorkersNumberDefault is a default number of workers.
20-
WorkersNumberDefault = 20
21-
2215
// SizeLimitDefault is a default write-cache size limit.
2316
SizeLimitDefault = 1 << 30
2417
)
@@ -51,22 +44,6 @@ func (x *Config) Path() string {
5144
return p
5245
}
5346

54-
// SmallObjectSize returns the value of "small_object_size" config parameter.
55-
//
56-
// Returns SmallSizeDefault if the value is not a positive number.
57-
func (x *Config) SmallObjectSize() uint64 {
58-
s := config.SizeInBytesSafe(
59-
(*config.Config)(x),
60-
"small_object_size",
61-
)
62-
63-
if s > 0 {
64-
return s
65-
}
66-
67-
return SmallSizeDefault
68-
}
69-
7047
// MaxObjectSize returns the value of "max_object_size" config parameter.
7148
//
7249
// Returns MaxSizeDefault if the value is not a positive number.
@@ -83,22 +60,6 @@ func (x *Config) MaxObjectSize() uint64 {
8360
return MaxSizeDefault
8461
}
8562

86-
// WorkersNumber returns the value of "workers_number" config parameter.
87-
//
88-
// Returns WorkersNumberDefault if the value is not a positive number.
89-
func (x *Config) WorkersNumber() int {
90-
c := config.IntSafe(
91-
(*config.Config)(x),
92-
"workers_number",
93-
)
94-
95-
if c > 0 {
96-
return int(c)
97-
}
98-
99-
return WorkersNumberDefault
100-
}
101-
10263
// SizeLimit returns the value of "capacity" config parameter.
10364
//
10465
// Returns SizeLimitDefault if the value is not a positive number.
@@ -121,8 +82,3 @@ func (x *Config) SizeLimit() uint64 {
12182
func (x *Config) NoSync() bool {
12283
return config.BoolSafe((*config.Config)(x), "no_sync")
12384
}
124-
125-
// BoltDB returns config instance for querying bolt db specific parameters.
126-
func (x *Config) BoltDB() *boltdbconfig.Config {
127-
return (*boltdbconfig.Config)(x)
128-
}

‎cmd/neofs-node/config/internal/validate/config.go

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -141,15 +141,11 @@ type shardDetails struct {
141141
ResyncMetabase bool `mapstructure:"resync_metabase"`
142142

143143
WriteCache struct {
144-
Enabled bool `mapstructure:"enabled"`
145-
Path string `mapstructure:"path"`
146-
Capacity string `mapstructure:"capacity"`
147-
NoSync bool `mapstructure:"no_sync"`
148-
SmallObjectSize string `mapstructure:"small_object_size"`
149-
MaxObjectSize string `mapstructure:"max_object_size"`
150-
WorkersNumber int `mapstructure:"workers_number"`
151-
MaxBatchDelay time.Duration `mapstructure:"max_batch_delay"`
152-
MaxBatchSize int `mapstructure:"max_batch_size"`
144+
Enabled bool `mapstructure:"enabled"`
145+
Path string `mapstructure:"path"`
146+
Capacity string `mapstructure:"capacity"`
147+
NoSync bool `mapstructure:"no_sync"`
148+
MaxObjectSize string `mapstructure:"max_object_size"`
153149
} `mapstructure:"writecache"`
154150

155151
Metabase struct {

‎cmd/neofs-node/storage.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,11 +111,7 @@ func (c *cfg) shardOpts() []shardOptsWithID {
111111
if wcRead := shCfg.WritecacheCfg; wcRead.Enabled {
112112
writeCacheOpts = append(writeCacheOpts,
113113
writecache.WithPath(wcRead.Path),
114-
writecache.WithMaxBatchSize(wcRead.MaxBatchSize),
115-
writecache.WithMaxBatchDelay(wcRead.MaxBatchDelay),
116114
writecache.WithMaxObjectSize(wcRead.MaxObjSize),
117-
writecache.WithSmallObjectSize(wcRead.SmallObjectSize),
118-
writecache.WithFlushWorkersCount(wcRead.FlushWorkerCount),
119115
writecache.WithMaxCacheSize(wcRead.SizeLimit),
120116
writecache.WithNoSync(wcRead.NoSync),
121117
writecache.WithLogger(c.log),

‎config/example/node.env

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,7 @@ NEOFS_STORAGE_SHARD_0_MODE=read-only
101101
NEOFS_STORAGE_SHARD_0_WRITECACHE_ENABLED=false
102102
NEOFS_STORAGE_SHARD_0_WRITECACHE_NO_SYNC=true
103103
NEOFS_STORAGE_SHARD_0_WRITECACHE_PATH=tmp/0/cache
104-
NEOFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384
105104
NEOFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728
106-
NEOFS_STORAGE_SHARD_0_WRITECACHE_WORKERS_NUMBER=30
107105
NEOFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472
108106
### Metabase config
109107
NEOFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta
@@ -142,9 +140,7 @@ NEOFS_STORAGE_SHARD_1_MODE=read-write
142140
### Write cache config
143141
NEOFS_STORAGE_SHARD_1_WRITECACHE_ENABLED=true
144142
NEOFS_STORAGE_SHARD_1_WRITECACHE_PATH=tmp/1/cache
145-
NEOFS_STORAGE_SHARD_1_WRITECACHE_SMALL_OBJECT_SIZE=16384
146143
NEOFS_STORAGE_SHARD_1_WRITECACHE_MAX_OBJECT_SIZE=134217728
147-
NEOFS_STORAGE_SHARD_1_WRITECACHE_WORKERS_NUMBER=30
148144
NEOFS_STORAGE_SHARD_1_WRITECACHE_CAPACITY=4294967296
149145
### Metabase config
150146
NEOFS_STORAGE_SHARD_1_METABASE_PATH=tmp/1/meta

‎config/example/node.json

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -128,9 +128,7 @@
128128
"enabled": false,
129129
"no_sync": true,
130130
"path": "tmp/0/cache",
131-
"small_object_size": 16384,
132131
"max_object_size": 134217728,
133-
"workers_number": 30,
134132
"capacity": 3221225472
135133
},
136134
"metabase": {
@@ -173,9 +171,7 @@
173171
"writecache": {
174172
"enabled": true,
175173
"path": "tmp/1/cache",
176-
"small_object_size": 16384,
177174
"max_object_size": 134217728,
178-
"workers_number": 30,
179175
"capacity": 4294967296
180176
},
181177
"metabase": {

‎config/example/node.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,9 +116,7 @@ storage:
116116

117117
writecache:
118118
enabled: true
119-
small_object_size: 16k # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes
120119
max_object_size: 134217728 # size threshold for "big" objects which bypass write-cache and go to the storage directly, bytes
121-
workers_number: 30 # number of write-cache flusher threads
122120

123121
metabase:
124122
perm: 0644 # permissions for metabase files(directories: +x for current user and group)

‎docs/storage-node-configuration.md

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -268,9 +268,7 @@ writecache:
268268
enabled: true
269269
path: /path/to/writecache
270270
capacity: 4294967296
271-
small_object_size: 16384
272271
max_object_size: 134217728
273-
workers_number: 30
274272
```
275273

276274
| Parameter | Type | Default value | Description |
@@ -279,11 +277,7 @@ writecache:
279277
| `path` | `string` | | Path to the metabase file. |
280278
| `capacity` | `size` | unrestricted | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. |
281279
| `no_sync` | `bool` | `false` | Disable write synchronization, makes writes faster, but can lead to data loss. |
282-
| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. |
283280
| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. |
284-
| `workers_number` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. |
285-
| `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. |
286-
| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. |
287281

288282
### `pilorama` subsection
289283

‎pkg/local_object_storage/shard/dump_test.go

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) {
4949
} else {
5050
sh = newCustomShard(t, t.TempDir(), true,
5151
[]writecache.Option{
52-
writecache.WithSmallObjectSize(wcSmallObjectSize),
5352
writecache.WithMaxObjectSize(wcBigObjectSize),
5453
writecache.WithLogger(zaptest.NewLogger(t)),
5554
},
@@ -302,7 +301,6 @@ func TestDumpIgnoreErrors(t *testing.T) {
302301
wcPath := filepath.Join(dir, "writecache")
303302
wcOpts := []writecache.Option{
304303
writecache.WithPath(wcPath),
305-
writecache.WithSmallObjectSize(wcSmallObjectSize),
306304
writecache.WithMaxObjectSize(wcBigObjectSize),
307305
}
308306
sh := newCustomShard(t, dir, true, wcOpts, bsOpts(2))

0 commit comments

Comments
 (0)