From 1186aa2cce71bb8d80e984fd8cb9c67af38e0794 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Wed, 12 Feb 2025 19:31:25 +0300 Subject: [PATCH] node/metabase: Bump version and fill metadata bucket on upgrade Closes #3117. Signed-off-by: Leonard Lyubich --- pkg/local_object_storage/metabase/VERSION.md | 12 + pkg/local_object_storage/metabase/metadata.go | 21 ++ pkg/local_object_storage/metabase/version.go | 54 +++- .../metabase/version_test.go | 237 ++++++++++++++++++ 4 files changed, 323 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/VERSION.md b/pkg/local_object_storage/metabase/VERSION.md index ad4ec24425..c21aca5aad 100644 --- a/pkg/local_object_storage/metabase/VERSION.md +++ b/pkg/local_object_storage/metabase/VERSION.md @@ -99,9 +99,21 @@ The lowest not used bucket index: 20. - Name: `19` + container ID - Key: first object ID - Value: objects for corresponding split chain +- Metadata bucket + - Name: `255` + container ID + - Keys without values + - `0` + object ID + - `1` + attribute + `0xFF` + `0|1` + fixed256(value) + object ID: integer attributes. \ + Sign byte is 0 for negatives, 1 otherwise. Bits are inverted for negatives also. + - `2` + attribute + `0xFF` + value + object ID: plain non-integer attributes + - `3` + object ID + attribute + `0xFF` + value # History +## Version 3 + +Last version without metadata bucket introduced with `ObjectService.SearchV2` API. + ## Version 2 - Container ID is encoded as 32-byte slice diff --git a/pkg/local_object_storage/metabase/metadata.go b/pkg/local_object_storage/metabase/metadata.go index 869f777054..30c9e5ee51 100644 --- a/pkg/local_object_storage/metabase/metadata.go +++ b/pkg/local_object_storage/metabase/metadata.go @@ -46,6 +46,27 @@ func invalidMetaBucketKeyErr(key []byte, cause error) error { return fmt.Errorf("invalid meta bucket key (prefix 0x%X): %w", key[0], cause) } +func putMetadataForObject(tx *bbolt.Tx, hdr object.Object, root, phy bool) error { + owner := hdr.Owner() + if owner.IsZero() { + return fmt.Errorf("invalid owner: %w", user.ErrZeroID) + } + pldHash, ok := hdr.PayloadChecksum() + if !ok { + return errors.New("missing payload checksum") + } + var ver version.Version + if v := hdr.Version(); v != nil { + ver = *v + } + var pldHmmHash []byte + if h, ok := hdr.PayloadHomomorphicHash(); ok { + pldHmmHash = h.Value() + } + return putMetadata(tx, hdr.GetContainerID(), hdr.GetID(), ver, owner, hdr.Type(), hdr.CreationEpoch(), hdr.PayloadSize(), pldHash.Value(), + pldHmmHash, hdr.SplitID().ToV2(), hdr.GetParentID(), hdr.GetFirstID(), hdr.Attributes(), root, phy) +} + // TODO: fill on migration. // TODO: cleaning on obj removal. func putMetadata(tx *bbolt.Tx, cnr cid.ID, id oid.ID, ver version.Version, owner user.ID, typ object.Type, creationEpoch uint64, diff --git a/pkg/local_object_storage/metabase/version.go b/pkg/local_object_storage/metabase/version.go index 297c0e7f69..c466fa5e64 100644 --- a/pkg/local_object_storage/metabase/version.go +++ b/pkg/local_object_storage/metabase/version.go @@ -1,17 +1,21 @@ package meta import ( + "bytes" "encoding/binary" "errors" "fmt" objectconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/object" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/util/logicerr" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/nspcc-dev/neofs-sdk-go/object" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "go.etcd.io/bbolt" ) // currentMetaVersion contains current metabase version. -const currentMetaVersion = 3 +const currentMetaVersion = 4 var versionKey = []byte("version") @@ -74,6 +78,7 @@ func getVersion(tx *bbolt.Tx) (uint64, bool) { var migrateFrom = map[uint64]func(*DB, *bbolt.Tx) error{ 2: migrateFrom2Version, + 3: migrateFrom3Version, } func migrateFrom2Version(db *DB, tx *bbolt.Tx) error { @@ -105,3 +110,50 @@ func migrateFrom2Version(db *DB, tx *bbolt.Tx) error { return updateVersion(tx, 3) } + +func migrateFrom3Version(_ *DB, tx *bbolt.Tx) error { + c := tx.Cursor() + pref := []byte{metadataPrefix} + if k, _ := c.Seek(pref); bytes.HasPrefix(k, pref) { + return fmt.Errorf("key with prefix 0x%X detected", pref) + } + err := tx.ForEach(func(name []byte, b *bbolt.Bucket) error { + switch name[0] { + default: + return nil + case primaryPrefix, tombstonePrefix, storageGroupPrefix, lockersPrefix, linkObjectsPrefix: + } + if len(name[1:]) != cid.Size { + return fmt.Errorf("invalid container bucket with prefix 0x%X: wrong CID len %d", name[0], len(name[1:])) + } + cnr := cid.ID(name[1:]) + err := b.ForEach(func(k, v []byte) error { + if len(k) != oid.Size { + return fmt.Errorf("wrong OID key len %d", len(k)) + } + id := oid.ID(k) + var hdr object.Object + if err := hdr.Unmarshal(v); err != nil { + return fmt.Errorf("decode header of object %s from bucket value: %w", id, err) + } + par := hdr.Parent() + if err := putMetadataForObject(tx, hdr, par == nil, true); err != nil { + return fmt.Errorf("put metadata for object %s: %w", id, err) + } + if par != nil { + if err := putMetadataForObject(tx, *par, true, false); err != nil { + return fmt.Errorf("put metadata for parent of object %s: %w", id, err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("process container 0x%X%s bucket: %w", name[0], cnr, err) + } + return nil + }) + if err != nil { + return err + } + return updateVersion(tx, 4) +} diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index cc6afc415c..57a398551a 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -3,19 +3,31 @@ package meta import ( "bytes" "encoding/binary" + "encoding/hex" "errors" "fmt" + "math/rand" "os" "path" "path/filepath" + "slices" + "strconv" "testing" objectconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/object" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" + "github.com/nspcc-dev/neofs-sdk-go/checksum" + "github.com/nspcc-dev/neofs-sdk-go/client" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" + objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test" + usertest "github.com/nspcc-dev/neofs-sdk-go/user/test" + "github.com/nspcc-dev/neofs-sdk-go/version" + "github.com/nspcc-dev/tzhash/tz" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" ) @@ -344,3 +356,228 @@ func TestMigrate2to3(t *testing.T) { }) require.NoError(t, err) } + +func TestMigrate3to4(t *testing.T) { + db := newDB(t) + + typs := []object.Type{object.TypeRegular, object.TypeTombstone, object.TypeStorageGroup, object.TypeLock, object.TypeLink} + objs := make([]object.Object, len(typs)) + var css, hcss [][]byte + for i := range objs { + objs[i].SetContainerID(cidtest.ID()) + id := oidtest.ID() + objs[i].SetID(id) + ver := version.New(uint32(100*i), uint32(100*i+1)) + objs[i].SetVersion(&ver) + objs[i].SetOwner(usertest.ID()) + objs[i].SetType(typs[i]) + objs[i].SetCreationEpoch(rand.Uint64()) + objs[i].SetPayloadSize(rand.Uint64()) + objs[i].SetPayloadChecksum(checksum.NewSHA256(id)) + css = append(css, id[:]) + var tzh [tz.Size]byte + rand.Read(tzh[:]) //nolint:staticcheck + objs[i].SetPayloadHomomorphicHash(checksum.NewTillichZemor(tzh)) + hcss = append(hcss, tzh[:]) + sid := objecttest.SplitID() + objs[i].SetSplitID(&sid) + objs[i].SetParentID(oidtest.ID()) + objs[i].SetFirstID(oidtest.ID()) + objs[i].SetAttributes(*object.NewAttribute("Index", strconv.Itoa(i))) + } + + var par object.Object + par.SetContainerID(objs[0].GetContainerID()) + par.SetID(oidtest.ID()) + ver := version.New(1000, 1001) + par.SetVersion(&ver) + par.SetOwner(usertest.ID()) + par.SetType(typs[0]) + par.SetCreationEpoch(rand.Uint64()) + par.SetPayloadSize(rand.Uint64()) + pcs := oidtest.ID() + par.SetPayloadChecksum(checksum.NewSHA256(pcs)) + var phcs [tz.Size]byte + rand.Read(phcs[:]) //nolint:staticcheck + par.SetPayloadHomomorphicHash(checksum.NewTillichZemor(phcs)) + sid := objecttest.SplitID() + par.SetSplitID(&sid) + par.SetParentID(oidtest.ID()) + par.SetFirstID(oidtest.ID()) + par.SetAttributes(*object.NewAttribute("Index", "9999")) + + objs[0].SetParent(&par) + + for _, item := range []struct { + pref byte + hdr *object.Object + }{ + {pref: 0x06, hdr: &objs[0]}, + {pref: 0x06, hdr: &par}, + {pref: 0x09, hdr: &objs[1]}, + {pref: 0x08, hdr: &objs[2]}, + {pref: 0x07, hdr: &objs[3]}, + {pref: 0x12, hdr: &objs[4]}, + } { + err := db.boltDB.Update(func(tx *bbolt.Tx) error { + cnr := item.hdr.GetContainerID() + bkt, err := tx.CreateBucketIfNotExists(slices.Concat([]byte{item.pref}, cnr[:])) + require.NoError(t, err) + id := item.hdr.GetID() + return bkt.Put(id[:], item.hdr.Marshal()) + }) + require.NoError(t, err) + } + + // force old version + err := db.boltDB.Update(func(tx *bbolt.Tx) error { + if err := tx.ForEach(func(name []byte, b *bbolt.Bucket) error { + if name[0] == 0xFF { + return tx.DeleteBucket(name) + } + return nil + }); err != nil { + return err + } + + bkt := tx.Bucket([]byte{0x05}) + require.NotNil(t, bkt) + return bkt.Put([]byte("version"), []byte{0x03, 0, 0, 0, 0, 0, 0, 0}) + }) + require.NoError(t, err) + // migrate + require.NoError(t, db.Init()) + // check + err = db.boltDB.View(func(tx *bbolt.Tx) error { + bkt := tx.Bucket([]byte{0x05}) + require.NotNil(t, bkt) + require.Equal(t, []byte{0x04, 0, 0, 0, 0, 0, 0, 0}, bkt.Get([]byte("version"))) + return nil + }) + require.NoError(t, err) + + res, _, err := db.Search(objs[0].GetContainerID(), nil, nil, nil, 1000) + require.NoError(t, err) + require.Len(t, res, 2) + require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == objs[0].GetID() })) + require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == par.GetID() })) + + for i := range objs[1:] { + res, _, err := db.Search(objs[1+i].GetContainerID(), nil, nil, nil, 1000) + require.NoError(t, err, i) + require.Len(t, res, 1, i) + require.Equal(t, objs[1+i].GetID(), res[0].ID, i) + } + + for _, tc := range []struct { + attr string + val string + cnr cid.ID + exp oid.ID + par bool + }{ + {attr: "$Object:version", val: "v0.1", cnr: objs[0].GetContainerID(), exp: objs[0].GetID()}, + {attr: "$Object:version", val: "v100.101", cnr: objs[1].GetContainerID(), exp: objs[1].GetID()}, + {attr: "$Object:version", val: "v200.201", cnr: objs[2].GetContainerID(), exp: objs[2].GetID()}, + {attr: "$Object:version", val: "v300.301", cnr: objs[3].GetContainerID(), exp: objs[3].GetID()}, + {attr: "$Object:version", val: "v400.401", cnr: objs[4].GetContainerID(), exp: objs[4].GetID()}, + {attr: "$Object:version", val: "v1000.1001", cnr: par.GetContainerID(), exp: par.GetID()}, + {attr: "$Object:ownerID", val: objs[0].Owner().String(), cnr: objs[0].GetContainerID(), exp: objs[0].GetID()}, + {attr: "$Object:ownerID", val: objs[1].Owner().String(), cnr: objs[1].GetContainerID(), exp: objs[1].GetID()}, + {attr: "$Object:ownerID", val: objs[2].Owner().String(), cnr: objs[2].GetContainerID(), exp: objs[2].GetID()}, + {attr: "$Object:ownerID", val: objs[3].Owner().String(), cnr: objs[3].GetContainerID(), exp: objs[3].GetID()}, + {attr: "$Object:ownerID", val: objs[4].Owner().String(), cnr: objs[4].GetContainerID(), exp: objs[4].GetID()}, + {attr: "$Object:ownerID", val: par.Owner().String(), cnr: par.GetContainerID(), exp: par.GetID()}, + {attr: "$Object:objectType", val: "REGULAR", cnr: objs[0].GetContainerID(), par: true}, + {attr: "$Object:objectType", val: "TOMBSTONE", cnr: objs[1].GetContainerID(), exp: objs[1].GetID()}, + {attr: "$Object:objectType", val: "STORAGE_GROUP", cnr: objs[2].GetContainerID(), exp: objs[2].GetID()}, + {attr: "$Object:objectType", val: "LOCK", cnr: objs[3].GetContainerID(), exp: objs[3].GetID()}, + {attr: "$Object:objectType", val: "LINK", cnr: objs[4].GetContainerID(), exp: objs[4].GetID()}, + {attr: "$Object:creationEpoch", val: strconv.FormatUint(objs[0].CreationEpoch(), 10), cnr: objs[0].GetContainerID(), exp: objs[0].GetID()}, + {attr: "$Object:creationEpoch", val: strconv.FormatUint(objs[1].CreationEpoch(), 10), cnr: objs[1].GetContainerID(), exp: objs[1].GetID()}, + {attr: "$Object:creationEpoch", val: strconv.FormatUint(objs[2].CreationEpoch(), 10), cnr: objs[2].GetContainerID(), exp: objs[2].GetID()}, + {attr: "$Object:creationEpoch", val: strconv.FormatUint(objs[3].CreationEpoch(), 10), cnr: objs[3].GetContainerID(), exp: objs[3].GetID()}, + {attr: "$Object:creationEpoch", val: strconv.FormatUint(objs[4].CreationEpoch(), 10), cnr: objs[4].GetContainerID(), exp: objs[4].GetID()}, + {attr: "$Object:creationEpoch", val: strconv.FormatUint(par.CreationEpoch(), 10), cnr: par.GetContainerID(), exp: par.GetID()}, + {attr: "$Object:payloadLength", val: strconv.FormatUint(objs[0].PayloadSize(), 10), cnr: objs[0].GetContainerID(), exp: objs[0].GetID()}, + {attr: "$Object:payloadLength", val: strconv.FormatUint(objs[1].PayloadSize(), 10), cnr: objs[1].GetContainerID(), exp: objs[1].GetID()}, + {attr: "$Object:payloadLength", val: strconv.FormatUint(objs[2].PayloadSize(), 10), cnr: objs[2].GetContainerID(), exp: objs[2].GetID()}, + {attr: "$Object:payloadLength", val: strconv.FormatUint(objs[3].PayloadSize(), 10), cnr: objs[3].GetContainerID(), exp: objs[3].GetID()}, + {attr: "$Object:payloadLength", val: strconv.FormatUint(objs[4].PayloadSize(), 10), cnr: objs[4].GetContainerID(), exp: objs[4].GetID()}, + {attr: "$Object:payloadLength", val: strconv.FormatUint(par.PayloadSize(), 10), cnr: par.GetContainerID(), exp: par.GetID()}, + {attr: "$Object:payloadHash", val: hex.EncodeToString(css[0]), cnr: objs[0].GetContainerID(), exp: objs[0].GetID()}, + {attr: "$Object:payloadHash", val: hex.EncodeToString(css[1]), cnr: objs[1].GetContainerID(), exp: objs[1].GetID()}, + {attr: "$Object:payloadHash", val: hex.EncodeToString(css[2]), cnr: objs[2].GetContainerID(), exp: objs[2].GetID()}, + {attr: "$Object:payloadHash", val: hex.EncodeToString(css[3]), cnr: objs[3].GetContainerID(), exp: objs[3].GetID()}, + {attr: "$Object:payloadHash", val: hex.EncodeToString(css[4]), cnr: objs[4].GetContainerID(), exp: objs[4].GetID()}, + {attr: "$Object:payloadHash", val: hex.EncodeToString(pcs[:]), cnr: par.GetContainerID(), exp: par.GetID()}, + {attr: "$Object:homomorphicHash", val: hex.EncodeToString(hcss[0]), cnr: objs[0].GetContainerID(), exp: objs[0].GetID()}, + {attr: "$Object:homomorphicHash", val: hex.EncodeToString(hcss[1]), cnr: objs[1].GetContainerID(), exp: objs[1].GetID()}, + {attr: "$Object:homomorphicHash", val: hex.EncodeToString(hcss[2]), cnr: objs[2].GetContainerID(), exp: objs[2].GetID()}, + {attr: "$Object:homomorphicHash", val: hex.EncodeToString(hcss[3]), cnr: objs[3].GetContainerID(), exp: objs[3].GetID()}, + {attr: "$Object:homomorphicHash", val: hex.EncodeToString(hcss[4]), cnr: objs[4].GetContainerID(), exp: objs[4].GetID()}, + {attr: "$Object:homomorphicHash", val: hex.EncodeToString(phcs[:]), cnr: par.GetContainerID(), exp: par.GetID()}, + {attr: "$Object:split.splitID", val: objs[0].SplitID().String(), cnr: objs[0].GetContainerID(), exp: objs[0].GetID()}, + {attr: "$Object:split.splitID", val: objs[1].SplitID().String(), cnr: objs[1].GetContainerID(), exp: objs[1].GetID()}, + {attr: "$Object:split.splitID", val: objs[2].SplitID().String(), cnr: objs[2].GetContainerID(), exp: objs[2].GetID()}, + {attr: "$Object:split.splitID", val: objs[3].SplitID().String(), cnr: objs[3].GetContainerID(), exp: objs[3].GetID()}, + {attr: "$Object:split.splitID", val: objs[4].SplitID().String(), cnr: objs[4].GetContainerID(), exp: objs[4].GetID()}, + {attr: "$Object:split.splitID", val: par.SplitID().String(), cnr: par.GetContainerID(), exp: par.GetID()}, + {attr: "$Object:split.parent", val: objs[0].GetParentID().String(), cnr: objs[0].GetContainerID(), exp: objs[0].GetID()}, + {attr: "$Object:split.parent", val: objs[1].GetParentID().String(), cnr: objs[1].GetContainerID(), exp: objs[1].GetID()}, + {attr: "$Object:split.parent", val: objs[2].GetParentID().String(), cnr: objs[2].GetContainerID(), exp: objs[2].GetID()}, + {attr: "$Object:split.parent", val: objs[3].GetParentID().String(), cnr: objs[3].GetContainerID(), exp: objs[3].GetID()}, + {attr: "$Object:split.parent", val: objs[4].GetParentID().String(), cnr: objs[4].GetContainerID(), exp: objs[4].GetID()}, + {attr: "$Object:split.parent", val: par.GetParentID().String(), cnr: par.GetContainerID(), exp: par.GetID()}, + {attr: "$Object:split.first", val: objs[0].GetFirstID().String(), cnr: objs[0].GetContainerID(), exp: objs[0].GetID()}, + {attr: "$Object:split.first", val: objs[1].GetFirstID().String(), cnr: objs[1].GetContainerID(), exp: objs[1].GetID()}, + {attr: "$Object:split.first", val: objs[2].GetFirstID().String(), cnr: objs[2].GetContainerID(), exp: objs[2].GetID()}, + {attr: "$Object:split.first", val: objs[3].GetFirstID().String(), cnr: objs[3].GetContainerID(), exp: objs[3].GetID()}, + {attr: "$Object:split.first", val: objs[4].GetFirstID().String(), cnr: objs[4].GetContainerID(), exp: objs[4].GetID()}, + {attr: "$Object:split.first", val: par.GetFirstID().String(), cnr: par.GetContainerID(), exp: par.GetID()}, + {attr: "Index", val: "0", cnr: objs[0].GetContainerID(), exp: objs[0].GetID()}, + {attr: "Index", val: "1", cnr: objs[1].GetContainerID(), exp: objs[1].GetID()}, + {attr: "Index", val: "2", cnr: objs[2].GetContainerID(), exp: objs[2].GetID()}, + {attr: "Index", val: "3", cnr: objs[3].GetContainerID(), exp: objs[3].GetID()}, + {attr: "Index", val: "4", cnr: objs[4].GetContainerID(), exp: objs[4].GetID()}, + {attr: "Index", val: "9999", cnr: par.GetContainerID(), exp: par.GetID()}, + } { + var fs object.SearchFilters + fs.AddFilter(tc.attr, tc.val, object.MatchStringEqual) + res, _, err := db.Search(tc.cnr, fs, nil, nil, 1000) + require.NoError(t, err, tc) + if !tc.par { + require.Len(t, res, 1, tc) + require.Equal(t, tc.exp, res[0].ID, tc) + } else { + require.Len(t, res, 2, tc) + require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == objs[0].GetID() })) + require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == par.GetID() })) + } + } + + for i := range objs { + var fs object.SearchFilters + fs.AddRootFilter() + res, _, err = db.Search(objs[i].GetContainerID(), fs, nil, nil, 1000) + require.NoError(t, err, i) + require.Len(t, res, 1, i) + if i == 0 { + require.Equal(t, par.GetID(), res[0].ID) + } else { + require.Equal(t, objs[i].GetID(), res[0].ID, i) + } + fs = fs[:0] + fs.AddPhyFilter() + res, _, err = db.Search(objs[i].GetContainerID(), fs, nil, nil, 1000) + require.NoError(t, err, i) + if i == 0 { + require.Len(t, res, 2) + require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == objs[0].GetID() })) + require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == par.GetID() })) + } else { + require.Len(t, res, 1) + require.Equal(t, objs[i].GetID(), res[0].ID, i) + } + } +}