Skip to content

Commit ef63ec1

Browse files
cthulhu-ridercthulhu-rider
authored
node/metabase: Support ROOT and PHY search filters in new meta bucket (#3124)
2 parents eb90734 + 650b801 commit ef63ec1

File tree

3 files changed

+40
-19
lines changed

3 files changed

+40
-19
lines changed

‎pkg/local_object_storage/metabase/metadata.go

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ const (
3434
attrIDFixedLen = 1 + oid.Size + utf8DelimiterLen // prefix first
3535
)
3636

37+
const binPropMarker = "1" // ROOT, PHY, etc.
38+
3739
var (
3840
maxUint256 = new(big.Int).SetBytes([]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
3941
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255})
@@ -45,10 +47,10 @@ func invalidMetaBucketKeyErr(key []byte, cause error) error {
4547
}
4648

4749
// TODO: fill on migration.
48-
// TODO: ROOT and PHY props.
4950
// TODO: cleaning on obj removal.
5051
func putMetadata(tx *bbolt.Tx, cnr cid.ID, id oid.ID, ver version.Version, owner user.ID, typ object.Type, creationEpoch uint64,
51-
payloadLen uint64, pldHash, pldHmmHash, splitID []byte, parentID, firstID oid.ID, attrs []object.Attribute) error {
52+
payloadLen uint64, pldHash, pldHmmHash, splitID []byte, parentID, firstID oid.ID, attrs []object.Attribute,
53+
root, phy bool) error {
5254
metaBkt, err := tx.CreateBucketIfNotExists(metaBucketKey(cnr))
5355
if err != nil {
5456
return fmt.Errorf("create meta bucket for container: %w", err)
@@ -96,6 +98,16 @@ func putMetadata(tx *bbolt.Tx, cnr cid.ID, id oid.ID, ver version.Version, owner
9698
return err
9799
}
98100
}
101+
if root {
102+
if err = putPlainAttribute(metaBkt, &keyBuf, id, object.FilterRoot, binPropMarker); err != nil {
103+
return err
104+
}
105+
}
106+
if phy {
107+
if err = putPlainAttribute(metaBkt, &keyBuf, id, object.FilterPhysical, binPropMarker); err != nil {
108+
return err
109+
}
110+
}
99111
for i := range attrs {
100112
ak, av := attrs[i].Key(), attrs[i].Value()
101113
if n, isInt := parseInt(av); isInt && n.Cmp(maxUint256Neg) >= 0 && n.Cmp(maxUint256) <= 0 {
@@ -195,7 +207,7 @@ func (db *DB) search(cnr cid.ID, fs object.SearchFilters, attrs []string, cursor
195207
func (db *DB) searchInBucket(metaBkt *bbolt.Bucket, fs object.SearchFilters, attrs []string,
196208
cursor *SearchCursor, count uint16) ([]client.SearchResultItem, *SearchCursor, error) {
197209
// TODO: make as much as possible outside the Bolt tx
198-
primMatcher := fs[0].Operation()
210+
primMatcher, primVal := convertFilterValue(fs[0])
199211
intPrimMatcher := isNumericOp(primMatcher)
200212
notPresentPrimMatcher := primMatcher == object.MatchNotPresent
201213
primAttr := fs[0].Header() // attribute emptiness already prevented
@@ -223,7 +235,7 @@ func (db *DB) searchInBucket(metaBkt *bbolt.Bucket, fs object.SearchFilters, att
223235
if primMatcher == object.MatchStringEqual || primMatcher == object.MatchCommonPrefix ||
224236
primMatcher == object.MatchNumGT || primMatcher == object.MatchNumGE {
225237
var err error
226-
if primSeekKey, primSeekPrefix, err = seekKeyForAttribute(primAttr, fs[0].Value()); err != nil {
238+
if primSeekKey, primSeekPrefix, err = seekKeyForAttribute(primAttr, primVal); err != nil {
227239
return nil, nil, fmt.Errorf("invalid primary filter value: %w", err)
228240
}
229241
} else {
@@ -279,11 +291,12 @@ nextPrimKey:
279291
if i > 0 && attr != primAttr {
280292
continue
281293
}
282-
checkedDBVal, fltVal, err := combineValues(attr, dbVal, fs[i].Value()) // TODO: deduplicate DB value preparation
294+
mch, val := convertFilterValue(fs[i])
295+
checkedDBVal, fltVal, err := combineValues(attr, dbVal, val) // TODO: deduplicate DB value preparation
283296
if err != nil {
284297
return nil, nil, fmt.Errorf("invalid key in meta bucket: invalid attribute %s value: %w", attr, err)
285298
}
286-
if !matchValues(checkedDBVal, fs[i].Operation(), fltVal) {
299+
if !matchValues(checkedDBVal, mch, fltVal) {
287300
continue nextPrimKey
288301
}
289302
// TODO: attribute value can be requested, it can be collected here, or we can
@@ -311,7 +324,7 @@ nextPrimKey:
311324
if j > 0 && fs[j].Header() != attr {
312325
continue
313326
}
314-
m := fs[j].Operation()
327+
m, val := convertFilterValue(fs[j])
315328
if dbVal == nil {
316329
if m == object.MatchNotPresent {
317330
continue
@@ -347,7 +360,7 @@ nextPrimKey:
347360
} else {
348361
checkedDBVal = dbVal
349362
}
350-
checkedDBVal, fltVal, err := combineValues(attr, checkedDBVal, fs[j].Value()) // TODO: deduplicate DB value preparation
363+
checkedDBVal, fltVal, err := combineValues(attr, checkedDBVal, val) // TODO: deduplicate DB value preparation
351364
if err != nil {
352365
return nil, nil, invalidMetaBucketKeyErr(primKey, fmt.Errorf("invalid attribute %s value: %w", attr, err))
353366
}
@@ -507,7 +520,7 @@ func seekKeyForAttribute(attr, fltVal string) ([]byte, []byte, error) {
507520
return nil, nil, fmt.Errorf("decode %q UUID attribute: %w", attr, err)
508521
}
509522
dbVal = uid[:]
510-
case object.FilterVersion, object.FilterType:
523+
case object.FilterVersion, object.FilterType, object.FilterRoot, object.FilterPhysical:
511524
}
512525
key := make([]byte, 1+len(attr)+utf8DelimiterLen+len(dbVal)) // prefix 1st
513526
key[0] = metaPrefixAttrIDPlain
@@ -794,3 +807,10 @@ func (x *metaAttributeSeeker) restoreVal(id []byte, attr string, stored []byte)
794807
}
795808
return string(stored), nil
796809
}
810+
811+
func convertFilterValue(f object.SearchFilter) (object.SearchMatchType, string) {
812+
if attr := f.Header(); attr == object.FilterRoot || attr == object.FilterPhysical {
813+
return object.MatchStringEqual, binPropMarker
814+
}
815+
return f.Operation(), f.Value()
816+
}

‎pkg/local_object_storage/metabase/metadata_test.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,8 @@ func TestPutMetadata(t *testing.T) {
128128
0, 0, 0, 0, 101, 118, 30, 154, 145, 227, 159, 231})
129129
assertIntAttr(t, mb, id, "$Object:payloadLength", []byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
130130
0, 0, 0, 0, 29, 7, 76, 78, 96, 175, 200, 130})
131+
assertAttr(t, mb, id, "$Object:ROOT", "1")
132+
assertAttr(t, mb, id, "$Object:PHY", "1")
131133
assertAttr(t, mb, id, "attr_1", "val_1")
132134
assertAttr(t, mb, id, "attr_2", "val_2")
133135
assertAttr(t, mb, id, "num_negative_overflow", "-115792089237316195423570985008687907853269984665640564039457584007913129639936")
@@ -670,24 +672,22 @@ func TestDB_SearchObjects(t *testing.T) {
670672
}
671673
})
672674
t.Run("ROOT", func(t *testing.T) {
673-
t.Skip("not supported yet")
674-
check("$Object:ROOT", 0, "", []uint{0, 1})
675675
for _, matcher := range []object.SearchMatchType{
676-
object.MatchStringEqual, object.MatchStringNotEqual, object.MatchNotPresent, object.MatchCommonPrefix,
676+
object.MatchUnspecified, object.MatchStringEqual, object.MatchStringNotEqual,
677677
object.MatchNumGT, object.MatchNumGE, object.MatchNumLT, object.MatchNumLE,
678678
} {
679-
check("$Object:ROOT", matcher, "", nil)
679+
check("$Object:ROOT", matcher, "", []uint{0, 1})
680680
}
681+
check("$Object:ROOT", object.MatchNotPresent, "", nil)
681682
})
682683
t.Run("PHY", func(t *testing.T) {
683-
t.Skip("not supported yet")
684-
check("$Object;PHY", 0, "", []uint{0, 1, 2, 3})
685684
for _, matcher := range []object.SearchMatchType{
686-
object.MatchStringEqual, object.MatchStringNotEqual, object.MatchNotPresent, object.MatchCommonPrefix,
685+
object.MatchUnspecified, object.MatchStringEqual, object.MatchStringNotEqual,
687686
object.MatchNumGT, object.MatchNumGE, object.MatchNumLT, object.MatchNumLE,
688687
} {
689-
check("$Object:PHY", matcher, "", nil)
688+
check("$Object:PHY", matcher, "", []uint{2, 3, 4, 5})
690689
}
690+
check("$Object:PHY", object.MatchNotPresent, "", nil)
691691
})
692692
t.Run("version", func(t *testing.T) {
693693
check := func(m object.SearchMatchType, v string, matchInds []uint) {

‎pkg/local_object_storage/metabase/put.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,8 @@ func (db *DB) put(
111111
return nil
112112
}
113113

114-
if par := obj.Parent(); par != nil && !isParent { // limit depth by two
114+
par := obj.Parent()
115+
if par != nil && !isParent { // limit depth by two
115116
if parID := par.GetID(); !parID.IsZero() { // skip the first object without useful info
116117
parentSI, err := splitInfoFromObject(obj)
117118
if err != nil {
@@ -171,7 +172,7 @@ func (db *DB) put(
171172
pldHmmHash = h.Value()
172173
}
173174
if err := putMetadata(tx, cnr, obj.GetID(), ver, *owner, obj.Type(), obj.CreationEpoch(), obj.PayloadSize(),
174-
pldHash.Value(), pldHmmHash, obj.SplitID().ToV2(), obj.GetParentID(), obj.GetFirstID(), obj.Attributes()); err != nil {
175+
pldHash.Value(), pldHmmHash, obj.SplitID().ToV2(), obj.GetParentID(), obj.GetFirstID(), obj.Attributes(), par == nil, !isParent); err != nil {
175176
return fmt.Errorf("put metadata: %w", err)
176177
}
177178

0 commit comments

Comments
 (0)