Skip to content

Commit 8554b6d

Browse files
roman-khimovroman-khimov
authored
Serve ObjectService.SearchV2 RPC (#3111)
2 parents 93cf0b0 + 2b89d17 commit 8554b6d

File tree

12 files changed

+858
-30
lines changed

12 files changed

+858
-30
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ Changelog for NeoFS Node
1111
- `logger.timestamp` config option (#3105)
1212
- Container system attributes verification on IR side (#3107)
1313
- IR `fschain.consensus.rpc.max_websocket_clients` and `fschain.consensus.rpc.session_pool_size` config options (#3126)
14+
- `ObjectService.SearchV2` SN API (#3111)
1415

1516
### Fixed
1617
- `neofs-cli object delete` command output (#3056)

‎cmd/neofs-node/object.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
containercore "github.com/nspcc-dev/neofs-node/pkg/core/container"
1616
"github.com/nspcc-dev/neofs-node/pkg/core/netmap"
1717
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
18+
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
1819
morphClient "github.com/nspcc-dev/neofs-node/pkg/morph/client"
1920
cntClient "github.com/nspcc-dev/neofs-node/pkg/morph/client/container"
2021
objectService "github.com/nspcc-dev/neofs-node/pkg/services/object"
@@ -303,10 +304,11 @@ func initObjectService(c *cfg) {
303304
)
304305

305306
storage := storageForObjectService{
307+
local: ls,
306308
putSvc: sPut,
307309
keys: keyStorage,
308310
}
309-
server := objectService.New(objSvc, mNumber, fsChain, storage, c.shared.basics.key.PrivateKey, c.metricsCollector, aclChecker, aclSvc)
311+
server := objectService.New(objSvc, mNumber, fsChain, storage, c.shared.basics.key.PrivateKey, c.metricsCollector, aclChecker, aclSvc, coreConstructor)
310312

311313
for _, srv := range c.cfgGRPC.servers {
312314
protoobject.RegisterObjectServiceServer(srv, server)
@@ -603,6 +605,11 @@ func (x *fsChainForObjects) ForEachContainerNodePublicKeyInLastTwoEpochs(id cid.
603605
return x.containerNodes.forEachContainerNodePublicKeyInLastTwoEpochs(id, f)
604606
}
605607

608+
// ForEachContainerNode implements [objectService.FSChain] interface.
609+
func (x *fsChainForObjects) ForEachContainerNode(cnr cid.ID, f func(netmapsdk.NodeInfo) bool) error {
610+
return x.containerNodes.forEachContainerNode(cnr, false, f)
611+
}
612+
606613
// IsOwnPublicKey checks whether given binary-encoded public key is assigned to
607614
// local storage node in the network map.
608615
//
@@ -616,10 +623,16 @@ func (x *fsChainForObjects) IsOwnPublicKey(pubKey []byte) bool {
616623
func (x *fsChainForObjects) LocalNodeUnderMaintenance() bool { return x.isMaintenance.Load() }
617624

618625
type storageForObjectService struct {
626+
local *engine.StorageEngine
619627
putSvc *putsvc.Service
620628
keys *util.KeyStorage
621629
}
622630

631+
// SearchObjects implements [objectService.Storage] interface.
632+
func (x storageForObjectService) SearchObjects(cnr cid.ID, fs objectSDK.SearchFilters, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, *meta.SearchCursor, error) {
633+
return x.local.Search(cnr, fs, attrs, cursor, count)
634+
}
635+
623636
func (x storageForObjectService) VerifyAndStoreObjectLocally(obj objectSDK.Object) error {
624637
return x.putSvc.ValidateAndStoreObjectLocally(obj)
625638
}

‎pkg/core/object/metadata.go

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
package object
2+
3+
import (
4+
"bytes"
5+
"math/big"
6+
"slices"
7+
"strings"
8+
9+
"github.com/nspcc-dev/neofs-sdk-go/client"
10+
)
11+
12+
// TODO: docs.
13+
func MergeSearchResults(lim uint16, withAttr bool, sets [][]client.SearchResultItem, mores []bool) ([]client.SearchResultItem, bool) {
14+
if lim == 0 || len(sets) == 0 {
15+
return nil, false
16+
}
17+
if len(sets) == 1 {
18+
n := min(uint16(len(sets[0])), lim)
19+
return sets[0][:n], n < lim || slices.Contains(mores, true)
20+
}
21+
lim = calcMaxUniqueSearchResults(lim, sets)
22+
res := make([]client.SearchResultItem, 0, lim)
23+
var more bool
24+
var minInt *big.Int
25+
for minInd := -1; ; minInd, minInt = -1, nil {
26+
for i := range sets {
27+
if len(sets[i]) == 0 {
28+
continue
29+
}
30+
if minInd < 0 {
31+
minInd = i
32+
if withAttr {
33+
minInt, _ = new(big.Int).SetString(sets[i][0].Attributes[0], 10)
34+
}
35+
continue
36+
}
37+
cmpID := bytes.Compare(sets[i][0].ID[:], sets[minInd][0].ID[:])
38+
if cmpID == 0 {
39+
continue
40+
}
41+
if withAttr {
42+
var curInt *big.Int
43+
if minInt != nil {
44+
curInt, _ = new(big.Int).SetString(sets[i][0].Attributes[0], 10)
45+
}
46+
var cmpAttr int
47+
if curInt != nil {
48+
cmpAttr = curInt.Cmp(minInt)
49+
} else {
50+
cmpAttr = strings.Compare(sets[i][0].Attributes[0], sets[minInd][0].Attributes[0])
51+
}
52+
if cmpAttr != 0 {
53+
if cmpAttr < 0 {
54+
minInd = i
55+
if minInt != nil {
56+
minInt = curInt
57+
} else {
58+
minInt, _ = new(big.Int).SetString(sets[i][0].Attributes[0], 10)
59+
}
60+
}
61+
continue
62+
}
63+
}
64+
if cmpID < 0 {
65+
minInd = i
66+
if withAttr {
67+
minInt, _ = new(big.Int).SetString(sets[minInd][0].Attributes[0], 10)
68+
}
69+
}
70+
}
71+
if minInd < 0 {
72+
break
73+
}
74+
res = append(res, sets[minInd][0])
75+
if uint16(len(res)) == lim {
76+
if more = len(sets[minInd]) > 1 || slices.Contains(mores, true); !more {
77+
loop:
78+
for i := range sets {
79+
if i == minInd {
80+
continue
81+
}
82+
for j := range sets[i] {
83+
if more = sets[i][j].ID != sets[minInd][0].ID; more {
84+
break loop
85+
}
86+
}
87+
}
88+
}
89+
break
90+
}
91+
for i := range sets {
92+
if i == minInd {
93+
continue
94+
}
95+
for j := range sets[i] {
96+
if sets[i][j].ID == sets[minInd][0].ID {
97+
sets[i] = sets[i][j+1:]
98+
break
99+
}
100+
}
101+
}
102+
sets[minInd] = sets[minInd][1:]
103+
}
104+
return res, more
105+
}
106+
107+
func calcMaxUniqueSearchResults(lim uint16, sets [][]client.SearchResultItem) uint16 {
108+
n := uint16(len(sets[0]))
109+
if n >= lim {
110+
return lim
111+
}
112+
for i := 1; i < len(sets); i++ {
113+
nextItem:
114+
for j := range sets[i] {
115+
for k := range i {
116+
for l := range sets[k] {
117+
if sets[k][l].ID == sets[i][j].ID {
118+
continue nextItem
119+
}
120+
}
121+
}
122+
if n++; n == lim {
123+
return n
124+
}
125+
}
126+
}
127+
return n
128+
}

‎pkg/core/object/metadata_test.go

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
package object_test
2+
3+
import (
4+
"bytes"
5+
"slices"
6+
"strconv"
7+
"testing"
8+
9+
. "github.com/nspcc-dev/neofs-node/pkg/core/object"
10+
"github.com/nspcc-dev/neofs-sdk-go/client"
11+
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
12+
"github.com/stretchr/testify/require"
13+
)
14+
15+
func searchResultFromIDs(n int) []client.SearchResultItem {
16+
ids := oidtest.IDs(n)
17+
s := make([]client.SearchResultItem, len(ids))
18+
for i := range ids {
19+
s[i].ID = ids[i]
20+
}
21+
slices.SortFunc(s, func(a, b client.SearchResultItem) int { return bytes.Compare(a.ID[:], b.ID[:]) })
22+
return s
23+
}
24+
25+
func assertMergeResult(t testing.TB, res, expRes []client.SearchResultItem, more, expMore bool) {
26+
require.Len(t, res, len(expRes))
27+
require.EqualValues(t, len(expRes), cap(res))
28+
require.Equal(t, expRes, res)
29+
require.Equal(t, expMore, more)
30+
}
31+
32+
func TestMergeSearchResults(t *testing.T) {
33+
t.Run("zero limit", func(t *testing.T) {
34+
res, more := MergeSearchResults(0, false, [][]client.SearchResultItem{searchResultFromIDs(2)}, nil)
35+
require.Nil(t, res)
36+
require.False(t, more)
37+
})
38+
t.Run("no sets", func(t *testing.T) {
39+
res, more := MergeSearchResults(1000, false, nil, nil)
40+
require.Nil(t, res)
41+
require.False(t, more)
42+
})
43+
t.Run("empty sets only", func(t *testing.T) {
44+
res, more := MergeSearchResults(1000, false, make([][]client.SearchResultItem, 1000), nil)
45+
require.Empty(t, res)
46+
require.False(t, more)
47+
})
48+
t.Run("with empty sets", func(t *testing.T) {
49+
all := []client.SearchResultItem{
50+
{ID: oidtest.ID(), Attributes: []string{"12", "d"}},
51+
{ID: oidtest.ID(), Attributes: []string{"23", "c"}},
52+
{ID: oidtest.ID(), Attributes: []string{"34", "b"}},
53+
{ID: oidtest.ID(), Attributes: []string{"45", "a"}},
54+
}
55+
sets := [][]client.SearchResultItem{
56+
nil,
57+
{all[0], all[2]},
58+
{},
59+
{all[1], all[3]},
60+
nil,
61+
all,
62+
}
63+
res, more := MergeSearchResults(1000, true, sets, nil)
64+
assertMergeResult(t, res, all, more, false)
65+
})
66+
t.Run("concat", func(t *testing.T) {
67+
t.Run("no attributes", func(t *testing.T) {
68+
all := searchResultFromIDs(10)
69+
var sets [][]client.SearchResultItem
70+
for i := range len(all) / 2 {
71+
sets = append(sets, []client.SearchResultItem{all[2*i], all[2*i+1]})
72+
}
73+
res, more := MergeSearchResults(1000, false, sets, nil)
74+
assertMergeResult(t, res, all, more, false)
75+
t.Run("reverse", func(t *testing.T) {
76+
var sets [][]client.SearchResultItem
77+
for i := range len(all) / 2 {
78+
sets = append(sets, []client.SearchResultItem{all[2*i], all[2*i+1]})
79+
}
80+
slices.Reverse(sets)
81+
res, more := MergeSearchResults(1000, false, sets, nil)
82+
assertMergeResult(t, res, all, more, false)
83+
})
84+
})
85+
t.Run("with attributes", func(t *testing.T) {
86+
all := searchResultFromIDs(10)
87+
slices.Reverse(all)
88+
for i := range all {
89+
all[i].Attributes = []string{strconv.Itoa(i)}
90+
}
91+
var sets [][]client.SearchResultItem
92+
for i := range len(all) / 2 {
93+
sets = append(sets, []client.SearchResultItem{all[2*i], all[2*i+1]})
94+
}
95+
res, more := MergeSearchResults(1000, true, sets, nil)
96+
assertMergeResult(t, res, all, more, false)
97+
t.Run("reverse", func(t *testing.T) {
98+
var sets [][]client.SearchResultItem
99+
for i := range len(all) / 2 {
100+
sets = append(sets, []client.SearchResultItem{all[2*i], all[2*i+1]})
101+
}
102+
slices.Reverse(sets)
103+
res, more := MergeSearchResults(1000, true, sets, nil)
104+
assertMergeResult(t, res, all, more, false)
105+
})
106+
})
107+
})
108+
t.Run("intersecting", func(t *testing.T) {
109+
all := searchResultFromIDs(10)
110+
var sets [][]client.SearchResultItem
111+
for i := range len(all) - 1 {
112+
sets = append(sets, []client.SearchResultItem{all[i], all[i+1]})
113+
}
114+
res, more := MergeSearchResults(1000, false, sets, nil)
115+
assertMergeResult(t, res, all, more, false)
116+
t.Run("with attributes", func(t *testing.T) {
117+
all := searchResultFromIDs(10)
118+
slices.Reverse(all)
119+
for i := range all {
120+
all[i].Attributes = []string{strconv.Itoa(i)}
121+
}
122+
var sets [][]client.SearchResultItem
123+
for i := range len(all) - 1 {
124+
sets = append(sets, []client.SearchResultItem{all[i], all[i+1]})
125+
}
126+
res, more := MergeSearchResults(1000, true, sets, nil)
127+
assertMergeResult(t, res, all, more, false)
128+
})
129+
})
130+
t.Run("cursors", func(t *testing.T) {
131+
all := searchResultFromIDs(10)
132+
t.Run("more items in last set", func(t *testing.T) {
133+
res, more := MergeSearchResults(5, false, [][]client.SearchResultItem{
134+
all[:3],
135+
all[:6],
136+
all[:2],
137+
}, nil)
138+
assertMergeResult(t, res, all[:5], more, true)
139+
})
140+
t.Run("more items in other set", func(t *testing.T) {
141+
res, more := MergeSearchResults(5, false, [][]client.SearchResultItem{
142+
all[:3],
143+
all[:5],
144+
all,
145+
}, nil)
146+
assertMergeResult(t, res, all[:5], more, true)
147+
})
148+
t.Run("flag", func(t *testing.T) {
149+
res, more := MergeSearchResults(5, false, [][]client.SearchResultItem{
150+
all[:1],
151+
all[:5],
152+
all[:2],
153+
}, []bool{
154+
true,
155+
false,
156+
false,
157+
})
158+
assertMergeResult(t, res, all[:5], more, true)
159+
})
160+
})
161+
t.Run("integers", func(t *testing.T) {
162+
vals := []string{
163+
"-111111111111111111111111111111111111111111111111111111",
164+
"-18446744073709551615",
165+
"-1", "0", "1",
166+
"18446744073709551615",
167+
"111111111111111111111111111111111111111111111111111111",
168+
}
169+
all := searchResultFromIDs(len(vals))
170+
slices.Reverse(all)
171+
for i := range all {
172+
all[i].Attributes = []string{vals[i]}
173+
}
174+
for _, sets := range [][][]client.SearchResultItem{
175+
{all},
176+
{all[:len(all)/2], all[len(all)/2:]},
177+
{all[len(all)/2:], all[:len(all)/2]},
178+
{all[6:7], all[0:1], all[5:6], all[1:2], all[4:5], all[2:3], all[3:4]},
179+
{all[5:], all[1:3], all[0:4], all[3:]},
180+
} {
181+
res, more := MergeSearchResults(uint16(len(all)), true, sets, nil)
182+
assertMergeResult(t, res, all, more, false)
183+
}
184+
})
185+
}

0 commit comments

Comments
 (0)