Skip to content

Commit 6cee197

Browse files
recsplit: compact Enum=true representation (#12970)
used wrong `bytesPerRec` It reducing hot random-read parts of `.efi` and some `.idx` ~25% For: #12852 PR is backward/forward compatible
1 parent 2c8003c commit 6cee197

File tree

4 files changed

+11
-5
lines changed

4 files changed

+11
-5
lines changed

erigon-lib/recsplit/index.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,7 @@ func (idx *Index) Lookup(bucketHash, fingerprint uint64) (uint64, bool) {
353353
}
354354
b := gr.ReadNext(idx.golombParam(m))
355355
rec := int(cumKeys) + int(remap16(remix(fingerprint+idx.startSeed[level]+b), m))
356+
356357
pos := 1 + 8 + idx.bytesPerRec*(rec+1)
357358

358359
found := binary.BigEndian.Uint64(idx.data[pos:]) & idx.recMask

erigon-lib/recsplit/recsplit.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -604,7 +604,11 @@ func (rs *RecSplit) Build(ctx context.Context) error {
604604
return fmt.Errorf("write number of keys: %w", err)
605605
}
606606
// Write number of bytes per index record
607-
rs.bytesPerRec = common.BitLenToByteLen(bits.Len64(rs.maxOffset))
607+
if rs.enums {
608+
rs.bytesPerRec = common.BitLenToByteLen(bits.Len64(rs.keysAdded + 1))
609+
} else {
610+
rs.bytesPerRec = common.BitLenToByteLen(bits.Len64(rs.maxOffset))
611+
}
608612
if err = rs.indexW.WriteByte(byte(rs.bytesPerRec)); err != nil {
609613
return fmt.Errorf("write bytes per record: %w", err)
610614
}

erigon-lib/recsplit/recsplit_test.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,8 +149,9 @@ func TestTwoLayerIndex(t *testing.T) {
149149
tmpDir := t.TempDir()
150150
indexFile := filepath.Join(tmpDir, "index")
151151
salt := uint32(1)
152+
N := 2571
152153
rs, err := NewRecSplit(RecSplitArgs{
153-
KeyCount: 100,
154+
KeyCount: N,
154155
BucketSize: 10,
155156
Salt: &salt,
156157
TmpDir: tmpDir,
@@ -162,7 +163,7 @@ func TestTwoLayerIndex(t *testing.T) {
162163
if err != nil {
163164
t.Fatal(err)
164165
}
165-
for i := 0; i < 100; i++ {
166+
for i := 0; i < N; i++ {
166167
if err = rs.AddKey([]byte(fmt.Sprintf("key %d", i)), uint64(i*17)); err != nil {
167168
t.Fatal(err)
168169
}
@@ -173,7 +174,7 @@ func TestTwoLayerIndex(t *testing.T) {
173174

174175
idx := MustOpen(indexFile)
175176
defer idx.Close()
176-
for i := 0; i < 100; i++ {
177+
for i := 0; i < N; i++ {
177178
reader := NewIndexReader(idx)
178179
e, _ := reader.Lookup([]byte(fmt.Sprintf("key %d", i)))
179180
if e != uint64(i) {

turbo/app/snapshots_cmd.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ func joinFlags(lists ...[]cli.Flag) (res []cli.Flag) {
8888

8989
var snapshotCommand = cli.Command{
9090
Name: "seg",
91-
Aliases: []string{"snapshots"},
91+
Aliases: []string{"snapshots", "segments"},
9292
Usage: `Managing historical data segments (partitions)`,
9393
Before: func(cliCtx *cli.Context) error {
9494
go mem.LogMemStats(cliCtx.Context, log.New())

0 commit comments

Comments
 (0)