Skip to content

Commit c2216fa

Browse files
annrpomannrpom
committed
db: enable new formats
1 parent b928a94 commit c2216fa

23 files changed

+1252
-560
lines changed

‎cockroachkvs/cockroachkvs_bench_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,18 @@ func BenchmarkRandSeekInSST(b *testing.B) {
7979
valueLen: 128, // ~200 KVs per data block
8080
version: sstable.TableFormatPebblev7,
8181
},
82+
{
83+
name: "v8/single-level",
84+
numKeys: 200 * 100, // ~100 data blocks.
85+
valueLen: 128, // ~200 KVs per data block
86+
version: sstable.TableFormatPebblev8,
87+
},
88+
{
89+
name: "v8/two-level",
90+
numKeys: 200 * 5000, // ~5000 data blocks
91+
valueLen: 128, // ~200 KVs per data block
92+
version: sstable.TableFormatPebblev8,
93+
},
8294
}
8395
keyCfg := KeyGenConfig{
8496
PrefixAlphabetLen: 26,

‎format_major_version.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,12 @@ const (
253253
// Previously, marking for compaction required a manifest rotation.
254254
FormatMarkForCompactionInVersionEdit
255255

256+
// FormatTieredStorage is a format major version that adds support for
257+
// tiered storage based on the age of a key-value pair. It introduces a new
258+
// columnar block format (among other things) that is required for tracking
259+
// the attribute used to derive the age.
260+
FormatTieredStorage
261+
256262
// -- Add new versions here --
257263

258264
// FormatNewest is the most recent format major version.
@@ -293,6 +299,8 @@ func (v FormatMajorVersion) resolveDefault() FormatMajorVersion {
293299
func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
294300
v = v.resolveDefault()
295301
switch {
302+
case v >= FormatTieredStorage:
303+
return sstable.TableFormatPebblev8
296304
case v >= formatFooterAttributes:
297305
return sstable.TableFormatPebblev7
298306
case v >= FormatTableFormatV6:
@@ -398,6 +406,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
398406
FormatMarkForCompactionInVersionEdit: func(d *DB) error {
399407
return d.finalizeFormatVersUpgrade(FormatMarkForCompactionInVersionEdit)
400408
},
409+
FormatTieredStorage: func(d *DB) error {
410+
return d.finalizeFormatVersUpgrade(FormatTieredStorage)
411+
},
401412
}
402413

403414
const formatVersionMarkerName = `format-version`
@@ -511,7 +522,11 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
511522
defer func() { d.mu.formatVers.ratcheting = false }()
512523

513524
for nextVers := d.FormatMajorVersion() + 1; nextVers <= formatVers; nextVers++ {
514-
if err := formatMajorVersionMigrations[nextVers](d); err != nil {
525+
migration, ok := formatMajorVersionMigrations[nextVers]
526+
if !ok || migration == nil {
527+
return errors.Errorf("pebble: no migration function defined for format version %d", nextVers)
528+
}
529+
if err := migration(d); err != nil {
515530
return errors.Wrapf(err, "migrating to version %d", nextVers)
516531
}
517532

‎format_major_version_test.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,12 @@ func TestFormatMajorVersionStableValues(t *testing.T) {
3838
require.Equal(t, FormatV2BlobFiles, FormatMajorVersion(26))
3939
require.Equal(t, FormatBackingValueSize, FormatMajorVersion(27))
4040
require.Equal(t, FormatMarkForCompactionInVersionEdit, FormatMajorVersion(28))
41+
require.Equal(t, FormatTieredStorage, FormatMajorVersion(29))
4142

4243
// When we add a new version, we should add a check for the new version above
4344
// in addition to updating the expected values below.
44-
require.Equal(t, FormatNewest, FormatMajorVersion(28))
45-
require.Equal(t, internalFormatNewest, FormatMajorVersion(28))
45+
require.Equal(t, FormatNewest, FormatMajorVersion(29))
46+
require.Equal(t, internalFormatNewest, FormatMajorVersion(29))
4647
}
4748

4849
func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
@@ -233,6 +234,7 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) {
233234
FormatV2BlobFiles: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
234235
FormatBackingValueSize: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
235236
FormatMarkForCompactionInVersionEdit: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
237+
FormatTieredStorage: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev8},
236238
}
237239

238240
// Valid versions.
@@ -260,6 +262,7 @@ func TestFormatMajorVersions_BlobFileFormat(t *testing.T) {
260262
FormatV2BlobFiles: blob.FileFormatV2,
261263
FormatBackingValueSize: blob.FileFormatV2,
262264
FormatMarkForCompactionInVersionEdit: blob.FileFormatV2,
265+
FormatTieredStorage: blob.FileFormatV2,
263266
}
264267

265268
// Valid versions.
@@ -297,7 +300,7 @@ func TestFormatMajorVersions_MaxTableFormat(t *testing.T) {
297300
},
298301
{
299302
fmv: FormatNewest,
300-
want: sstable.TableFormatPebblev7,
303+
want: sstable.TableFormatPebblev8,
301304
},
302305
}
303306
for _, tc := range testCases {

‎open_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -563,7 +563,7 @@ func TestNewDBFilenames(t *testing.T) {
563563
"LOCK",
564564
"MANIFEST-000001",
565565
"OPTIONS-000002",
566-
"marker.format-version.000015.028",
566+
"marker.format-version.000016.029",
567567
"marker.manifest.000001.MANIFEST-000001",
568568
},
569569
}

‎sstable/testdata/rewriter_v8‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
build block-size=1 index-block-size=1 filter
2+
a@xyz#1,SET:a
3+
b@xyz#1,SET:b
4+
c@xyz#1,SET:c
5+
----
6+
point: [a@xyz#1,SET-c@xyz#1,SET]
7+
seqnums: [1-1]
8+
9+
rewrite from=@xyz to=@123 block-size=1 index-block-size=1 filter comparer=split-4b-suffix
10+
----
11+
rewrite failed: mismatched Comparer pebble.internal.testkeys vs comparer-split-4b-suffix, replacement requires same splitter to copy filters
12+
13+
build block-size=1 index-block-size=1 filter
14+
aa@xyz#1,SET:a
15+
ba@xyz#1,SET:b
16+
ca@xyz#1,SET:c
17+
----
18+
point: [aa@xyz#1,SET-ca@xyz#1,SET]
19+
seqnums: [1-1]
20+
21+
rewrite from=yz to=23 block-size=1 index-block-size=1 filter
22+
----
23+
rewrite failed: rewriting data blocks: key aa@xyz#1,SET has suffix 0x4078797a; require 0x797a
24+
25+
rewrite from=a@xyz to=a@123 block-size=1 index-block-size=1 filter
26+
----
27+
rewrite failed: rewriting data blocks: key aa@xyz#1,SET has suffix 0x4078797a; require 0x614078797a
28+
29+
build block-size=1 index-block-size=1 filter
30+
a@0#1,SET:a
31+
b@0#1,SET:b
32+
c@0#1,SET:c
33+
----
34+
point: [a@0#1,SET-c@0#1,SET]
35+
seqnums: [1-1]
36+
37+
layout
38+
----
39+
sstable
40+
├── data offset: 0 length: 87
41+
├── data offset: 92 length: 87
42+
├── data offset: 184 length: 87
43+
├── index offset: 276 length: 38
44+
├── index offset: 319 length: 39
45+
├── index offset: 363 length: 37
46+
├── top-index offset: 405 length: 52
47+
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 462 length: 69
48+
├── properties offset: 536 length: 483
49+
├── meta-index offset: 1024 length: 88
50+
└── footer offset: 1117 length: 61
51+
52+
scan
53+
----
54+
a@0#1,SET:a
55+
b@0#1,SET:b
56+
c@0#1,SET:c
57+
58+
get
59+
b@0
60+
f@0
61+
c@0
62+
----
63+
b
64+
get f@0: pebble: not found
65+
c
66+
67+
rewrite from=@0 to=@123 block-size=1 index-block-size=1 filter
68+
----
69+
point: [a@123#1,SET-c@123#1,SET]
70+
seqnums: [1-1]
71+
72+
layout
73+
----
74+
sstable
75+
├── data offset: 0 length: 87
76+
├── data offset: 92 length: 87
77+
├── data offset: 184 length: 87
78+
├── index offset: 276 length: 40
79+
├── index offset: 321 length: 41
80+
├── index offset: 367 length: 37
81+
├── top-index offset: 409 length: 56
82+
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
83+
├── properties offset: 544 length: 483
84+
├── meta-index offset: 1032 length: 88
85+
└── footer offset: 1125 length: 61
86+
87+
scan
88+
----
89+
a@123#1,SET:a
90+
b@123#1,SET:b
91+
c@123#1,SET:c
92+
93+
get
94+
b@123
95+
f@123
96+
c@123
97+
----
98+
b
99+
get f@123: pebble: not found
100+
c
101+
102+
rewrite from=@123 to=@456 block-size=1 index-block-size=1 filter concurrency=2
103+
----
104+
point: [a@456#1,SET-c@456#1,SET]
105+
seqnums: [1-1]
106+
107+
layout
108+
----
109+
sstable
110+
├── data offset: 0 length: 87
111+
├── data offset: 92 length: 87
112+
├── data offset: 184 length: 87
113+
├── index offset: 276 length: 40
114+
├── index offset: 321 length: 41
115+
├── index offset: 367 length: 37
116+
├── top-index offset: 409 length: 56
117+
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
118+
├── properties offset: 544 length: 483
119+
├── meta-index offset: 1032 length: 88
120+
└── footer offset: 1125 length: 61
121+
122+
scan
123+
----
124+
a@456#1,SET:a
125+
b@456#1,SET:b
126+
c@456#1,SET:c
127+
128+
get
129+
b@456
130+
f@456
131+
c@456
132+
----
133+
b
134+
get f@456: pebble: not found
135+
c
136+
137+
rewrite from=@456 to=@123 block-size=1 index-block-size=1 filter concurrency=3
138+
----
139+
point: [a@123#1,SET-c@123#1,SET]
140+
seqnums: [1-1]
141+
142+
layout
143+
----
144+
sstable
145+
├── data offset: 0 length: 87
146+
├── data offset: 92 length: 87
147+
├── data offset: 184 length: 87
148+
├── index offset: 276 length: 40
149+
├── index offset: 321 length: 41
150+
├── index offset: 367 length: 37
151+
├── top-index offset: 409 length: 56
152+
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 470 length: 69
153+
├── properties offset: 544 length: 483
154+
├── meta-index offset: 1032 length: 88
155+
└── footer offset: 1125 length: 61
156+
157+
scan
158+
----
159+
a@123#1,SET:a
160+
b@123#1,SET:b
161+
c@123#1,SET:c
162+
163+
get
164+
b@123
165+
f@123
166+
c@123
167+
----
168+
b
169+
get f@123: pebble: not found
170+
c
171+
172+
173+
rewrite from=@123 to=@0 block-size=1 index-block-size=1 filter concurrency=4
174+
----
175+
point: [a@0#1,SET-c@0#1,SET]
176+
seqnums: [1-1]
177+
178+
layout
179+
----
180+
sstable
181+
├── data offset: 0 length: 87
182+
├── data offset: 92 length: 87
183+
├── data offset: 184 length: 87
184+
├── index offset: 276 length: 38
185+
├── index offset: 319 length: 39
186+
├── index offset: 363 length: 37
187+
├── top-index offset: 405 length: 52
188+
├── fullfilter.rocksdb.BuiltinBloomFilter offset: 462 length: 69
189+
├── properties offset: 536 length: 483
190+
├── meta-index offset: 1024 length: 88
191+
└── footer offset: 1117 length: 61
192+
193+
scan
194+
----
195+
a@0#1,SET:a
196+
b@0#1,SET:b
197+
c@0#1,SET:c
198+
199+
get
200+
b@0
201+
f@0
202+
c@0
203+
----
204+
b
205+
get f@0: pebble: not found
206+
c
207+
208+
# Rewrite a table that contain only range keys.
209+
210+
build block-size=1 index-block-size=1 filter
211+
Span: a-b:{(#1,RANGEKEYSET,@0)}
212+
Span: b-c:{(#1,RANGEKEYSET,@0)}
213+
Span: c-d:{(#1,RANGEKEYSET,@0)}
214+
----
215+
rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET]
216+
seqnums: [1-1]
217+
218+
scan-range-key
219+
----
220+
a-b:{(#1,RANGEKEYSET,@0)}
221+
b-c:{(#1,RANGEKEYSET,@0)}
222+
c-d:{(#1,RANGEKEYSET,@0)}
223+
224+
rewrite from=@0 to=@123 block-size=1 index-block-size=1 filter
225+
----
226+
rangekey: [a#1,RANGEKEYSET-d#inf,RANGEKEYSET]
227+
seqnums: [1-1]
228+
229+
scan-range-key
230+
----
231+
a-b:{(#1,RANGEKEYSET,@123)}
232+
b-c:{(#1,RANGEKEYSET,@123)}
233+
c-d:{(#1,RANGEKEYSET,@123)}
234+
235+
build block-size=1 index-block-size=1 filter
236+
a#1,SET:a
237+
b#1,SET:b
238+
c#1,SET:c
239+
----
240+
point: [a#1,SET-c#1,SET]
241+
seqnums: [1-1]
242+
243+
rewrite from= to=@123 block-size=1 index-block-size=1 filter
244+
----
245+
point: [a@123#1,SET-c@123#1,SET]
246+
seqnums: [1-1]

0 commit comments

Comments
 (0)