Skip to content

Commit 9123670

Browse files
snazysnazy
committed
Add Varint type for variable-length integer encoding
1 parent c4fe067 commit 9123670

File tree

5 files changed

+229
-0
lines changed

5 files changed

+229
-0
lines changed

‎bom/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ dependencies {
3333
api(project(":polaris-immutables"))
3434
api(project(":polaris-misc-types"))
3535
api(project(":polaris-version"))
36+
api(project(":polaris-persistence-varint"))
3637

3738
api(project(":polaris-config-docs-annotations"))
3839
api(project(":polaris-config-docs-generator"))

‎gradle/projects.main.properties

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ polaris-immutables=tools/immutables
3939
polaris-container-spec-helper=tools/container-spec-helper
4040
polaris-version=tools/version
4141
polaris-misc-types=tools/misc-types
42+
polaris-persistence-varint=nosql/persistence/varint
4243

4344
polaris-config-docs-annotations=tools/config-docs/annotations
4445
polaris-config-docs-generator=tools/config-docs/generator

‎nosql/persistence/varint/build.gradle.kts

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
plugins { id("polaris-server") }
21+
22+
dependencies {
23+
implementation(libs.guava)
24+
25+
testFixturesApi(libs.assertj.core)
26+
}
27+
28+
description = "Provides variable length integer encoding"

‎nosql/persistence/varint/src/main/java/org/apache/polaris/persistence/varint/VarInt.java

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.persistence.varint;
20+
21+
import static com.google.common.base.Preconditions.checkArgument;
22+
23+
import com.google.common.primitives.Ints;
24+
import java.nio.ByteBuffer;
25+
26+
/** Utility class to de-serialize <em>positive</em> integer values in a space efficient way. */
27+
public final class VarInt {
28+
// var-int encoded length of Long.MAX_VALUE
29+
private static final int MAX_LEN = 9;
30+
// 7 bits
31+
private static final int MAX_SHIFT_LEN = 7 * MAX_LEN;
32+
33+
private VarInt() {}
34+
35+
public static int varIntLen(long v) {
36+
checkArgument(v >= 0);
37+
int l = 0;
38+
while (true) {
39+
l++;
40+
if (v <= 0x7f) {
41+
return l;
42+
}
43+
v >>= 7;
44+
}
45+
}
46+
47+
public static ByteBuffer putVarInt(ByteBuffer b, long v) {
48+
checkArgument(v >= 0);
49+
while (true) {
50+
if (v <= 0x7f) {
51+
// Current "byte" is <= 0x7f - encode as is. The highest bit (0x80) is not set, meaning that
52+
// this is the last encoded byte value.
53+
return b.put((byte) v);
54+
}
55+
56+
// Current value is > 0x7f - encode its lower 7 bits and set the "more data follows" flag
57+
// (0x80).
58+
b.put((byte) (v | 0x80));
59+
60+
v >>= 7;
61+
}
62+
}
63+
64+
public static int readVarInt(ByteBuffer b) {
65+
return Ints.checkedCast(readVarLong(b));
66+
}
67+
68+
public static long readVarLong(ByteBuffer b) {
69+
long r = 0;
70+
for (int shift = 0; ; shift += 7) {
71+
checkArgument(shift < MAX_SHIFT_LEN, "Illegal variable length integer representation");
72+
long v = b.get() & 0xff;
73+
r |= (v & 0x7f) << shift;
74+
if ((v & 0x80) == 0) {
75+
break;
76+
}
77+
}
78+
return r;
79+
}
80+
81+
public static void skipVarInt(ByteBuffer b) {
82+
for (int shift = 0; ; shift += 7) {
83+
checkArgument(shift < MAX_SHIFT_LEN, "Illegal variable length integer representation");
84+
int v = b.get() & 0xff;
85+
if ((v & 0x80) == 0) {
86+
break;
87+
}
88+
}
89+
}
90+
}

‎nosql/persistence/varint/src/test/java/org/apache/polaris/persistence/varint/TestVarInt.java

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.persistence.varint;
20+
21+
import static org.junit.jupiter.params.provider.Arguments.arguments;
22+
23+
import java.nio.ByteBuffer;
24+
import java.util.Arrays;
25+
import java.util.stream.Stream;
26+
import org.assertj.core.api.SoftAssertions;
27+
import org.assertj.core.api.junit.jupiter.InjectSoftAssertions;
28+
import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;
29+
import org.junit.jupiter.api.Test;
30+
import org.junit.jupiter.api.extension.ExtendWith;
31+
import org.junit.jupiter.params.ParameterizedTest;
32+
import org.junit.jupiter.params.provider.Arguments;
33+
import org.junit.jupiter.params.provider.MethodSource;
34+
35+
@ExtendWith(SoftAssertionsExtension.class)
36+
public class TestVarInt {
37+
@InjectSoftAssertions SoftAssertions soft;
38+
39+
@Test
40+
public void negative() {
41+
var buf = ByteBuffer.allocate(9);
42+
soft.assertThatIllegalArgumentException().isThrownBy(() -> VarInt.putVarInt(buf, -1L));
43+
soft.assertThatIllegalArgumentException()
44+
.isThrownBy(() -> VarInt.putVarInt(buf, Long.MIN_VALUE));
45+
}
46+
47+
@ParameterizedTest
48+
@MethodSource
49+
public void varInt(long value, byte[] binary) {
50+
var buf = ByteBuffer.allocate(9);
51+
VarInt.putVarInt(buf, value);
52+
soft.assertThat(buf.position()).isEqualTo(binary.length);
53+
soft.assertThat(Arrays.copyOf(buf.array(), buf.position())).containsExactly(binary);
54+
soft.assertThat(VarInt.varIntLen(value)).isEqualTo(binary.length);
55+
56+
var read = buf.duplicate().flip();
57+
VarInt.skipVarInt(read);
58+
soft.assertThat(read.position()).isEqualTo(binary.length);
59+
60+
if (value > Integer.MAX_VALUE) {
61+
soft.assertThatIllegalArgumentException()
62+
.isThrownBy(() -> VarInt.readVarInt(buf.duplicate().flip()))
63+
.withMessageStartingWith("Out of range: ");
64+
soft.assertThat(VarInt.readVarLong(buf.duplicate().flip())).isEqualTo(value);
65+
} else {
66+
soft.assertThat(VarInt.readVarInt(buf.duplicate().flip())).isEqualTo(value);
67+
soft.assertThat(VarInt.readVarLong(buf.duplicate().flip())).isEqualTo(value);
68+
}
69+
}
70+
71+
@Test
72+
public void notVarInt() {
73+
var buf = new byte[] {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
74+
soft.assertThatIllegalArgumentException()
75+
.isThrownBy(() -> VarInt.readVarInt(ByteBuffer.wrap(buf)));
76+
soft.assertThatIllegalArgumentException()
77+
.isThrownBy(() -> VarInt.skipVarInt(ByteBuffer.wrap(buf)));
78+
}
79+
80+
static Stream<Arguments> varInt() {
81+
return Stream.of(
82+
// one byte
83+
arguments(0L, new byte[] {0}),
84+
arguments(1L, new byte[] {1}),
85+
arguments(42L, new byte[] {42}),
86+
arguments(127L, new byte[] {127}),
87+
// 2 bytes
88+
arguments(128L, new byte[] {(byte) 0x80, 1}),
89+
// 21 bite -> 3 x 7 bits
90+
arguments(0x1fffff, new byte[] {-1, -1, 127}),
91+
// 28 bits -> 4 x 7 bits
92+
arguments(0xfffffff, new byte[] {-1, -1, -1, 127}),
93+
// 35 bits -> 5 x 7 bits
94+
arguments(0x7ffffffffL, new byte[] {-1, -1, -1, -1, 127}),
95+
arguments(0x321321321L, new byte[] {-95, -90, -56, -119, 50}),
96+
// 42 bits -> 6 x 7 bits
97+
arguments(0x3ffffffffffL, new byte[] {-1, -1, -1, -1, -1, 127}),
98+
// 49 bits -> 7 x 7 bits
99+
arguments(0x1ffffffffffffL, new byte[] {-1, -1, -1, -1, -1, -1, 127}),
100+
// 56 bits -> 8 x 7 bits
101+
arguments(0xffffffffffffffL, new byte[] {-1, -1, -1, -1, -1, -1, -1, 127}),
102+
arguments(0x32132132132132L, new byte[] {-78, -62, -52, -112, -109, -28, -124, 25}),
103+
// 63 bits -> 9 x 7 bits
104+
arguments(Long.MAX_VALUE, new byte[] {-1, -1, -1, -1, -1, -1, -1, -1, 127}),
105+
arguments(
106+
Long.MAX_VALUE - 0x1111111111111111L,
107+
new byte[] {-18, -35, -69, -9, -18, -35, -69, -9, 110}));
108+
}
109+
}

0 commit comments

Comments
 (0)