Skip to content

fix deserialization issue of Nested data type #1242

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 15, 2023
Merged

fix deserialization issue of Nested data type #1242

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
* incorrect nested array value. [#1221](https://github.com/ClickHouse/clickhouse-java/issues/1221)
* potential endless loop when handling batch update error. [#1233](https://github.com/ClickHouse/clickhouse-java/issues/1233)
* exception when deserializing Array(FixedString(2)) from RowBinary. [#1235](https://github.com/ClickHouse/clickhouse-java/issues/1235)
* deserialization failure of Nested data type. [#1240](https://github.com/ClickHouse/clickhouse-java/issues/1240)

## 0.4.0, 2023-01-19
### Breaking changes
Expand Down
42 changes: 25 additions & 17 deletions clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,25 +132,30 @@ public NestedDeserializer(ClickHouseDataConfig config, ClickHouseColumn column,
}
values = new ClickHouseValue[len];
for (int i = 0; i < len; i++) {
values[i] = nestedCols.get(i).newArrayValue(config);
values[i] = nestedCols.get(i).newValue(config);
}
}

@Override
public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream input) throws IOException {
int size = input.readVarInt();
int len = values.length;
Object[][] vals = new Object[len][];
for (int i = 0; i < len; i++) {
ClickHouseDeserializer d = deserializers[i];
vals[i] = d.deserialize(values[i], input).asArray();
Object[][] vals = new Object[size][];
for (int i = 0; i < size; i++) {
Object[] objs = new Object[len];
for (int j = 0; j < len; j++) {
ClickHouseDeserializer d = deserializers[j];
objs[j] = d.deserialize(values[j], input).asObject();
}
vals[i] = objs;
}
// ClickHouseNestedValue.of(r, c.getNestedColumns(), values)
return ref.update(vals);
}
}

public static class NestedSerializer extends ClickHouseSerializer.CompositeSerializer {
private final ClickHouseArraySequence[] values;
private final ClickHouseValue[] values;

public NestedSerializer(ClickHouseDataConfig config, ClickHouseColumn column,
ClickHouseSerializer... serializers) {
Expand All @@ -162,17 +167,23 @@ public NestedSerializer(ClickHouseDataConfig config, ClickHouseColumn column,
throw new IllegalArgumentException(
ClickHouseUtils.format("Expect %d serializers but got %d", len, serializers.length));
}
values = new ClickHouseArraySequence[len];
values = new ClickHouseValue[len];
for (int i = 0; i < len; i++) {
values[i] = nestedCols.get(i).newArrayValue(config);
values[i] = nestedCols.get(i).newValue(config);
}
}

@Override
public void serialize(ClickHouseValue value, ClickHouseOutputStream output) throws IOException {
Object[][] vals = (Object[][]) value.asObject();
for (int i = 0, len = values.length; i < len; i++) {
serializers[i].serialize(values[i].update(vals[i]), output);
int size = vals.length;
int len = values.length;
output.writeVarInt(size);
for (int i = 0; i < size; i++) {
Object[] objs = vals[i];
for (int j = 0; j < len; j++) {
serializers[j].serialize(values[j].update(objs[j]), output);
}
}
}
}
Expand Down Expand Up @@ -503,7 +514,7 @@ public ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, Click
break;
case Nested:
deserializer = new NestedDeserializer(config, column,
getArrayDeserializers(config, column.getNestedColumns()));
getDeserializers(config, column.getNestedColumns()));
break;
case Tuple:
deserializer = new TupleDeserializer(config, column,
Expand Down Expand Up @@ -657,16 +668,13 @@ public ClickHouseSerializer getSerializer(ClickHouseDataConfig config, ClickHous
getSerializer(config, column.getNestedColumns().get(0)));
break;
case Map:
serializer = new MapSerializer(config, column,
getSerializers(config, column.getNestedColumns()));
serializer = new MapSerializer(config, column, getSerializers(config, column.getNestedColumns()));
break;
case Nested:
serializer = new NestedSerializer(config, column,
getArraySerializers(config, column.getNestedColumns()));
serializer = new NestedSerializer(config, column, getSerializers(config, column.getNestedColumns()));
break;
case Tuple:
serializer = new TupleSerializer(config, column,
getSerializers(config, column.getNestedColumns()));
serializer = new TupleSerializer(config, column, getSerializers(config, column.getNestedColumns()));
break;
// special
case Nothing:
Expand Down
13 changes: 6 additions & 7 deletions clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseNestedValue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,12 @@ protected static Object[][] check(List<ClickHouseColumn> columns, Object[][] val
throw new IllegalArgumentException("Non-null columns and value are required");
}

// if (columns.isEmpty()) {
// throw new IllegalArgumentException("At least one column must be specified for
// nested type");
// }

if (value.length != 0 && value.length != columns.size()) {
throw new IllegalArgumentException("Columns and values should have same length");
int size = columns.size();
for (int i = 0, len = value.length; i < len; i++) {
Object[] objs = value[i];
if (objs == null || objs.length != size) {
throw new IllegalArgumentException("Columns and values should have same length");
}
}

return value;
Expand Down
67 changes: 56 additions & 11 deletions ...house-data/src/test/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.io.IOException;
import java.math.BigInteger;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
Expand Down Expand Up @@ -560,8 +561,9 @@ public void testSerializeMap() throws IOException {
public void testDeserializeNested() throws IOException {
ClickHouseDataConfig config = new ClickHouseTestDataConfig();
ClickHouseValue value = deserialize(null, config,
ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)"),
BinaryStreamUtilsTest.generateInput(1, 1, 1, 0, 1, 0x32, 1, 3, 0));
ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Tuple(x Int16))"),
// two nested rows: values([(1,'a',(0)),(0,'b',(1))])
BinaryStreamUtilsTest.generateInput(2, 1, 0, 1, 0x61, 0, 0, 0, 0, 1, 0x62, 1, 0));
Assert.assertTrue(value instanceof ClickHouseNestedValue);

List<ClickHouseColumn> columns = ((ClickHouseNestedValue) value).getColumns();
Expand All @@ -570,27 +572,28 @@ public void testDeserializeNested() throws IOException {
Assert.assertEquals(columns.get(0).getColumnName(), "n1");
Assert.assertEquals(columns.get(1).getColumnName(), "n2");
Assert.assertEquals(columns.get(2).getColumnName(), "n3");
Assert.assertEquals(values.length, 3);
Assert.assertEquals(values[0], new UnsignedByte[] { UnsignedByte.ONE });
Assert.assertEquals(values[1], new String[] { "2" });
Assert.assertEquals(values[2], new Short[] { (short) 3 });
Assert.assertEquals(values.length, 2);
Assert.assertEquals(values[0],
new Object[] { UnsignedByte.ONE, "a", new ArrayList<>(Collections.singleton((short) 0)) });
Assert.assertEquals(values[1],
new Object[] { UnsignedByte.ZERO, "b", new ArrayList<>(Collections.singleton((short) 1)) });
}

@Test(groups = { "unit" })
public void testSerializeNested() throws IOException {
ClickHouseDataConfig config = new ClickHouseTestDataConfig();
ClickHouseValue value = ClickHouseNestedValue.of(
ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)")
ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Tuple(x Int16))")
.getNestedColumns(),
new Object[][] { new Short[] { Short.valueOf("1") }, new String[] { "2" },
new Short[] { Short.valueOf("3") } });
new Object[][] { { UnsignedByte.ONE, "a", new ArrayList<>(Collections.singleton((short) 0)) },
{ UnsignedByte.ZERO, "b", new ArrayList<>(Collections.singleton((short) 1)) } });
ByteArrayOutputStream bas = new ByteArrayOutputStream();
ClickHouseOutputStream out = ClickHouseOutputStream.of(bas);
serialize(value, config,
ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)"), out);
ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Tuple(x Int16))"), out);
out.flush();
Assert.assertEquals(bas.toByteArray(),
BinaryStreamUtilsTest.generateBytes(1, 1, 1, 0, 1, 0x32, 1, 3, 0));
BinaryStreamUtilsTest.generateBytes(2, 1, 0, 1, 0x61, 0, 0, 0, 0, 1, 0x62, 1, 0));
}

@Test(groups = { "unit" })
Expand Down Expand Up @@ -639,4 +642,46 @@ public void testSerializeTuple() throws IOException {
BinaryStreamUtilsTest.generateBytes(1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0x05, 0xa8, 0xc0));
}

@Test(groups = { "unit" })
public void testDeserializeJson() throws IOException {
ClickHouseDataConfig config = new ClickHouseTestDataConfig();
// '{"n":7,"s":"string","o":{"x":1,"y":[3,2,1]},"a":[{"a":1},{"b":{"c":2}}]}'
// ([(1,(0)),(0,(2))],7,(1,[3,2,1]),'string')
ClickHouseColumn column = ClickHouseColumn.of("t",
"Tuple(a Nested(a Int8, b Tuple(c Int8)), n Int8, o Tuple(x Int8, y Array(Int8)), s String)");
ClickHouseValue value = deserialize(null, config, column, BinaryStreamUtilsTest.generateInput(2, 1, 0, 0, 2, 7,
1, 3, 3, 2, 1, 6, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67));
Assert.assertTrue(value instanceof ClickHouseTupleValue);
List<Object> values = (List<Object>) value.asObject();
Assert.assertEquals(values.size(), 4);
Object[] objs = (Object[]) values.get(0);
Assert.assertEquals(objs.length, 2);
Assert.assertEquals(objs[0], new Object[] { (byte) 1, Arrays.asList((byte) 0) });
Assert.assertEquals(objs[1], new Object[] { (byte) 0, Arrays.asList((byte) 2) });
Assert.assertEquals(values.get(1), (byte) 7);
List<Object> list = (List<Object>) values.get(2);
Assert.assertEquals(list.size(), 2);
Assert.assertEquals(list.get(0), (byte) 1);
Assert.assertEquals(list.get(1), new byte[] { 3, 2, 1 });
Assert.assertEquals(values.get(3), "string");
}

@Test(groups = { "unit" })
public void testSerializeJson() throws IOException {
ClickHouseDataConfig config = new ClickHouseTestDataConfig();
ClickHouseValue value = ClickHouseTupleValue.of(new Object[][] {
{ (byte) 1, Arrays.asList((byte) 0) },
{ (byte) 0, Arrays.asList((byte) 2) }
}, (byte) 7, Arrays.asList((byte) 1, new byte[] { 3, 2, 1 }), "string");
ByteArrayOutputStream bas = new ByteArrayOutputStream();
ClickHouseOutputStream out = ClickHouseOutputStream.of(bas);
serialize(value, config,
ClickHouseColumn.of("t",
"Tuple(a Nested(a Int8, b Tuple(c Int8)), n Int8, o Tuple(x Int8, y Array(Int8)), s String)"),
out);
out.flush();
Assert.assertEquals(bas.toByteArray(), BinaryStreamUtilsTest.generateBytes(2, 1, 0, 0, 2, 7,
1, 3, 3, 2, 1, 6, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67));
}
}
2 changes: 1 addition & 1 deletion clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseNestedValueTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ public void testSingleValue() {
false, 3, 9);
checkNull(
ClickHouseNestedValue.of(ClickHouseColumn.parse("a String not null, b Int8"),
new Object[][] { new String[] { "a" }, new Byte[] { (byte) 1 } }).resetToNullOrEmpty(),
new Object[][] { { "a", (byte) 1 }, { "b", (byte) 2 } }).resetToNullOrEmpty(),
false, 3, 9);

checkValue(
Expand Down