diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cc507855..d9d254fc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * incorrect nested array value. [#1221](https://github.com/ClickHouse/clickhouse-java/issues/1221) * potential endless loop when handling batch update error. [#1233](https://github.com/ClickHouse/clickhouse-java/issues/1233) * exception when deserializing Array(FixedString(2)) from RowBinary. [#1235](https://github.com/ClickHouse/clickhouse-java/issues/1235) +* deserialization failure of Nested data type. [#1240](https://github.com/ClickHouse/clickhouse-java/issues/1240) ## 0.4.0, 2023-01-19 ### Breaking changes diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessor.java index b5f18c06b..296209176 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessor.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessor.java @@ -132,17 +132,22 @@ public NestedDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, } values = new ClickHouseValue[len]; for (int i = 0; i < len; i++) { - values[i] = nestedCols.get(i).newArrayValue(config); + values[i] = nestedCols.get(i).newValue(config); } } @Override public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { + int size = input.readVarInt(); int len = values.length; - Object[][] vals = new Object[len][]; - for (int i = 0; i < len; i++) { - ClickHouseDeserializer d = deserializers[i]; - vals[i] = d.deserialize(values[i], input).asArray(); + Object[][] vals = new Object[size][]; + for (int i = 0; i < size; i++) { + Object[] objs = new Object[len]; + for (int j = 0; j < len; j++) { + ClickHouseDeserializer d = deserializers[j]; + objs[j] = d.deserialize(values[j], input).asObject(); + } + vals[i] = objs; } // ClickHouseNestedValue.of(r, c.getNestedColumns(), values) return ref.update(vals); @@ -150,7 +155,7 @@ public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream in } public static class NestedSerializer extends ClickHouseSerializer.CompositeSerializer { - private final ClickHouseArraySequence[] values; + private final ClickHouseValue[] values; public NestedSerializer(ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseSerializer... serializers) { @@ -162,17 +167,23 @@ public NestedSerializer(ClickHouseDataConfig config, ClickHouseColumn column, throw new IllegalArgumentException( ClickHouseUtils.format("Expect %d serializers but got %d", len, serializers.length)); } - values = new ClickHouseArraySequence[len]; + values = new ClickHouseValue[len]; for (int i = 0; i < len; i++) { - values[i] = nestedCols.get(i).newArrayValue(config); + values[i] = nestedCols.get(i).newValue(config); } } @Override public void serialize(ClickHouseValue value, ClickHouseOutputStream output) throws IOException { Object[][] vals = (Object[][]) value.asObject(); - for (int i = 0, len = values.length; i < len; i++) { - serializers[i].serialize(values[i].update(vals[i]), output); + int size = vals.length; + int len = values.length; + output.writeVarInt(size); + for (int i = 0; i < size; i++) { + Object[] objs = vals[i]; + for (int j = 0; j < len; j++) { + serializers[j].serialize(values[j].update(objs[j]), output); + } } } } @@ -503,7 +514,7 @@ public ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, Click break; case Nested: deserializer = new NestedDeserializer(config, column, - getArrayDeserializers(config, column.getNestedColumns())); + getDeserializers(config, column.getNestedColumns())); break; case Tuple: deserializer = new TupleDeserializer(config, column, @@ -657,16 +668,13 @@ public ClickHouseSerializer getSerializer(ClickHouseDataConfig config, ClickHous getSerializer(config, column.getNestedColumns().get(0))); break; case Map: - serializer = new MapSerializer(config, column, - getSerializers(config, column.getNestedColumns())); + serializer = new MapSerializer(config, column, getSerializers(config, column.getNestedColumns())); break; case Nested: - serializer = new NestedSerializer(config, column, - getArraySerializers(config, column.getNestedColumns())); + serializer = new NestedSerializer(config, column, getSerializers(config, column.getNestedColumns())); break; case Tuple: - serializer = new TupleSerializer(config, column, - getSerializers(config, column.getNestedColumns())); + serializer = new TupleSerializer(config, column, getSerializers(config, column.getNestedColumns())); break; // special case Nothing: diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseNestedValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseNestedValue.java index a8b67b0dd..635b5166f 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseNestedValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseNestedValue.java @@ -67,13 +67,12 @@ protected static Object[][] check(List columns, Object[][] val throw new IllegalArgumentException("Non-null columns and value are required"); } - // if (columns.isEmpty()) { - // throw new IllegalArgumentException("At least one column must be specified for - // nested type"); - // } - - if (value.length != 0 && value.length != columns.size()) { - throw new IllegalArgumentException("Columns and values should have same length"); + int size = columns.size(); + for (int i = 0, len = value.length; i < len; i++) { + Object[] objs = value[i]; + if (objs == null || objs.length != size) { + throw new IllegalArgumentException("Columns and values should have same length"); + } } return value; diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessorTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessorTest.java index e411c9289..12daa6e4f 100644 --- a/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessorTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessorTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.math.BigInteger; import java.net.InetAddress; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; @@ -560,8 +561,9 @@ public void testSerializeMap() throws IOException { public void testDeserializeNested() throws IOException { ClickHouseDataConfig config = new ClickHouseTestDataConfig(); ClickHouseValue value = deserialize(null, config, - ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)"), - BinaryStreamUtilsTest.generateInput(1, 1, 1, 0, 1, 0x32, 1, 3, 0)); + ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Tuple(x Int16))"), + // two nested rows: values([(1,'a',(0)),(0,'b',(1))]) + BinaryStreamUtilsTest.generateInput(2, 1, 0, 1, 0x61, 0, 0, 0, 0, 1, 0x62, 1, 0)); Assert.assertTrue(value instanceof ClickHouseNestedValue); List columns = ((ClickHouseNestedValue) value).getColumns(); @@ -570,27 +572,28 @@ public void testDeserializeNested() throws IOException { Assert.assertEquals(columns.get(0).getColumnName(), "n1"); Assert.assertEquals(columns.get(1).getColumnName(), "n2"); Assert.assertEquals(columns.get(2).getColumnName(), "n3"); - Assert.assertEquals(values.length, 3); - Assert.assertEquals(values[0], new UnsignedByte[] { UnsignedByte.ONE }); - Assert.assertEquals(values[1], new String[] { "2" }); - Assert.assertEquals(values[2], new Short[] { (short) 3 }); + Assert.assertEquals(values.length, 2); + Assert.assertEquals(values[0], + new Object[] { UnsignedByte.ONE, "a", new ArrayList<>(Collections.singleton((short) 0)) }); + Assert.assertEquals(values[1], + new Object[] { UnsignedByte.ZERO, "b", new ArrayList<>(Collections.singleton((short) 1)) }); } @Test(groups = { "unit" }) public void testSerializeNested() throws IOException { ClickHouseDataConfig config = new ClickHouseTestDataConfig(); ClickHouseValue value = ClickHouseNestedValue.of( - ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)") + ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Tuple(x Int16))") .getNestedColumns(), - new Object[][] { new Short[] { Short.valueOf("1") }, new String[] { "2" }, - new Short[] { Short.valueOf("3") } }); + new Object[][] { { UnsignedByte.ONE, "a", new ArrayList<>(Collections.singleton((short) 0)) }, + { UnsignedByte.ZERO, "b", new ArrayList<>(Collections.singleton((short) 1)) } }); ByteArrayOutputStream bas = new ByteArrayOutputStream(); ClickHouseOutputStream out = ClickHouseOutputStream.of(bas); serialize(value, config, - ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)"), out); + ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Tuple(x Int16))"), out); out.flush(); Assert.assertEquals(bas.toByteArray(), - BinaryStreamUtilsTest.generateBytes(1, 1, 1, 0, 1, 0x32, 1, 3, 0)); + BinaryStreamUtilsTest.generateBytes(2, 1, 0, 1, 0x61, 0, 0, 0, 0, 1, 0x62, 1, 0)); } @Test(groups = { "unit" }) @@ -639,4 +642,46 @@ public void testSerializeTuple() throws IOException { BinaryStreamUtilsTest.generateBytes(1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0x05, 0xa8, 0xc0)); } + + @Test(groups = { "unit" }) + public void testDeserializeJson() throws IOException { + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); + // '{"n":7,"s":"string","o":{"x":1,"y":[3,2,1]},"a":[{"a":1},{"b":{"c":2}}]}' + // ([(1,(0)),(0,(2))],7,(1,[3,2,1]),'string') + ClickHouseColumn column = ClickHouseColumn.of("t", + "Tuple(a Nested(a Int8, b Tuple(c Int8)), n Int8, o Tuple(x Int8, y Array(Int8)), s String)"); + ClickHouseValue value = deserialize(null, config, column, BinaryStreamUtilsTest.generateInput(2, 1, 0, 0, 2, 7, + 1, 3, 3, 2, 1, 6, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67)); + Assert.assertTrue(value instanceof ClickHouseTupleValue); + List values = (List) value.asObject(); + Assert.assertEquals(values.size(), 4); + Object[] objs = (Object[]) values.get(0); + Assert.assertEquals(objs.length, 2); + Assert.assertEquals(objs[0], new Object[] { (byte) 1, Arrays.asList((byte) 0) }); + Assert.assertEquals(objs[1], new Object[] { (byte) 0, Arrays.asList((byte) 2) }); + Assert.assertEquals(values.get(1), (byte) 7); + List list = (List) values.get(2); + Assert.assertEquals(list.size(), 2); + Assert.assertEquals(list.get(0), (byte) 1); + Assert.assertEquals(list.get(1), new byte[] { 3, 2, 1 }); + Assert.assertEquals(values.get(3), "string"); + } + + @Test(groups = { "unit" }) + public void testSerializeJson() throws IOException { + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); + ClickHouseValue value = ClickHouseTupleValue.of(new Object[][] { + { (byte) 1, Arrays.asList((byte) 0) }, + { (byte) 0, Arrays.asList((byte) 2) } + }, (byte) 7, Arrays.asList((byte) 1, new byte[] { 3, 2, 1 }), "string"); + ByteArrayOutputStream bas = new ByteArrayOutputStream(); + ClickHouseOutputStream out = ClickHouseOutputStream.of(bas); + serialize(value, config, + ClickHouseColumn.of("t", + "Tuple(a Nested(a Int8, b Tuple(c Int8)), n Int8, o Tuple(x Int8, y Array(Int8)), s String)"), + out); + out.flush(); + Assert.assertEquals(bas.toByteArray(), BinaryStreamUtilsTest.generateBytes(2, 1, 0, 0, 2, 7, + 1, 3, 3, 2, 1, 6, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67)); + } } diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseNestedValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseNestedValueTest.java index 01de693bc..2be183609 100644 --- a/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseNestedValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseNestedValueTest.java @@ -105,7 +105,7 @@ public void testSingleValue() { false, 3, 9); checkNull( ClickHouseNestedValue.of(ClickHouseColumn.parse("a String not null, b Int8"), - new Object[][] { new String[] { "a" }, new Byte[] { (byte) 1 } }).resetToNullOrEmpty(), + new Object[][] { { "a", (byte) 1 }, { "b", (byte) 2 } }).resetToNullOrEmpty(), false, 3, 9); checkValue(