Skip to content

Commit 98e1b2e

Browse files
chenghuichenchenghuichen
committed
support datatype of ROW/STRUCT
1 parent 7f5306f commit 98e1b2e

File tree

2 files changed

+34
-27
lines changed

2 files changed

+34
-27
lines changed

‎paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java‎

Lines changed: 34 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -220,18 +220,15 @@ public void testJavaWriteReadPkTable() throws Exception {
220220
new ArrayList<>(fileStoreTable.newSnapshotReader().read().dataSplits());
221221
TableRead read = fileStoreTable.newRead();
222222
List<String> res =
223-
getResult(
224-
read,
225-
splits,
226-
row -> DataFormatTestUtil.toStringNoRowKind(row, table.rowType()));
223+
getResult(read, splits, row -> rowToStringWithStruct(row, table.rowType()));
227224
assertThat(res)
228225
.containsExactlyInAnyOrder(
229-
"1, Apple, Fruit, 1.5, 1970-01-01T00:16:40, 1970-01-01T00:33:20, +I[store1, 1001, +I[Beijing, China]]",
230-
"2, Banana, Fruit, 0.8, 1970-01-01T00:16:40.001, 1970-01-01T00:33:20.001, +I[store1, 1002, +I[Shanghai, China]]",
231-
"3, Carrot, Vegetable, 0.6, 1970-01-01T00:16:40.002, 1970-01-01T00:33:20.002, +I[store2, 1003, +I[Tokyo, Japan]]",
232-
"4, Broccoli, Vegetable, 1.2, 1970-01-01T00:16:40.003, 1970-01-01T00:33:20.003, +I[store2, 1004, +I[Seoul, Korea]]",
233-
"5, Chicken, Meat, 5.0, 1970-01-01T00:16:40.004, 1970-01-01T00:33:20.004, +I[store3, 1005, +I[NewYork, USA]]",
234-
"6, Beef, Meat, 8.0, 1970-01-01T00:16:40.005, 1970-01-01T00:33:20.005, +I[store3, 1006, +I[London, UK]]");
226+
"+I[1, Apple, Fruit, 1.5, 1970-01-01T00:16:40, 1970-01-01T00:33:20, (store1, 1001, (Beijing, China))]",
227+
"+I[2, Banana, Fruit, 0.8, 1970-01-01T00:16:40.001, 1970-01-01T00:33:20.001, (store1, 1002, (Shanghai, China))]",
228+
"+I[3, Carrot, Vegetable, 0.6, 1970-01-01T00:16:40.002, 1970-01-01T00:33:20.002, (store2, 1003, (Tokyo, Japan))]",
229+
"+I[4, Broccoli, Vegetable, 1.2, 1970-01-01T00:16:40.003, 1970-01-01T00:33:20.003, (store2, 1004, (Seoul, Korea))]",
230+
"+I[5, Chicken, Meat, 5.0, 1970-01-01T00:16:40.004, 1970-01-01T00:33:20.004, (store3, 1005, (NewYork, USA))]",
231+
"+I[6, Beef, Meat, 8.0, 1970-01-01T00:16:40.005, 1970-01-01T00:33:20.005, (store3, 1006, (London, UK))]");
235232
}
236233
}
237234

@@ -380,10 +377,7 @@ public void testReadPkTable() throws Exception {
380377
new ArrayList<>(fileStoreTable.newSnapshotReader().read().dataSplits());
381378
TableRead read = fileStoreTable.newRead();
382379
List<String> res =
383-
getResult(
384-
read,
385-
splits,
386-
row -> DataFormatTestUtil.toStringNoRowKind(row, table.rowType()));
380+
getResult(read, splits, row -> rowToStringWithStruct(row, table.rowType()));
387381
System.out.println("Result for " + format + " : " + res);
388382
assertThat(table.rowType().getFieldTypes().get(4)).isEqualTo(DataTypes.TIMESTAMP());
389383
assertThat(table.rowType().getFieldTypes().get(5))
@@ -393,12 +387,12 @@ public void testReadPkTable() throws Exception {
393387
assertThat(metadataType.getFieldTypes().get(2)).isInstanceOf(RowType.class);
394388
assertThat(res)
395389
.containsExactlyInAnyOrder(
396-
"1, Apple, Fruit, 1.5, 1970-01-01T00:16:40, 1970-01-01T00:33:20, +I[store1, 1001, +I[Beijing, China]]",
397-
"2, Banana, Fruit, 0.8, 1970-01-01T00:16:40.001, 1970-01-01T00:33:20.001, +I[store1, 1002, +I[Shanghai, China]]",
398-
"3, Carrot, Vegetable, 0.6, 1970-01-01T00:16:40.002, 1970-01-01T00:33:20.002, +I[store2, 1003, +I[Tokyo, Japan]]",
399-
"4, Broccoli, Vegetable, 1.2, 1970-01-01T00:16:40.003, 1970-01-01T00:33:20.003, +I[store2, 1004, +I[Seoul, Korea]]",
400-
"5, Chicken, Meat, 5.0, 1970-01-01T00:16:40.004, 1970-01-01T00:33:20.004, +I[store3, 1005, +I[NewYork, USA]]",
401-
"6, Beef, Meat, 8.0, 1970-01-01T00:16:40.005, 1970-01-01T00:33:20.005, +I[store3, 1006, +I[London, UK]]");
390+
"+I[1, Apple, Fruit, 1.5, 1970-01-01T00:16:40, 1970-01-01T00:33:20, (store1, 1001, (Beijing, China))]",
391+
"+I[2, Banana, Fruit, 0.8, 1970-01-01T00:16:40.001, 1970-01-01T00:33:20.001, (store1, 1002, (Shanghai, China))]",
392+
"+I[3, Carrot, Vegetable, 0.6, 1970-01-01T00:16:40.002, 1970-01-01T00:33:20.002, (store2, 1003, (Tokyo, Japan))]",
393+
"+I[4, Broccoli, Vegetable, 1.2, 1970-01-01T00:16:40.003, 1970-01-01T00:33:20.003, (store2, 1004, (Seoul, Korea))]",
394+
"+I[5, Chicken, Meat, 5.0, 1970-01-01T00:16:40.004, 1970-01-01T00:33:20.004, (store3, 1005, (NewYork, USA))]",
395+
"+I[6, Beef, Meat, 8.0, 1970-01-01T00:16:40.005, 1970-01-01T00:33:20.005, (store3, 1006, (London, UK))]");
402396
}
403397
}
404398

@@ -463,4 +457,24 @@ protected GenericRow createRow3Cols(Object... values) {
463457
protected GenericRow createRow3ColsWithKind(RowKind rowKind, Object... values) {
464458
return GenericRow.ofKind(rowKind, values[0], values[1], values[2]);
465459
}
460+
461+
private static String rowToStringWithStruct(InternalRow row, RowType type) {
462+
StringBuilder build = new StringBuilder();
463+
build.append(row.getRowKind().shortString()).append("[");
464+
for (int i = 0; i < type.getFieldCount(); i++) {
465+
if (i != 0) {
466+
build.append(", ");
467+
}
468+
if (row.isNullAt(i)) {
469+
build.append("NULL");
470+
} else {
471+
InternalRow.FieldGetter fieldGetter =
472+
InternalRow.createFieldGetter(type.getTypeAt(i), i);
473+
Object field = fieldGetter.getFieldOrNull(row);
474+
build.append(DataFormatTestUtil.getDataFieldString(field, type.getTypeAt(i)));
475+
}
476+
}
477+
build.append("]");
478+
return build.toString();
479+
}
466480
}

‎paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -234,12 +234,6 @@ def test_read_pk_table(self, file_format):
234234
self.assertEqual(metadata_fields[1].name, 'created_at')
235235
self.assertEqual(metadata_fields[2].name, 'location')
236236
self.assertIsInstance(metadata_fields[2].type, RowType)
237-
else:
238-
from pypaimon.schema.data_types import RowType
239-
self.assertIsInstance(table.fields[4].type, RowType)
240-
metadata_fields = table.fields[4].type.fields
241-
self.assertEqual(len(metadata_fields), 3)
242-
self.assertIsInstance(metadata_fields[2].type, RowType)
243237

244238
# Data order may vary due to partitioning/bucketing, so compare as sets
245239
expected_names = {'Apple', 'Banana', 'Carrot', 'Broccoli', 'Chicken', 'Beef'}
@@ -249,7 +243,6 @@ def test_read_pk_table(self, file_format):
249243
# Verify metadata column can be read and contains nested structures
250244
if 'metadata' in res.columns:
251245
self.assertFalse(res['metadata'].isnull().all())
252-
print(f"Format: {file_format}, Metadata column sample: {res['metadata'].iloc[0]}")
253246

254247
# For primary key tables, verify that _VALUE_KIND is written correctly
255248
# by checking if we can read the raw data with system fields

0 commit comments

Comments
 (0)