Skip to content

Commit 5009501

Browse files
chenghuichenchenghuichen
committed
avro bugfix
1 parent 9500936 commit 5009501

File tree

2 files changed

+71
-69
lines changed

2 files changed

+71
-69
lines changed

‎paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java‎

Lines changed: 68 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -106,50 +106,53 @@ public void before() throws Exception {
106106
@Test
107107
@EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
108108
public void testJavaWriteReadAppendTable() throws Exception {
109-
Identifier identifier = identifier("mixed_test_append_tablej");
110-
Schema schema =
111-
Schema.newBuilder()
112-
.column("id", DataTypes.INT())
113-
.column("name", DataTypes.STRING())
114-
.column("category", DataTypes.STRING())
115-
.column("value", DataTypes.DOUBLE())
116-
.partitionKeys("category")
117-
.option("dynamic-partition-overwrite", "false")
118-
.build();
119-
120-
catalog.createTable(identifier, schema, true);
121-
Table table = catalog.getTable(identifier);
122-
FileStoreTable fileStoreTable = (FileStoreTable) table;
123-
124-
try (StreamTableWrite write = fileStoreTable.newWrite(commitUser);
125-
InnerTableCommit commit = fileStoreTable.newCommit(commitUser)) {
126-
127-
write.write(createRow4Cols(1, "Apple", "Fruit", 1.5));
128-
write.write(createRow4Cols(2, "Banana", "Fruit", 0.8));
129-
write.write(createRow4Cols(3, "Carrot", "Vegetable", 0.6));
130-
write.write(createRow4Cols(4, "Broccoli", "Vegetable", 1.2));
131-
write.write(createRow4Cols(5, "Chicken", "Meat", 5.0));
132-
write.write(createRow4Cols(6, "Beef", "Meat", 8.0));
133-
134-
commit.commit(0, write.prepareCommit(true, 0));
109+
for (String format : Arrays.asList("parquet", "orc", "avro")) {
110+
Identifier identifier = identifier("mixed_test_append_tablej_" + format);
111+
Schema schema =
112+
Schema.newBuilder()
113+
.column("id", DataTypes.INT())
114+
.column("name", DataTypes.STRING())
115+
.column("category", DataTypes.STRING())
116+
.column("value", DataTypes.DOUBLE())
117+
.partitionKeys("category")
118+
.option("dynamic-partition-overwrite", "false")
119+
.option("file.format", format)
120+
.build();
121+
122+
catalog.createTable(identifier, schema, true);
123+
Table table = catalog.getTable(identifier);
124+
FileStoreTable fileStoreTable = (FileStoreTable) table;
125+
126+
try (StreamTableWrite write = fileStoreTable.newWrite(commitUser);
127+
InnerTableCommit commit = fileStoreTable.newCommit(commitUser)) {
128+
129+
write.write(createRow4Cols(1, "Apple", "Fruit", 1.5));
130+
write.write(createRow4Cols(2, "Banana", "Fruit", 0.8));
131+
write.write(createRow4Cols(3, "Carrot", "Vegetable", 0.6));
132+
write.write(createRow4Cols(4, "Broccoli", "Vegetable", 1.2));
133+
write.write(createRow4Cols(5, "Chicken", "Meat", 5.0));
134+
write.write(createRow4Cols(6, "Beef", "Meat", 8.0));
135+
136+
commit.commit(0, write.prepareCommit(true, 0));
137+
}
138+
139+
List<Split> splits =
140+
new ArrayList<>(fileStoreTable.newSnapshotReader().read().dataSplits());
141+
TableRead read = fileStoreTable.newRead();
142+
List<String> res =
143+
getResult(
144+
read,
145+
splits,
146+
row -> DataFormatTestUtil.toStringNoRowKind(row, table.rowType()));
147+
assertThat(res)
148+
.containsExactlyInAnyOrder(
149+
"1, Apple, Fruit, 1.5",
150+
"2, Banana, Fruit, 0.8",
151+
"3, Carrot, Vegetable, 0.6",
152+
"4, Broccoli, Vegetable, 1.2",
153+
"5, Chicken, Meat, 5.0",
154+
"6, Beef, Meat, 8.0");
135155
}
136-
137-
List<Split> splits =
138-
new ArrayList<>(fileStoreTable.newSnapshotReader().read().dataSplits());
139-
TableRead read = fileStoreTable.newRead();
140-
List<String> res =
141-
getResult(
142-
read,
143-
splits,
144-
row -> DataFormatTestUtil.toStringNoRowKind(row, table.rowType()));
145-
assertThat(res)
146-
.containsExactlyInAnyOrder(
147-
"1, Apple, Fruit, 1.5",
148-
"2, Banana, Fruit, 0.8",
149-
"3, Carrot, Vegetable, 0.6",
150-
"4, Broccoli, Vegetable, 1.2",
151-
"5, Chicken, Meat, 5.0",
152-
"6, Beef, Meat, 8.0");
153156
}
154157

155158
@Test
@@ -357,26 +360,28 @@ public void testPKDeletionVectorWriteMultiBatchRawConvertable() throws Exception
357360
@Test
358361
@EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
359362
public void testReadPkTable() throws Exception {
360-
Identifier identifier = identifier("mixed_test_pk_tablep_parquet");
361-
Table table = catalog.getTable(identifier);
362-
FileStoreTable fileStoreTable = (FileStoreTable) table;
363-
List<Split> splits =
364-
new ArrayList<>(fileStoreTable.newSnapshotReader().read().dataSplits());
365-
TableRead read = fileStoreTable.newRead();
366-
List<String> res =
367-
getResult(
368-
read,
369-
splits,
370-
row -> DataFormatTestUtil.toStringNoRowKind(row, table.rowType()));
371-
System.out.println("Result: " + res);
372-
assertThat(res)
373-
.containsExactlyInAnyOrder(
374-
"1, Apple, Fruit, 1.5",
375-
"2, Banana, Fruit, 0.8",
376-
"3, Carrot, Vegetable, 0.6",
377-
"4, Broccoli, Vegetable, 1.2",
378-
"5, Chicken, Meat, 5.0",
379-
"6, Beef, Meat, 8.0");
363+
for (String format : Arrays.asList("parquet", "orc", "avro")) {
364+
Identifier identifier = identifier("mixed_test_pk_tablep_" + format);
365+
Table table = catalog.getTable(identifier);
366+
FileStoreTable fileStoreTable = (FileStoreTable) table;
367+
List<Split> splits =
368+
new ArrayList<>(fileStoreTable.newSnapshotReader().read().dataSplits());
369+
TableRead read = fileStoreTable.newRead();
370+
List<String> res =
371+
getResult(
372+
read,
373+
splits,
374+
row -> DataFormatTestUtil.toStringNoRowKind(row, table.rowType()));
375+
System.out.println("Result: " + res);
376+
assertThat(res)
377+
.containsExactlyInAnyOrder(
378+
"1, Apple, Fruit, 1.5",
379+
"2, Banana, Fruit, 0.8",
380+
"3, Carrot, Vegetable, 0.6",
381+
"4, Broccoli, Vegetable, 1.2",
382+
"5, Chicken, Meat, 5.0",
383+
"6, Beef, Meat, 8.0");
384+
}
380385
}
381386

382387
// Helper method from TableTestBase

‎paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,9 @@ def table_sort_by(table: pa.Table, column_name: str, order: str = 'ascending') -
3535

3636
def get_file_format_params():
3737
if sys.version_info[:2] == (3, 6):
38-
return [('parquet',)]
38+
return [('parquet',), ('orc',), ('avro',)]
3939
else:
40-
return [('parquet',), ('lance',)]
40+
return [('parquet',), ('orc',), ('avro',), ('lance',)]
4141

4242

4343
class JavaPyReadWriteTest(unittest.TestCase):
@@ -165,10 +165,7 @@ def test_py_write_read_pk_table(self, file_format):
165165
def test_read_pk_table(self, file_format):
166166
# For parquet, read from Java-written table (no format suffix)
167167
# For lance, read from Java-written table (with format suffix)
168-
if file_format == 'parquet':
169-
table_name = 'default.mixed_test_pk_tablej'
170-
else:
171-
table_name = f'default.mixed_test_pk_tablej_{file_format}'
168+
table_name = f'default.mixed_test_pk_tablej_{file_format}'
172169
table = self.catalog.get_table(table_name)
173170
read_builder = table.new_read_builder()
174171
table_scan = read_builder.new_scan()

0 commit comments

Comments
 (0)