Merge pull request #430 from openaddresses/set-parquet-row-group-size

Set row group size so that the buffer doesn't grow indefinitely
openaddresses · Feb 19, 2025 · 6e4ef24 · 6e4ef24
2 parents 861dfa9 + f63341d
commit 6e4ef24
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/task/collect.js b/task/collect.js
@@ -334,6 +334,7 @@ async function parquet_datas(tmp, datas, name) {
         notes: { type: 'UTF8', optional: true }
     });
     const writer = await parquet.ParquetWriter.openFile(schema, path.resolve(tmp, `${name}.parquet`));
+    writer.setRowGroupSize(16384);
 
     for (const data of datas) {
         const resolved_data_filename = path.resolve(tmp, 'sources', data);