Fixed writing nullable parquet values that are marked in the schema a…

…s optional (#1025) * Fixed writing nullable parquet values that are marked in the schema as optional * Fixed failing validation test * Fixed reading empty chunks without snappy extension
flow-php · Mar 28, 2024 · 37a630b · 37a630b
1 parent 4e3e08f
commit 37a630b
Show file tree

Hide file tree

Showing 14 changed files with 222 additions and 135 deletions.
diff --git a/composer.lock b/composer.lock
diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/RLEBitPackedHybrid.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/RLEBitPackedHybrid.php
@@ -37,6 +37,10 @@ public function decodeBitPacked(BinaryReader $reader, int $bitWidth, int $varInt
         $readBytes = $reader->readBytes(\min($remainingByteCount, $totalByteCount));
         $actualByteCount = $readBytes->count();
 
+        if ($actualByteCount === 0) {
+            return;
+        }
+
         $bitMask = (1 << $bitWidth) - 1;
         $byteIndex = 0;
         $currentByte = $readBytes[$byteIndex];

diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/PageReader.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/PageReader.php
@@ -79,7 +79,7 @@ public function readDictionary(FlatColumn $column, PageHeader $pageHeader, Compr
                 (new Codec($this->options))
                     ->decompress(
                         /** @phpstan-ignore-next-line */
-                        \fread($stream, $pageHeader->compressedPageSize()),
+                        $pageHeader->compressedPageSize() === 0 ? '' : \fread($stream, $pageHeader->compressedPageSize()),
                         $codec
                     ),
                 $column,

diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/RowGroupBuilder.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/RowGroupBuilder.php
@@ -5,7 +5,6 @@
 namespace Flow\Parquet\ParquetFile;
 
 use Flow\Parquet\Data\DataConverter;
-use Flow\Parquet\Exception\InvalidArgumentException;
 use Flow\Parquet\ParquetFile\RowGroupBuilder\Validator\{ColumnDataValidator, DisabledValidator};
 use Flow\Parquet\ParquetFile\RowGroupBuilder\{ColumnChunkBuilder, Flattener, PageSizeCalculator, RowGroupContainer, RowGroupStatistics};
 use Flow\Parquet\{Option, Options};
@@ -49,9 +48,6 @@ public function addRow(array $row) : void
         $flatRow = [];
 
         foreach ($this->schema->columns() as $column) {
-            if (!\array_key_exists($column->name(), $row)) {
-                throw new InvalidArgumentException(\sprintf("Column '%s' not found in row", $column->name()));
-            }
             $flatRow[] = $this->flattener->flattenColumn($column, $row);
         }
 

diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/RowGroupBuilder/Flattener.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/RowGroupBuilder/Flattener.php
@@ -18,7 +18,7 @@ public function flattenColumn(Column $column, array $row) : array
         if (!\array_key_exists($column->name(), $row)) {
             $this->validator->validate($column, null);
 
-            return [];
+            return [$column->name() => null];
         }
 
         /**

diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/Column.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/Column.php
@@ -20,6 +20,8 @@ public function isMap() : bool;
 
     public function isMapElement() : bool;
 
+    public function isRequired() : bool;
+
     public function isStruct() : bool;
 
     public function isStructElement() : bool;

diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/FlatColumn.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/FlatColumn.php
@@ -243,6 +243,11 @@ public function isMapElement() : bool
         return false;
     }
 
+    public function isRequired() : bool
+    {
+        return $this->repetition !== Repetition::OPTIONAL;
+    }
+
     public function isStruct() : bool
     {
         return false;

diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/NestedColumn.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/NestedColumn.php
@@ -264,6 +264,11 @@ public function isMapElement() : bool
         return false;
     }
 
+    public function isRequired() : bool
+    {
+        return $this->repetition !== Repetition::OPTIONAL;
+    }
+
     public function isStruct() : bool
     {
         if ($this->isMap()) {