Added possibility to set cache batch size (#1034)

* Added possibility to set cache batch size

* Static analyze fixes

* Fixed failing tests

Author: norberttech

phpstan.neon

@@ -1,5 +1,6 @@
 parameters:
     level: 8
+    treatPhpDocTypesAsCertain: false
     checkGenericClassInNonGenericObjectType: false
     checkMissingIterableValueType: false
     paths:

psalm.xml

@@ -38,6 +38,7 @@
         </ignoreFiles>
     </projectFiles>
     <issueHandlers>
+        <DocblockTypeContradiction errorLevel="suppress" />
         <RiskyTruthyFalsyComparison errorLevel="suppress" />
         <LessSpecificReturnStatement errorLevel="suppress" />
         <MoreSpecificReturnType errorLevel="suppress" />

src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVDetector.php

@@ -26,7 +26,6 @@ final class CSVDetector
      */
     public function __construct($resource, ?Option $fallback = new Option(',', '"', '\\'), ?Options $options = null)
     {
-        /** @psalm-suppress DocblockTypeContradiction */
         if (!\is_resource($resource)) {
             throw new InvalidArgumentException('Argument must be a valid resource');
         }

src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/GoogleSheetExtractor.php

@@ -48,9 +48,7 @@ public function extract(FlowContext $context) : \Generator
         /**
          * @var array[] $values
          *
-         * @psalm-suppress RedundantConditionGivenDocblockType, DocblockTypeContradiction
-         *
-         * @phpstan-ignore-next-line
+         * @psalm-suppress RedundantConditionGivenDocblockType
          */
         $values = $response->getValues() ?? [];
 
@@ -116,9 +114,7 @@ function (array $rowData) use ($headers, $shouldPutInputIntoRows) {
             /**
              * @var array[] $values
              *
-             * @psalm-suppress RedundantConditionGivenDocblockType, DocblockTypeContradiction
-             *
-             * @phpstan-ignore-next-line
+             * @psalm-suppress RedundantConditionGivenDocblockType
              */
             $values = $response->getValues() ?? [];
         }

src/core/etl/src/Flow/ETL/Config.php

@@ -4,6 +4,7 @@
 
 namespace Flow\ETL;
 
+use Flow\ETL\Exception\InvalidArgumentException;
 use Flow\ETL\Filesystem\FilesystemStreams;
 use Flow\ETL\Pipeline\Optimizer;
 use Flow\ETL\Row\EntryFactory;
@@ -19,6 +20,9 @@ final class Config
 
     public const EXTERNAL_SORT_MAX_MEMORY_ENV = 'FLOW_EXTERNAL_SORT_MAX_MEMORY';
 
+    /**
+     * @param int<1, max> $cacheBatchSize
+     */
     public function __construct(
         private readonly string $id,
         private readonly Serializer $serializer,
@@ -27,8 +31,12 @@ public function __construct(
         private readonly FilesystemStreams $filesystemStreams,
         private readonly Optimizer $optimizer,
         private readonly bool $putInputIntoRows,
-        private readonly EntryFactory $entryFactory
+        private readonly EntryFactory $entryFactory,
+        private readonly int $cacheBatchSize
     ) {
+        if ($this->cacheBatchSize < 1) {
+            throw new InvalidArgumentException('Cache batch size must be greater than 0');
+        }
     }
 
     public static function builder() : ConfigBuilder
@@ -46,6 +54,14 @@ public function cache() : Cache
         return $this->cache;
     }
 
+    /**
+     * @return int<1, max>
+     */
+    public function cacheBatchSize() : int
+    {
+        return $this->cacheBatchSize;
+    }
+
     public function entryFactory() : EntryFactory
     {
         return $this->entryFactory;

src/core/etl/src/Flow/ETL/ConfigBuilder.php

@@ -5,17 +5,23 @@
 namespace Flow\ETL;
 
 use Flow\ETL\Cache\LocalFilesystemCache;
+use Flow\ETL\Exception\InvalidArgumentException;
 use Flow\ETL\ExternalSort\MemorySort;
 use Flow\ETL\Filesystem\{FilesystemStreams, LocalFilesystem};
 use Flow\ETL\Monitoring\Memory\Unit;
 use Flow\ETL\Pipeline\Optimizer;
 use Flow\ETL\Row\Factory\NativeEntryFactory;
-use Flow\Serializer\{CompressingSerializer, NativePHPSerializer, Serializer};
+use Flow\Serializer\{NativePHPSerializer, Serializer};
 
 final class ConfigBuilder
 {
     private ?Cache $cache;
 
+    /**
+     * @var int<1, max>
+     */
+    private int $cacheBatchSize = 1000;
+
     private ?ExternalSort $externalSort;
 
     private ?Filesystem $filesystem;
@@ -46,7 +52,7 @@ public function build() : Config
     {
         $this->id ??= \uniqid('flow_php', true);
         $entryFactory = new NativeEntryFactory();
-        $this->serializer ??= new CompressingSerializer(new NativePHPSerializer());
+        $this->serializer ??= new NativePHPSerializer();
         $cachePath = \is_string(\getenv(Config::CACHE_DIR_ENV)) && \realpath(\getenv(Config::CACHE_DIR_ENV))
             ? \getenv(Config::CACHE_DIR_ENV)
             : \sys_get_temp_dir() . '/flow_php/cache';
@@ -91,7 +97,8 @@ public function build() : Config
             new FilesystemStreams($this->filesystem),
             $this->optimizer,
             $this->putInputIntoRows,
-            $entryFactory
+            $entryFactory,
+            $this->cacheBatchSize
         );
     }
 
@@ -102,6 +109,20 @@ public function cache(Cache $cache) : self
         return $this;
     }
 
+    /**
+     * @throws InvalidArgumentException
+     */
+    public function cacheBatchSize(int $cacheBatchSize) : self
+    {
+        if ($cacheBatchSize < 1) {
+            throw new InvalidArgumentException('Cache batch size must be greater than 0');
+        }
+
+        $this->cacheBatchSize = $cacheBatchSize;
+
+        return $this;
+    }
+
     public function dontPutInputIntoRows() : self
     {
         $this->putInputIntoRows = false;

src/core/etl/src/Flow/ETL/DataFrame.php

@@ -144,9 +144,16 @@ public function batchSize(int $size) : self
      * @lazy
      *
      * @param null|string $id
+     *
+     * @throws InvalidArgumentException
      */
-    public function cache(?string $id = null) : self
+    public function cache(?string $id = null, ?int $cacheBatchSize = null) : self
     {
+        if ($cacheBatchSize !== null && $cacheBatchSize < 1) {
+            throw new InvalidArgumentException('Cache batch size must be greater than 0');
+        }
+
+        $this->batchSize($cacheBatchSize ?? $this->context->config->cacheBatchSize());
         $this->pipeline = new CachingPipeline($this->pipeline, $id);
 
         return $this;

src/core/etl/src/Flow/ETL/Join/Expression.php

@@ -18,8 +18,6 @@ public function __construct(
     }
 
     /**
-     * @psalm-suppress DocblockTypeContradiction
-     *
      * @param array<string, string>|Comparison $comparison
      */
     public static function on(array|Comparison $comparison, string $joinPrefix = '') : self

src/core/etl/src/Flow/ETL/Monitoring/Memory/Unit.php

@@ -43,10 +43,13 @@ public static function fromString(string $memoryString) : self
 
         switch (\strtoupper($unit)) {
             case 'K':
+            case 'B':
                 return self::fromKb((int) \substr($limit, 0, -1));
             case 'M':
+            case 'MB':
                 return self::fromMb((int) \substr($limit, 0, -1));
             case 'G':
+            case 'GB':
                 return self::fromGb((int) \substr($limit, 0, -1));
 
             default:

src/core/etl/src/Flow/ETL/Pipeline/BatchingPipeline.php

@@ -22,11 +22,6 @@ public function __construct(private readonly Pipeline $pipeline, private readonl
     {
         $this->nextPipeline = $pipeline->cleanCopy();
 
-        /**
-         * @psalm-suppress DocblockTypeContradiction
-         *
-         * @phpstan-ignore-next-line
-         */
         if ($this->size <= 0) {
             throw new InvalidArgumentException('Batch size must be greater than 0, given: ' . $this->size);
         }