Extractor test helper (#1291)

* Unify how we generate an array from Extractors

* Use the new test case

* Clean-up files

* Add methods for modifying and asserting rows

* Use the new methods

* Add method for multi-row counting

* Use the methods in the rest of the files

* Import function

* Leave this class pure to test the implementation

* Import of build-in function is forbidden

* Convert helper function to an assertion

* Remove unused method

* Convert helper method to an assertion

* Fix code style

* Follow the naming conventions of PHPUnit

* Use a more descriptive name

* Split logic into 2 methods

* Rename test case and fix code style issues

* Revert "Rename test case and fix code style issues"

This reverts commit 43a88c8.

* Rename test case

* Fix code style

* Improve assertion naming

* Assert by flatten first the rows

* Use the same implementation

* Allow to override default flow context

* Allow to override default error message

* Make methods static and final as PHPUnit does

* Move class into the ETL root

* Integration should extend the base FlowTestCase

* Use DSL functions instead of objects

* Use self instead of directly accessing the FlowTestCase

---------

Co-authored-by: Norbert Orzechowicz <[email protected]>

src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/Loader/XMLLoaderTest.php

@@ -7,9 +7,9 @@
 use function Flow\ETL\Adapter\XML\{from_xml, to_xml};
 use function Flow\ETL\DSL\{df, from_array, overwrite, ref};
 use Flow\ETL\Tests\Double\FakeExtractor;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-final class XMLLoaderTest extends IntegrationTestCase
+final class XMLLoaderTest extends FlowIntegrationTestCase
 {
     public function test_partitioning_xml_file() : void
     {

src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php

@@ -12,10 +12,10 @@
     Flow,
     FlowContext,
     PHP\Type\Caster,
-    Tests\Integration\IntegrationTestCase};
+    Tests\Integration\FlowIntegrationTestCase};
 use Flow\Filesystem\Path;
 
-final class XMLParserExtractorTest extends IntegrationTestCase
+final class XMLParserExtractorTest extends FlowIntegrationTestCase
 {
     public function test_limit() : void
     {

src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLReaderExtractorTest.php

@@ -7,10 +7,10 @@
 use function Flow\ETL\DSL\type_string;
 use Flow\ETL\Adapter\XML\XMLReaderExtractor;
 use Flow\ETL\Extractor\Signal;
-use Flow\ETL\{Config, Flow, FlowContext, PHP\Type\Caster, Tests\Integration\IntegrationTestCase};
+use Flow\ETL\{Config, Flow, FlowContext, PHP\Type\Caster, Tests\Integration\FlowIntegrationTestCase};
 use Flow\Filesystem\Path;
 
-final class XMLReaderExtractorTest extends IntegrationTestCase
+final class XMLReaderExtractorTest extends FlowIntegrationTestCase
 {
     public function test_limit() : void
     {

src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLTest.php

@@ -7,9 +7,9 @@
 use function Flow\ETL\Adapter\XML\from_xml;
 use function Flow\ETL\DSL\{datetime_schema, df, int_schema, ref, schema, type_int};
 use function Flow\Filesystem\DSL\path;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-final class XMLTest extends IntegrationTestCase
+final class XMLTest extends FlowIntegrationTestCase
 {
     public function test_transforming_xml_into_a_tabular_dataset() : void
     {

src/bridge/filesystem/async-aws/tests/Flow/Filesystem/Bridge/AsyncAWS/Tests/Integration/AsyncAWSS3TestCase.php

@@ -6,10 +6,10 @@
 
 use function Flow\Filesystem\Bridge\AsyncAWS\DSL\aws_s3_client;
 use AsyncAws\S3\S3Client;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 use Flow\Filesystem\Path;
 
-abstract class AsyncAWSS3TestCase extends IntegrationTestCase
+abstract class AsyncAWSS3TestCase extends FlowIntegrationTestCase
 {
     protected function setUp() : void
     {

src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php

@@ -0,0 +1,94 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\ETL\Tests;
+
+use function Flow\ETL\DSL\{flow_context, rows};
+use Flow\ETL\{Extractor, FlowContext, Rows};
+use PHPUnit\Framework\TestCase;
+
+abstract class FlowTestCase extends TestCase
+{
+    final public static function assertExtractedBatchesCount(
+        int $expectedCount,
+        Extractor $extractor,
+        ?FlowContext $flowContext = null,
+        string $message = '',
+    ) : void {
+        $flowContext = $flowContext ?? flow_context();
+
+        static::assertCount(
+            $expectedCount,
+            \iterator_to_array($extractor->extract($flowContext)),
+            $message
+        );
+    }
+
+    final public static function assertExtractedBatchesSize(
+        int $expectedCount,
+        Extractor $extractor,
+        ?FlowContext $flowContext = null,
+        string $message = '',
+    ) : void {
+        $flowContext = $flowContext ?? flow_context();
+        $extractorContainsBatches = false;
+
+        foreach ($extractor->extract($flowContext) as $rows) {
+            static::assertCount($expectedCount, $rows, $message);
+            $extractorContainsBatches = true;
+        }
+
+        if (!$extractorContainsBatches) {
+            static::fail('Extractor does not contain any batches');
+        }
+    }
+
+    final public static function assertExtractedRowsAsArrayEquals(
+        array $expectedArray,
+        Extractor $extractor,
+        ?FlowContext $flowContext = null,
+        string $message = '',
+    ) : void {
+        $flowContext = $flowContext ?? flow_context();
+        $extractedRows = rows();
+
+        foreach ($extractor->extract($flowContext) as $nextRows) {
+            $extractedRows = $extractedRows->merge($nextRows);
+        }
+
+        static::assertEquals($expectedArray, $extractedRows->toArray(), $message);
+    }
+
+    final public static function assertExtractedRowsCount(
+        int $expectedCount,
+        Extractor $extractor,
+        ?FlowContext $flowContext = null,
+        string $message = '',
+    ) : void {
+        $flowContext = $flowContext ?? flow_context();
+        $totalRows = 0;
+
+        foreach ($extractor->extract($flowContext) as $rows) {
+            $totalRows += $rows->count();
+        }
+
+        static::assertSame($expectedCount, $totalRows, $message);
+    }
+
+    final public static function assertExtractedRowsEquals(
+        Rows $expectedRows,
+        Extractor $extractor,
+        ?FlowContext $flowContext = null,
+        string $message = '',
+    ) : void {
+        $flowContext = $flowContext ?? flow_context();
+        $extractedRows = rows();
+
+        foreach ($extractor->extract($flowContext) as $nextRows) {
+            $extractedRows = $extractedRows->merge($nextRows);
+        }
+
+        static::assertEquals($expectedRows, $extractedRows, $message);
+    }
+}

src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheBaseTestSuite.php

@@ -7,9 +7,9 @@
 use function Flow\ETL\DSL\{row, rows, str_entry};
 use Flow\ETL\Cache\{CacheIndex};
 use Flow\ETL\Exception\KeyNotInCacheException;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-abstract class CacheBaseTestSuite extends IntegrationTestCase
+abstract class CacheBaseTestSuite extends FlowIntegrationTestCase
 {
     protected function setUp() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AnalyzeTest.php

@@ -6,9 +6,9 @@
 
 use function Flow\ETL\Adapter\Text\from_text;
 use function Flow\ETL\DSL\{datetime_schema, df, float_schema, from_array, int_schema, schema, str_schema};
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-final class AnalyzeTest extends IntegrationTestCase
+final class AnalyzeTest extends FlowIntegrationTestCase
 {
     public function test_analyzing_csv_file_with_auto_cast() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/BranchingTest.php

@@ -6,9 +6,9 @@
 
 use function Flow\ETL\DSL\{df, from_array, lit, ref, to_branch, to_memory};
 use Flow\ETL\Memory\ArrayMemory;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-final class BranchingTest extends IntegrationTestCase
+final class BranchingTest extends FlowIntegrationTestCase
 {
     public function test_branching() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/CacheTest.php

@@ -8,10 +8,10 @@
 use Flow\ETL\Cache\CacheIndex;
 use Flow\ETL\Cache\Implementation\InMemoryCache;
 use Flow\ETL\Tests\Double\FakeExtractor;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 use Flow\ETL\{Extractor, FlowContext, Rows};
 
-final class CacheTest extends IntegrationTestCase
+final class CacheTest extends FlowIntegrationTestCase
 {
     public function test_cache() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfigBuilderTest.php

@@ -7,9 +7,9 @@
 use function Flow\ETL\DSL\config_builder;
 use Flow\ETL\Config\Cache\CacheConfig;
 use Flow\ETL\Sort\SortAlgorithms;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-final class ConfigBuilderTest extends IntegrationTestCase
+final class ConfigBuilderTest extends FlowIntegrationTestCase
 {
     public function test_creating_custom_cache_dir() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/DataFrameJsonTest.php

@@ -6,9 +6,9 @@
 
 use Flow\ETL\DataFrame;
 use Flow\ETL\Exception\InvalidArgumentException;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-final class DataFrameJsonTest extends IntegrationTestCase
+final class DataFrameJsonTest extends FlowIntegrationTestCase
 {
     public function test_building_data_frame_from_json() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/DisplayTest.php

@@ -30,10 +30,10 @@ enum_entry,
     type_string,
     xml_entry};
 use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 use Flow\ETL\{Extractor, FlowContext, Rows};
 
-final class DisplayTest extends IntegrationTestCase
+final class DisplayTest extends FlowIntegrationTestCase
 {
     public function test_display() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/FilterTest.php

@@ -5,9 +5,9 @@
 namespace Flow\ETL\Tests\Integration\DataFrame;
 
 use function Flow\ETL\DSL\{df, from_array, ref};
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-final class FilterTest extends IntegrationTestCase
+final class FilterTest extends FlowIntegrationTestCase
 {
     public function test_multiple_filters() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByTest.php

@@ -35,10 +35,10 @@
     uuid_schema,
     window};
 use Flow\ETL\Memory\ArrayMemory;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 use Flow\ETL\{Loader, Rows};
 
-final class GroupByTest extends IntegrationTestCase
+final class GroupByTest extends FlowIntegrationTestCase
 {
     public function test_group_by_array() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinTest.php

@@ -6,10 +6,10 @@
 
 use function Flow\ETL\DSL\{datetime_entry, df, from_rows, int_entry, row, rows, str_entry};
 use Flow\ETL\Join\Expression;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 use Flow\ETL\{Flow, Join\Join, Loader};
 
-final class JoinTest extends IntegrationTestCase
+final class JoinTest extends FlowIntegrationTestCase
 {
     public function test_join_inner() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php

@@ -16,10 +16,10 @@
     type_structure};
 use Flow\ETL\Exception\InvalidArgumentException;
 use Flow\ETL\Row\Entry\{IntegerEntry};
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 use Flow\ETL\{Extractor, FlowContext, Row, Rows};
 
-final class LimitTest extends IntegrationTestCase
+final class LimitTest extends FlowIntegrationTestCase
 {
     public function test_exceeding_the_limit_in_one_rows_set() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/MapTest.php

@@ -12,9 +12,9 @@
     type_list,
     type_string};
 use Flow\ETL\Row;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-final class MapTest extends IntegrationTestCase
+final class MapTest extends FlowIntegrationTestCase
 {
     public function test_using_map_to_replace_nullable_lists() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php

@@ -20,11 +20,11 @@
     rows_partitioned,
     str_entry};
 use function Flow\Filesystem\DSL\partition;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 use Flow\ETL\{Rows};
 use Flow\Filesystem\Partition;
 
-final class PartitioningTest extends IntegrationTestCase
+final class PartitioningTest extends FlowIntegrationTestCase
 {
     public function test_dropping_partitions() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/RenameTest.php

@@ -5,11 +5,11 @@
 namespace Flow\ETL\Tests\Integration\DataFrame;
 
 use function Flow\ETL\DSL\{bool_entry, df, from_rows, int_entry, json_entry, ref, str_entry};
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 use Flow\ETL\Transformer\StyleConverter\StringStyles;
 use Flow\ETL\{Row, Rows};
 
-final class RenameTest extends IntegrationTestCase
+final class RenameTest extends FlowIntegrationTestCase
 {
     public function test_rename() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SchemaTest.php

@@ -23,9 +23,9 @@
     string_entry};
 use Flow\ETL\Pipeline\SynchronousPipeline;
 use Flow\ETL\Row\Schema;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-final class SchemaTest extends IntegrationTestCase
+final class SchemaTest extends FlowIntegrationTestCase
 {
     public function test_extraction_according_to_schema() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SortTest.php

@@ -8,9 +8,9 @@
 use Flow\ETL\Config;
 use Flow\ETL\Monitoring\Memory\Unit;
 use Flow\ETL\Tests\Double\{FakeExtractor};
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-final class SortTest extends IntegrationTestCase
+final class SortTest extends FlowIntegrationTestCase
 {
     public function test_etl_sort_by_external_sort() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/CacheExtractorTest.php

@@ -8,9 +8,9 @@
 use Flow\ETL\Cache\CacheIndex;
 use Flow\ETL\Cache\Implementation\InMemoryCache;
 use Flow\ETL\Extractor\CacheExtractor;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-final class CacheExtractorTest extends IntegrationTestCase
+final class CacheExtractorTest extends FlowIntegrationTestCase
 {
     public function test_extracting_rows_from_cache() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php

@@ -5,10 +5,10 @@
 namespace Flow\ETL\Tests\Integration\Extractor;
 
 use function Flow\ETL\DSL\{flow_context, from_path_partitions, rows};
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 use Flow\Filesystem\Path;
 
-final class PathPartitionsExtractorTest extends IntegrationTestCase
+final class PathPartitionsExtractorTest extends FlowIntegrationTestCase
 {
     public function test_extracting_data_from_path_partitions() : void
     {

src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/FilesystemStreamsTestCase.php

@@ -5,9 +5,9 @@
 namespace Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams;
 
 use Flow\ETL\Filesystem\FilesystemStreams;
-use Flow\ETL\Tests\Integration\IntegrationTestCase;
+use Flow\ETL\Tests\Integration\FlowIntegrationTestCase;
 
-abstract class FilesystemStreamsTestCase extends IntegrationTestCase
+abstract class FilesystemStreamsTestCase extends FlowIntegrationTestCase
 {
     protected function filesDirectory() : string
     {