diff --git a/examples/topics/aggregations/average/code.php b/examples/topics/aggregations/average/code.php index e6e24da3d..8a38f3ee4 100644 --- a/examples/topics/aggregations/average/code.php +++ b/examples/topics/aggregations/average/code.php @@ -2,11 +2,11 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{average, df, from_rows, int_entry, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{average, data_frame, from_rows, int_entry, ref, row, rows, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->from(from_rows(rows( row(int_entry('a', 100)), row(int_entry('a', 100)), @@ -15,12 +15,5 @@ row(int_entry('a', 400)) ))) ->aggregate(average(ref('a'))) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// +-------+ -// | a_avg | -// +-------+ -// | 240 | -// +-------+ -// 1 rows diff --git a/examples/topics/aggregations/average/output.txt b/examples/topics/aggregations/average/output.txt new file mode 100644 index 000000000..5720e8c2e --- /dev/null +++ b/examples/topics/aggregations/average/output.txt @@ -0,0 +1,6 @@ ++-------+ +| a_avg | ++-------+ +| 240 | ++-------+ +1 rows diff --git a/examples/topics/aggregations/first/code.php b/examples/topics/aggregations/first/code.php index a2f3722fc..337cca90c 100644 --- a/examples/topics/aggregations/first/code.php +++ b/examples/topics/aggregations/first/code.php @@ -2,11 +2,11 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, first, from_rows, int_entry, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{data_frame, first, from_rows, int_entry, ref, row, rows, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->from(from_rows(rows( row(int_entry('a', 100)), row(int_entry('a', 100)), @@ -15,12 +15,5 @@ row(int_entry('a', 400)) ))) ->aggregate(first(ref('a'))) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// +-----+ -// | a | -// +-----+ -// | 100 | -// +-----+ -// 1 rows diff --git a/examples/topics/aggregations/first/output.txt b/examples/topics/aggregations/first/output.txt new file mode 100644 index 000000000..f927339a6 --- /dev/null +++ b/examples/topics/aggregations/first/output.txt @@ -0,0 +1,6 @@ ++-----+ +| a | ++-----+ +| 100 | ++-----+ +1 rows diff --git a/examples/topics/aggregations/group_by/code.php b/examples/topics/aggregations/group_by/code.php index ce6a579f2..d0d395514 100644 --- a/examples/topics/aggregations/group_by/code.php +++ b/examples/topics/aggregations/group_by/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, from_array, ref, to_output}; +use function Flow\ETL\DSL\{data_frame, from_array, ref, to_stream}; require __DIR__ . '/../../../autoload.php'; @@ -20,13 +20,5 @@ ['id' => 10, 'group' => 'B'], ])) ->groupBy(ref('group')) - ->write(to_output(truncate: false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// +-------+ -// | group | -// +-------+ -// | A | -// | B | -// +-------+ -// 2 rows diff --git a/examples/topics/aggregations/group_by/output.txt b/examples/topics/aggregations/group_by/output.txt new file mode 100644 index 000000000..0056816f0 --- /dev/null +++ b/examples/topics/aggregations/group_by/output.txt @@ -0,0 +1,7 @@ ++-------+ +| group | ++-------+ +| A | +| B | ++-------+ +2 rows diff --git a/examples/topics/aggregations/group_by_sum/code.php b/examples/topics/aggregations/group_by_sum/code.php index 13e29813c..1fd5e1665 100644 --- a/examples/topics/aggregations/group_by_sum/code.php +++ b/examples/topics/aggregations/group_by_sum/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, from_array, ref, sum, to_output}; +use function Flow\ETL\DSL\{data_frame, from_array, ref, sum, to_stream}; require __DIR__ . '/../../../autoload.php'; @@ -21,13 +21,5 @@ ])) ->groupBy(ref('group')) ->aggregate(sum(ref('value'))) - ->write(to_output(truncate: false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// +-------+-----------+ -// | group | value_sum | -// +-------+-----------+ -// | A | 1800 | -// | B | 460 | -// +-------+-----------+ -// 2 rows diff --git a/examples/topics/aggregations/group_by_sum/output.txt b/examples/topics/aggregations/group_by_sum/output.txt new file mode 100644 index 000000000..1c451b075 --- /dev/null +++ b/examples/topics/aggregations/group_by_sum/output.txt @@ -0,0 +1,7 @@ ++-------+-----------+ +| group | value_sum | ++-------+-----------+ +| A | 1800 | +| B | 460 | ++-------+-----------+ +2 rows diff --git a/examples/topics/aggregations/last/code.php b/examples/topics/aggregations/last/code.php index 8fee54226..fc8ea6fa5 100644 --- a/examples/topics/aggregations/last/code.php +++ b/examples/topics/aggregations/last/code.php @@ -2,11 +2,11 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, from_rows, int_entry, last, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{data_frame, from_rows, int_entry, last, ref, row, rows, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->from(from_rows(rows( row(int_entry('a', 100)), row(int_entry('a', 100)), @@ -15,12 +15,5 @@ row(int_entry('a', 400)) ))) ->aggregate(last(ref('a'))) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// +-----+ -// | a | -// +-----+ -// | 400 | -// +-----+ -// 1 rows diff --git a/examples/topics/aggregations/last/output.txt b/examples/topics/aggregations/last/output.txt new file mode 100644 index 000000000..912b1b55e --- /dev/null +++ b/examples/topics/aggregations/last/output.txt @@ -0,0 +1,6 @@ ++-----+ +| a | ++-----+ +| 400 | ++-----+ +1 rows diff --git a/examples/topics/aggregations/max/code.php b/examples/topics/aggregations/max/code.php index e51196e49..d1e3eadfa 100644 --- a/examples/topics/aggregations/max/code.php +++ b/examples/topics/aggregations/max/code.php @@ -2,11 +2,11 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, from_rows, int_entry, max, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{data_frame, from_rows, int_entry, max, ref, row, rows, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->from(from_rows(rows( row(int_entry('a', 100)), row(int_entry('a', 100)), @@ -15,12 +15,5 @@ row(int_entry('a', 400)) ))) ->aggregate(max(ref('a'))) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// +-------+ -// | a_max | -// +-------+ -// | 400 | -// +-------+ -// 1 rows diff --git a/examples/topics/aggregations/max/output.txt b/examples/topics/aggregations/max/output.txt new file mode 100644 index 000000000..5f52ad384 --- /dev/null +++ b/examples/topics/aggregations/max/output.txt @@ -0,0 +1,6 @@ ++-------+ +| a_max | ++-------+ +| 400 | ++-------+ +1 rows diff --git a/examples/topics/aggregations/min/code.php b/examples/topics/aggregations/min/code.php index 9143e5667..e07c8f3ce 100644 --- a/examples/topics/aggregations/min/code.php +++ b/examples/topics/aggregations/min/code.php @@ -2,11 +2,11 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, from_rows, int_entry, min, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{data_frame, from_rows, int_entry, min, ref, row, rows, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->from(from_rows(rows( row(int_entry('a', 100)), row(int_entry('a', 100)), @@ -15,12 +15,5 @@ row(int_entry('a', 400)) ))) ->aggregate(min(ref('a'))) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// +-------+ -// | a_min | -// +-------+ -// | 100 | -// +-------+ -// 1 rows diff --git a/examples/topics/aggregations/min/output.txt b/examples/topics/aggregations/min/output.txt new file mode 100644 index 000000000..114a5011c --- /dev/null +++ b/examples/topics/aggregations/min/output.txt @@ -0,0 +1,6 @@ ++-------+ +| a_min | ++-------+ +| 100 | ++-------+ +1 rows diff --git a/examples/topics/aggregations/sum/code.php b/examples/topics/aggregations/sum/code.php index d7fc50c26..087df2ff8 100644 --- a/examples/topics/aggregations/sum/code.php +++ b/examples/topics/aggregations/sum/code.php @@ -2,11 +2,11 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, from_rows, int_entry, ref, row, rows, sum, to_output}; +use function Flow\ETL\DSL\{data_frame, from_rows, int_entry, ref, row, rows, sum, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->from(from_rows(rows( row(int_entry('a', 100)), row(int_entry('a', 100)), @@ -15,5 +15,5 @@ row(int_entry('a', 400)) ))) ->aggregate(sum(ref('a'))) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/aggregations/sum/output.txt b/examples/topics/aggregations/sum/output.txt new file mode 100644 index 000000000..06bbcd0c2 --- /dev/null +++ b/examples/topics/aggregations/sum/output.txt @@ -0,0 +1,6 @@ ++-------+ +| a_sum | ++-------+ +| 1200 | ++-------+ +1 rows diff --git a/examples/topics/data_frame/cache/code.php b/examples/topics/data_frame/cache/code.php index 3336a0aca..c480b6c0e 100644 --- a/examples/topics/data_frame/cache/code.php +++ b/examples/topics/data_frame/cache/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{config_builder, df, from_cache, ref, to_output}; +use function Flow\ETL\DSL\{config_builder, data_frame, from_cache, ref, to_stream}; use Flow\ETL\Adapter\Http\DynamicExtractor\NextRequestFactory; use Flow\ETL\Adapter\Http\PsrHttpClientDynamicExtractor; use Flow\ETL\Cache\PSRSimpleCache; @@ -41,7 +41,7 @@ public function create(?ResponseInterface $previousResponse = null) : ?RequestIn ) ); -df(config_builder()->cache($adapter)) +data_frame(config_builder()->cache($adapter)) ->read( from_cache( id: 'github_api', @@ -55,5 +55,5 @@ public function create(?ResponseInterface $previousResponse = null) : ?RequestIn ->renameAll('unpacked.', '') ->drop('unpacked') ->select('name', 'html_url', 'blog', 'login', 'public_repos', 'followers', 'created_at') - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/data_frame/cache/output.txt b/examples/topics/data_frame/cache/output.txt new file mode 100644 index 000000000..60c5a9fca --- /dev/null +++ b/examples/topics/data_frame/cache/output.txt @@ -0,0 +1,6 @@ ++----------+----------+---------------------+--------------+-----------+-----------------------------+----------------------+ +| login | name | blog | public_repos | followers | html_url | created_at | ++----------+----------+---------------------+--------------+-----------+-----------------------------+----------------------+ +| flow-php | Flow PHP | http://flow-php.com | 30 | 83 | https://github.com/flow-php | 2020-10-26T18:40:27Z | ++----------+----------+---------------------+--------------+-----------+-----------------------------+----------------------+ +1 rows diff --git a/examples/topics/data_frame/data_frame/code.php b/examples/topics/data_frame/data_frame/code.php index 72724e944..fa76fdf5d 100644 --- a/examples/topics/data_frame/data_frame/code.php +++ b/examples/topics/data_frame/data_frame/code.php @@ -2,15 +2,13 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{array_entry, array_expand, df, from_rows, int_entry, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{array_entry, array_expand, data_frame, from_rows, int_entry, ref, row, rows, to_output, to_stream}; -// flow.phar run examples/topics/phar/data_frame/code.php -// when executing data processing pipeline through phar make sure to not use any trigger, like ->run(); -// this is handled by the phar internally. -return df() +data_frame() ->read(from_rows(rows( row(int_entry('id', 1), array_entry('array', ['a' => 1, 'b' => 2, 'c' => 3])), ))) ->write(to_output(false)) ->withEntry('expanded', array_expand(ref('array'))) - ->write(to_output(false)); + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) + ->run(); diff --git a/examples/topics/data_frame/data_frame/output.txt b/examples/topics/data_frame/data_frame/output.txt new file mode 100644 index 000000000..ccb027ada --- /dev/null +++ b/examples/topics/data_frame/data_frame/output.txt @@ -0,0 +1,8 @@ ++----+---------------------+----------+ +| id | array | expanded | ++----+---------------------+----------+ +| 1 | {"a":1,"b":2,"c":3} | 1 | +| 1 | {"a":1,"b":2,"c":3} | 2 | +| 1 | {"a":1,"b":2,"c":3} | 3 | ++----+---------------------+----------+ +3 rows diff --git a/examples/topics/data_frame/overwrite/code.php b/examples/topics/data_frame/overwrite/code.php index f0d6772c5..31f392370 100644 --- a/examples/topics/data_frame/overwrite/code.php +++ b/examples/topics/data_frame/overwrite/code.php @@ -3,26 +3,23 @@ declare(strict_types=1); use function Flow\ETL\Adapter\CSV\{from_csv, to_csv}; -use function Flow\ETL\DSL\{df, overwrite}; +use function Flow\ETL\DSL\{data_frame, overwrite, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->read(from_csv(__DIR__ . '/input/file.csv')) ->saveMode(overwrite()) ->write(to_csv(__DIR__ . '/output/file.csv')) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); -df() +data_frame() ->read(from_csv(__DIR__ . '/output/file.csv')) ->saveMode(overwrite()) ->drop('name') ->write(to_csv(__DIR__ . '/output/file.csv')) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// content of /output/file.csv: -// id -// 1 -// 2 -// 3 -// 4 diff --git a/examples/topics/data_frame/overwrite/output.txt b/examples/topics/data_frame/overwrite/output.txt new file mode 100644 index 000000000..940c8504f --- /dev/null +++ b/examples/topics/data_frame/overwrite/output.txt @@ -0,0 +1,9 @@ ++----+ +| id | ++----+ +| 1 | +| 2 | +| 3 | +| 4 | ++----+ +4 rows diff --git a/examples/topics/data_source/array/code.php b/examples/topics/data_source/array/code.php index 21d24e53c..23a82e66e 100644 --- a/examples/topics/data_source/array/code.php +++ b/examples/topics/data_source/array/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, from_array, to_output}; +use function Flow\ETL\DSL\{data_frame, from_array, to_stream}; require __DIR__ . '/../../../autoload.php'; @@ -14,5 +14,6 @@ ['id' => 4], ['id' => 5], ])) - ->write(to_output(false)) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/data_source/array/output.txt b/examples/topics/data_source/array/output.txt new file mode 100644 index 000000000..49329e964 --- /dev/null +++ b/examples/topics/data_source/array/output.txt @@ -0,0 +1,10 @@ ++----+ +| id | ++----+ +| 1 | +| 2 | +| 3 | +| 4 | +| 5 | ++----+ +5 rows diff --git a/examples/topics/data_source/csv/code.php b/examples/topics/data_source/csv/code.php index 462cb58b1..aa3c95f7e 100644 --- a/examples/topics/data_source/csv/code.php +++ b/examples/topics/data_source/csv/code.php @@ -3,7 +3,7 @@ declare(strict_types=1); use function Flow\ETL\Adapter\CSV\from_csv; -use function Flow\ETL\DSL\{data_frame, to_output}; +use function Flow\ETL\DSL\{data_frame, to_stream}; require __DIR__ . '/../../../autoload.php'; @@ -17,5 +17,6 @@ escape: '\\', characters_read_in_line: 1000 )) - ->write(to_output(false)) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/data_source/csv/output.txt b/examples/topics/data_source/csv/output.txt new file mode 100644 index 000000000..8a40f4aaf --- /dev/null +++ b/examples/topics/data_source/csv/output.txt @@ -0,0 +1,9 @@ ++----+--------+------------------+--------+ +| id | name | email | active | ++----+--------+------------------+--------+ +| 1 | John | john@email.com | true | +| 2 | Paul | paul@email.com | true | +| 3 | George | george@email.com | false | +| 4 | Ringo | rino@email.com | true | ++----+--------+------------------+--------+ +4 rows diff --git a/examples/topics/data_source/data_frame/code.php b/examples/topics/data_source/data_frame/code.php index 733cb3221..bdcb8dd1b 100644 --- a/examples/topics/data_source/data_frame/code.php +++ b/examples/topics/data_source/data_frame/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, from_array, from_data_frame, lit, to_output}; +use function Flow\ETL\DSL\{data_frame, from_array, from_data_frame, lit, ref, to_stream}; require __DIR__ . '/../../../autoload.php'; @@ -19,8 +19,9 @@ ['id' => 5], ] )) - ->withEntry('timestamp', lit(\time())) + ->withEntry('timestamp', ref('id')->multiply(lit(10000))) ) ) - ->write(to_output(false)) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/data_source/data_frame/output.txt b/examples/topics/data_source/data_frame/output.txt new file mode 100644 index 000000000..ceb7d43e7 --- /dev/null +++ b/examples/topics/data_source/data_frame/output.txt @@ -0,0 +1,10 @@ ++----+-----------+ +| id | timestamp | ++----+-----------+ +| 1 | 10000 | +| 2 | 20000 | +| 3 | 30000 | +| 4 | 40000 | +| 5 | 50000 | ++----+-----------+ +5 rows diff --git a/examples/topics/data_source/http_dynamic/code.php b/examples/topics/data_source/http_dynamic/code.php index c1c2a9deb..ad2e50a09 100644 --- a/examples/topics/data_source/http_dynamic/code.php +++ b/examples/topics/data_source/http_dynamic/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, ref, to_output}; +use function Flow\ETL\DSL\{data_frame, ref, to_stream}; use Flow\ETL\Adapter\Http\DynamicExtractor\NextRequestFactory; use Flow\ETL\Adapter\Http\PsrHttpClientDynamicExtractor; use Http\Client\Curl\Client; @@ -38,5 +38,5 @@ public function create(?Message\ResponseInterface $previousResponse = null) : ?M ->renameAll('unpacked.', '') ->drop('unpacked') ->select('name', 'html_url', 'blog', 'login', 'public_repos', 'followers', 'created_at') - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/data_source/http_dynamic/output.txt b/examples/topics/data_source/http_dynamic/output.txt new file mode 100644 index 000000000..0fddfdbba --- /dev/null +++ b/examples/topics/data_source/http_dynamic/output.txt @@ -0,0 +1,6 @@ ++----------+----------+---------------------+--------------+-----------+-----------------------------+----------------------+ +| login | name | blog | public_repos | followers | html_url | created_at | ++----------+----------+---------------------+--------------+-----------+-----------------------------+----------------------+ +| flow-php | Flow PHP | http://flow-php.com | 30 | 85 | https://github.com/flow-php | 2020-10-26T18:40:27Z | ++----------+----------+---------------------+--------------+-----------+-----------------------------+----------------------+ +1 rows diff --git a/examples/topics/data_source/json/code.php b/examples/topics/data_source/json/code.php index 6f8809156..8aaf9079a 100644 --- a/examples/topics/data_source/json/code.php +++ b/examples/topics/data_source/json/code.php @@ -3,7 +3,7 @@ declare(strict_types=1); use function Flow\ETL\Adapter\JSON\from_json; -use function Flow\ETL\DSL\{data_frame, to_output}; +use function Flow\ETL\DSL\{data_frame, to_stream}; require __DIR__ . '/../../../autoload.php'; @@ -11,5 +11,6 @@ ->read(from_json( __DIR__ . '/input/dataset.json', )) - ->write(to_output(false)) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/data_source/json/output.txt b/examples/topics/data_source/json/output.txt new file mode 100644 index 000000000..8a40f4aaf --- /dev/null +++ b/examples/topics/data_source/json/output.txt @@ -0,0 +1,9 @@ ++----+--------+------------------+--------+ +| id | name | email | active | ++----+--------+------------------+--------+ +| 1 | John | john@email.com | true | +| 2 | Paul | paul@email.com | true | +| 3 | George | george@email.com | false | +| 4 | Ringo | rino@email.com | true | ++----+--------+------------------+--------+ +4 rows diff --git a/examples/topics/data_source/parquet/code.php b/examples/topics/data_source/parquet/code.php index 6fabc00ff..210052554 100644 --- a/examples/topics/data_source/parquet/code.php +++ b/examples/topics/data_source/parquet/code.php @@ -3,7 +3,7 @@ declare(strict_types=1); use function Flow\ETL\Adapter\Parquet\from_parquet; -use function Flow\ETL\DSL\{data_frame, to_output}; +use function Flow\ETL\DSL\{data_frame, to_stream}; require __DIR__ . '/../../../autoload.php'; @@ -11,5 +11,6 @@ ->read(from_parquet( __DIR__ . '/input/dataset.parquet', )) - ->write(to_output(false)) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/data_source/parquet/output.txt b/examples/topics/data_source/parquet/output.txt new file mode 100644 index 000000000..8a40f4aaf --- /dev/null +++ b/examples/topics/data_source/parquet/output.txt @@ -0,0 +1,9 @@ ++----+--------+------------------+--------+ +| id | name | email | active | ++----+--------+------------------+--------+ +| 1 | John | john@email.com | true | +| 2 | Paul | paul@email.com | true | +| 3 | George | george@email.com | false | +| 4 | Ringo | rino@email.com | true | ++----+--------+------------------+--------+ +4 rows diff --git a/examples/topics/data_source/sequence_date/code.php b/examples/topics/data_source/sequence_date/code.php index 033d442bd..83c4c678a 100644 --- a/examples/topics/data_source/sequence_date/code.php +++ b/examples/topics/data_source/sequence_date/code.php @@ -2,16 +2,17 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, from_sequence_date_period, to_output}; +use function Flow\ETL\DSL\{data_frame, from_sequence_date_period, to_stream}; require __DIR__ . '/../../../autoload.php'; data_frame() ->read(from_sequence_date_period( 'date', - new DateTimeImmutable('now'), + new DateTimeImmutable('2024-01-01 00:00:00 UTC'), new DateInterval('P1D'), new DateTimeImmutable('now + 60 days'), )) - ->write(to_output(false)) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/data_source/sequence_date/output.txt b/examples/topics/data_source/sequence_date/output.txt new file mode 100644 index 000000000..79dd94951 --- /dev/null +++ b/examples/topics/data_source/sequence_date/output.txt @@ -0,0 +1,148 @@ ++---------------------------+ +| date | ++---------------------------+ +| 2024-01-01T00:00:00+00:00 | +| 2024-01-02T00:00:00+00:00 | +| 2024-01-03T00:00:00+00:00 | +| 2024-01-04T00:00:00+00:00 | +| 2024-01-05T00:00:00+00:00 | +| 2024-01-06T00:00:00+00:00 | +| 2024-01-07T00:00:00+00:00 | +| 2024-01-08T00:00:00+00:00 | +| 2024-01-09T00:00:00+00:00 | +| 2024-01-10T00:00:00+00:00 | +| 2024-01-11T00:00:00+00:00 | +| 2024-01-12T00:00:00+00:00 | +| 2024-01-13T00:00:00+00:00 | +| 2024-01-14T00:00:00+00:00 | +| 2024-01-15T00:00:00+00:00 | +| 2024-01-16T00:00:00+00:00 | +| 2024-01-17T00:00:00+00:00 | +| 2024-01-18T00:00:00+00:00 | +| 2024-01-19T00:00:00+00:00 | +| 2024-01-20T00:00:00+00:00 | +| 2024-01-21T00:00:00+00:00 | +| 2024-01-22T00:00:00+00:00 | +| 2024-01-23T00:00:00+00:00 | +| 2024-01-24T00:00:00+00:00 | +| 2024-01-25T00:00:00+00:00 | +| 2024-01-26T00:00:00+00:00 | +| 2024-01-27T00:00:00+00:00 | +| 2024-01-28T00:00:00+00:00 | +| 2024-01-29T00:00:00+00:00 | +| 2024-01-30T00:00:00+00:00 | +| 2024-01-31T00:00:00+00:00 | +| 2024-02-01T00:00:00+00:00 | +| 2024-02-02T00:00:00+00:00 | +| 2024-02-03T00:00:00+00:00 | +| 2024-02-04T00:00:00+00:00 | +| 2024-02-05T00:00:00+00:00 | +| 2024-02-06T00:00:00+00:00 | +| 2024-02-07T00:00:00+00:00 | +| 2024-02-08T00:00:00+00:00 | +| 2024-02-09T00:00:00+00:00 | +| 2024-02-10T00:00:00+00:00 | +| 2024-02-11T00:00:00+00:00 | +| 2024-02-12T00:00:00+00:00 | +| 2024-02-13T00:00:00+00:00 | +| 2024-02-14T00:00:00+00:00 | +| 2024-02-15T00:00:00+00:00 | +| 2024-02-16T00:00:00+00:00 | +| 2024-02-17T00:00:00+00:00 | +| 2024-02-18T00:00:00+00:00 | +| 2024-02-19T00:00:00+00:00 | +| 2024-02-20T00:00:00+00:00 | +| 2024-02-21T00:00:00+00:00 | +| 2024-02-22T00:00:00+00:00 | +| 2024-02-23T00:00:00+00:00 | +| 2024-02-24T00:00:00+00:00 | +| 2024-02-25T00:00:00+00:00 | +| 2024-02-26T00:00:00+00:00 | +| 2024-02-27T00:00:00+00:00 | +| 2024-02-28T00:00:00+00:00 | +| 2024-02-29T00:00:00+00:00 | +| 2024-03-01T00:00:00+00:00 | +| 2024-03-02T00:00:00+00:00 | +| 2024-03-03T00:00:00+00:00 | +| 2024-03-04T00:00:00+00:00 | +| 2024-03-05T00:00:00+00:00 | +| 2024-03-06T00:00:00+00:00 | +| 2024-03-07T00:00:00+00:00 | +| 2024-03-08T00:00:00+00:00 | +| 2024-03-09T00:00:00+00:00 | +| 2024-03-10T00:00:00+00:00 | +| 2024-03-11T00:00:00+00:00 | +| 2024-03-12T00:00:00+00:00 | +| 2024-03-13T00:00:00+00:00 | +| 2024-03-14T00:00:00+00:00 | +| 2024-03-15T00:00:00+00:00 | +| 2024-03-16T00:00:00+00:00 | +| 2024-03-17T00:00:00+00:00 | +| 2024-03-18T00:00:00+00:00 | +| 2024-03-19T00:00:00+00:00 | +| 2024-03-20T00:00:00+00:00 | +| 2024-03-21T00:00:00+00:00 | +| 2024-03-22T00:00:00+00:00 | +| 2024-03-23T00:00:00+00:00 | +| 2024-03-24T00:00:00+00:00 | +| 2024-03-25T00:00:00+00:00 | +| 2024-03-26T00:00:00+00:00 | +| 2024-03-27T00:00:00+00:00 | +| 2024-03-28T00:00:00+00:00 | +| 2024-03-29T00:00:00+00:00 | +| 2024-03-30T00:00:00+00:00 | +| 2024-03-31T00:00:00+00:00 | +| 2024-04-01T00:00:00+00:00 | +| 2024-04-02T00:00:00+00:00 | +| 2024-04-03T00:00:00+00:00 | +| 2024-04-04T00:00:00+00:00 | +| 2024-04-05T00:00:00+00:00 | +| 2024-04-06T00:00:00+00:00 | +| 2024-04-07T00:00:00+00:00 | +| 2024-04-08T00:00:00+00:00 | +| 2024-04-09T00:00:00+00:00 | +| 2024-04-10T00:00:00+00:00 | +| 2024-04-11T00:00:00+00:00 | +| 2024-04-12T00:00:00+00:00 | +| 2024-04-13T00:00:00+00:00 | +| 2024-04-14T00:00:00+00:00 | +| 2024-04-15T00:00:00+00:00 | +| 2024-04-16T00:00:00+00:00 | +| 2024-04-17T00:00:00+00:00 | +| 2024-04-18T00:00:00+00:00 | +| 2024-04-19T00:00:00+00:00 | +| 2024-04-20T00:00:00+00:00 | +| 2024-04-21T00:00:00+00:00 | +| 2024-04-22T00:00:00+00:00 | +| 2024-04-23T00:00:00+00:00 | +| 2024-04-24T00:00:00+00:00 | +| 2024-04-25T00:00:00+00:00 | +| 2024-04-26T00:00:00+00:00 | +| 2024-04-27T00:00:00+00:00 | +| 2024-04-28T00:00:00+00:00 | +| 2024-04-29T00:00:00+00:00 | +| 2024-04-30T00:00:00+00:00 | +| 2024-05-01T00:00:00+00:00 | +| 2024-05-02T00:00:00+00:00 | +| 2024-05-03T00:00:00+00:00 | +| 2024-05-04T00:00:00+00:00 | +| 2024-05-05T00:00:00+00:00 | +| 2024-05-06T00:00:00+00:00 | +| 2024-05-07T00:00:00+00:00 | +| 2024-05-08T00:00:00+00:00 | +| 2024-05-09T00:00:00+00:00 | +| 2024-05-10T00:00:00+00:00 | +| 2024-05-11T00:00:00+00:00 | +| 2024-05-12T00:00:00+00:00 | +| 2024-05-13T00:00:00+00:00 | +| 2024-05-14T00:00:00+00:00 | +| 2024-05-15T00:00:00+00:00 | +| 2024-05-16T00:00:00+00:00 | +| 2024-05-17T00:00:00+00:00 | +| 2024-05-18T00:00:00+00:00 | +| 2024-05-19T00:00:00+00:00 | +| 2024-05-20T00:00:00+00:00 | +| 2024-05-21T00:00:00+00:00 | +| 2024-05-22T00:00:00+00:00 | ++---------------------------+ +143 rows diff --git a/examples/topics/data_source/sequence_date_recurrences/code.php b/examples/topics/data_source/sequence_date_recurrences/code.php index 62e85800b..34816617e 100644 --- a/examples/topics/data_source/sequence_date_recurrences/code.php +++ b/examples/topics/data_source/sequence_date_recurrences/code.php @@ -2,16 +2,17 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, from_sequence_date_period_recurrences, to_output}; +use function Flow\ETL\DSL\{data_frame, from_sequence_date_period_recurrences, to_stream}; require __DIR__ . '/../../../autoload.php'; data_frame() ->read(from_sequence_date_period_recurrences( 'date', - new DateTimeImmutable('now'), + new DateTimeImmutable('2024-01-01 00:00:00 UTC'), new DateInterval('P1D'), recurrences: 60 )) - ->write(to_output(false)) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/data_source/sequence_date_recurrences/output.txt b/examples/topics/data_source/sequence_date_recurrences/output.txt new file mode 100644 index 000000000..206c0344e --- /dev/null +++ b/examples/topics/data_source/sequence_date_recurrences/output.txt @@ -0,0 +1,66 @@ ++---------------------------+ +| date | ++---------------------------+ +| 2024-01-01T00:00:00+00:00 | +| 2024-01-02T00:00:00+00:00 | +| 2024-01-03T00:00:00+00:00 | +| 2024-01-04T00:00:00+00:00 | +| 2024-01-05T00:00:00+00:00 | +| 2024-01-06T00:00:00+00:00 | +| 2024-01-07T00:00:00+00:00 | +| 2024-01-08T00:00:00+00:00 | +| 2024-01-09T00:00:00+00:00 | +| 2024-01-10T00:00:00+00:00 | +| 2024-01-11T00:00:00+00:00 | +| 2024-01-12T00:00:00+00:00 | +| 2024-01-13T00:00:00+00:00 | +| 2024-01-14T00:00:00+00:00 | +| 2024-01-15T00:00:00+00:00 | +| 2024-01-16T00:00:00+00:00 | +| 2024-01-17T00:00:00+00:00 | +| 2024-01-18T00:00:00+00:00 | +| 2024-01-19T00:00:00+00:00 | +| 2024-01-20T00:00:00+00:00 | +| 2024-01-21T00:00:00+00:00 | +| 2024-01-22T00:00:00+00:00 | +| 2024-01-23T00:00:00+00:00 | +| 2024-01-24T00:00:00+00:00 | +| 2024-01-25T00:00:00+00:00 | +| 2024-01-26T00:00:00+00:00 | +| 2024-01-27T00:00:00+00:00 | +| 2024-01-28T00:00:00+00:00 | +| 2024-01-29T00:00:00+00:00 | +| 2024-01-30T00:00:00+00:00 | +| 2024-01-31T00:00:00+00:00 | +| 2024-02-01T00:00:00+00:00 | +| 2024-02-02T00:00:00+00:00 | +| 2024-02-03T00:00:00+00:00 | +| 2024-02-04T00:00:00+00:00 | +| 2024-02-05T00:00:00+00:00 | +| 2024-02-06T00:00:00+00:00 | +| 2024-02-07T00:00:00+00:00 | +| 2024-02-08T00:00:00+00:00 | +| 2024-02-09T00:00:00+00:00 | +| 2024-02-10T00:00:00+00:00 | +| 2024-02-11T00:00:00+00:00 | +| 2024-02-12T00:00:00+00:00 | +| 2024-02-13T00:00:00+00:00 | +| 2024-02-14T00:00:00+00:00 | +| 2024-02-15T00:00:00+00:00 | +| 2024-02-16T00:00:00+00:00 | +| 2024-02-17T00:00:00+00:00 | +| 2024-02-18T00:00:00+00:00 | +| 2024-02-19T00:00:00+00:00 | +| 2024-02-20T00:00:00+00:00 | +| 2024-02-21T00:00:00+00:00 | +| 2024-02-22T00:00:00+00:00 | +| 2024-02-23T00:00:00+00:00 | +| 2024-02-24T00:00:00+00:00 | +| 2024-02-25T00:00:00+00:00 | +| 2024-02-26T00:00:00+00:00 | +| 2024-02-27T00:00:00+00:00 | +| 2024-02-28T00:00:00+00:00 | +| 2024-02-29T00:00:00+00:00 | +| 2024-03-01T00:00:00+00:00 | ++---------------------------+ +61 rows diff --git a/examples/topics/data_source/sequence_number/code.php b/examples/topics/data_source/sequence_number/code.php index ab86f36d5..d4b0b3d77 100644 --- a/examples/topics/data_source/sequence_number/code.php +++ b/examples/topics/data_source/sequence_number/code.php @@ -2,11 +2,12 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, from_sequence_number, to_output}; +use function Flow\ETL\DSL\{data_frame, from_sequence_number, to_stream}; require __DIR__ . '/../../../autoload.php'; data_frame() ->read(from_sequence_number('id', start: 0, end: 1000, step: 100)) - ->write(to_output(false)) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/data_source/sequence_number/output.txt b/examples/topics/data_source/sequence_number/output.txt new file mode 100644 index 000000000..fe20b9a85 --- /dev/null +++ b/examples/topics/data_source/sequence_number/output.txt @@ -0,0 +1,16 @@ ++------+ +| id | ++------+ +| 0 | +| 100 | +| 200 | +| 300 | +| 400 | +| 500 | +| 600 | +| 700 | +| 800 | +| 900 | +| 1000 | ++------+ +11 rows diff --git a/examples/topics/data_source/xml/code.php b/examples/topics/data_source/xml/code.php index ca3c587d4..a64aa7c34 100644 --- a/examples/topics/data_source/xml/code.php +++ b/examples/topics/data_source/xml/code.php @@ -3,7 +3,7 @@ declare(strict_types=1); use function Flow\ETL\Adapter\XML\from_xml; -use function Flow\ETL\DSL\{data_frame, ref, to_output}; +use function Flow\ETL\DSL\{data_frame, ref, to_stream}; require __DIR__ . '/../../../autoload.php'; @@ -17,5 +17,6 @@ ->withEntry('active', ref('node')->xpath('active')->domNodeValue()) ->withEntry('email', ref('node')->xpath('email')->domNodeValue()) ->drop('node') - ->write(to_output(false)) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/data_source/xml/output.txt b/examples/topics/data_source/xml/output.txt new file mode 100644 index 000000000..b7dbc290e --- /dev/null +++ b/examples/topics/data_source/xml/output.txt @@ -0,0 +1,15 @@ ++----+---------+--------+---------------------+ +| id | name | active | email | ++----+---------+--------+---------------------+ +| 1 | Alice | true | alice@example.com | +| 2 | Bob | false | bob@example.com | +| 3 | Charlie | true | charlie@example.com | +| 4 | David | false | david@example.com | +| 5 | Emma | true | emma@example.com | +| 6 | Frank | false | frank@example.com | +| 7 | Grace | true | grace@example.com | +| 8 | Harry | false | harry@example.com | +| 9 | Isla | true | isla@example.com | +| 10 | James | false | james@example.com | ++----+---------+--------+---------------------+ +10 rows diff --git a/examples/topics/join/left_anti/code.php b/examples/topics/join/left_anti/code.php index c80abe9df..9f01e7060 100644 --- a/examples/topics/join/left_anti/code.php +++ b/examples/topics/join/left_anti/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, from_array, to_output}; +use function Flow\ETL\DSL\{data_frame, from_array, to_stream}; use Flow\ETL\Join\{Expression, Join}; require __DIR__ . '/../../../autoload.php'; @@ -31,14 +31,5 @@ Expression::on(['id' => 'id']), Join::left_anti ) - ->write(to_output()) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// Output -// -// +--+---------+ -// |id| sku| -// +--+---------+ -// | 1|PRODUCT01| -// +--+---------+ -// 1 rows diff --git a/examples/topics/join/left_anti/output.txt b/examples/topics/join/left_anti/output.txt new file mode 100644 index 000000000..095ca30e5 --- /dev/null +++ b/examples/topics/join/left_anti/output.txt @@ -0,0 +1,6 @@ ++----+-----------+ +| id | sku | ++----+-----------+ +| 1 | PRODUCT01 | ++----+-----------+ +1 rows diff --git a/examples/topics/join/left_anti_each/code.php b/examples/topics/join/left_anti_each/code.php index 732ab7457..4f06df129 100644 --- a/examples/topics/join/left_anti_each/code.php +++ b/examples/topics/join/left_anti_each/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, int_entry, str_entry, to_output}; +use function Flow\ETL\DSL\{data_frame, df, int_entry, str_entry, to_stream}; use Flow\ETL\Join\Comparison\Equal; use Flow\ETL\Join\{Expression, Join}; use Flow\ETL\{DataFrame, DataFrameFactory, Extractor, FlowContext, Row, Rows}; @@ -51,23 +51,12 @@ private function findRowsInDatabase(Rows $rows) : Rows * right size is much bigger then a left side. In that case it's better to reduce the ride side * by fetching from the storage only what is relevant for the left side. */ -df() +data_frame() ->extract($apiExtractor) ->joinEach( $db_data_frame, Expression::on(new Equal('id', 'id')), // by using All or Any comparisons, more than one entry can be used to prepare the condition Join::left_anti ) - ->write(to_output()) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// Output: -// -// +-----+-------------+ -// | id| sku| -// +-----+-------------+ -// |10001|PRODUCT10_001| -// |10002|PRODUCT10_002| -// |10003|PRODUCT10_003| -// +-----+-------------+ -// 3 rows diff --git a/examples/topics/join/left_anti_each/output.txt b/examples/topics/join/left_anti_each/output.txt new file mode 100644 index 000000000..dc5cccc73 --- /dev/null +++ b/examples/topics/join/left_anti_each/output.txt @@ -0,0 +1,8 @@ ++-------+---------------+ +| id | sku | ++-------+---------------+ +| 10001 | PRODUCT10_001 | +| 10002 | PRODUCT10_002 | +| 10003 | PRODUCT10_003 | ++-------+---------------+ +3 rows diff --git a/examples/topics/partitioning/partition_pruning/code.php b/examples/topics/partitioning/partition_pruning/code.php index 9b145fb77..7101ea40f 100644 --- a/examples/topics/partitioning/partition_pruning/code.php +++ b/examples/topics/partitioning/partition_pruning/code.php @@ -3,7 +3,7 @@ declare(strict_types=1); use function Flow\ETL\Adapter\CSV\from_csv; -use function Flow\ETL\DSL\{data_frame, lit, ref, to_output}; +use function Flow\ETL\DSL\{data_frame, lit, ref, to_stream}; require __DIR__ . '/../../../autoload.php'; @@ -11,16 +11,5 @@ ->read(from_csv(__DIR__ . '/input/color=*/sku=*/*.csv')) ->filterPartitions(ref('color')->notEquals(lit('green'))) ->collect() - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// +----+-------+-----------+ -// | id | color | sku | -// +----+-------+-----------+ -// | 2 | red | PRODUCT02 | -// | 3 | red | PRODUCT03 | -// | 1 | red | PRODUCT01 | -// | 8 | blue | PRODUCT02 | -// | 7 | blue | PRODUCT01 | -// +----+-------+-----------+ -// 5 rows diff --git a/examples/topics/partitioning/partition_pruning/output.txt b/examples/topics/partitioning/partition_pruning/output.txt new file mode 100644 index 000000000..45432feb5 --- /dev/null +++ b/examples/topics/partitioning/partition_pruning/output.txt @@ -0,0 +1,10 @@ ++----+-------+-----------+ +| id | color | sku | ++----+-------+-----------+ +| 2 | red | PRODUCT02 | +| 3 | red | PRODUCT03 | +| 1 | red | PRODUCT01 | +| 8 | blue | PRODUCT02 | +| 7 | blue | PRODUCT01 | ++----+-------+-----------+ +5 rows diff --git a/examples/topics/partitioning/partitioning/code.php b/examples/topics/partitioning/partitioning/code.php index 0606b6381..ac2b46c60 100644 --- a/examples/topics/partitioning/partitioning/code.php +++ b/examples/topics/partitioning/partitioning/code.php @@ -3,7 +3,7 @@ declare(strict_types=1); use function Flow\ETL\Adapter\CSV\to_csv; -use function Flow\ETL\DSL\{data_frame, from_array, ref}; +use function Flow\ETL\DSL\{data_frame, from_array, overwrite, ref}; require __DIR__ . '/../../../autoload.php'; @@ -21,28 +21,6 @@ ] )) ->partitionBy(ref('color'), ref('sku')) + ->mode(overwrite()) ->write(to_csv(__DIR__ . '/output/products.csv')) ->run(); - -// output -// ├── color=blue -// │ ├── sku=PRODUCT01 -// │ │ └── products.csv -// │ └── sku=PRODUCT02 -// │ └── products.csv -// ├── color=green -// │ ├── sku=PRODUCT01 -// │ │ └── products.csv -// │ ├── sku=PRODUCT02 -// │ │ └── products.csv -// │ └── sku=PRODUCT03 -// │ └── products.csv -// └── color=red -// ├── sku=PRODUCT01 -// │ └── products.csv -// ├── sku=PRODUCT02 -// │ └── products.csv -// └── sku=PRODUCT03 -// └── products.csv -// -// 12 directories, 8 files diff --git a/examples/topics/partitioning/partitioning/output.txt b/examples/topics/partitioning/partitioning/output.txt new file mode 100644 index 000000000..7d4e3a5c7 --- /dev/null +++ b/examples/topics/partitioning/partitioning/output.txt @@ -0,0 +1,22 @@ +output +├── color=blue +│ ├── sku=PRODUCT01 +│ │ └── products.csv +│ └── sku=PRODUCT02 +│ └── products.csv +├── color=green +│ ├── sku=PRODUCT01 +│ │ └── products.csv +│ ├── sku=PRODUCT02 +│ │ └── products.csv +│ └── sku=PRODUCT03 +│ └── products.csv +└── color=red + ├── sku=PRODUCT01 + │ └── products.csv + ├── sku=PRODUCT02 + │ └── products.csv + └── sku=PRODUCT03 + └── products.csv + +12 directories, 8 files \ No newline at end of file diff --git a/examples/topics/partitioning/reading/code.php b/examples/topics/partitioning/reading/code.php index e7f66f082..e90a17376 100644 --- a/examples/topics/partitioning/reading/code.php +++ b/examples/topics/partitioning/reading/code.php @@ -3,26 +3,11 @@ declare(strict_types=1); use function Flow\ETL\Adapter\CSV\from_csv; -use function Flow\ETL\DSL\{data_frame, to_output}; +use function Flow\ETL\DSL\{data_frame, to_stream}; require __DIR__ . '/../../../autoload.php'; data_frame() ->read(from_csv(__DIR__ . '/input/color=*/sku=*/*.csv')) - ->collect() - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); - -// +----+-------+-----------+ -// | id | color | sku | -// +----+-------+-----------+ -// | 5 | green | PRODUCT02 | -// | 6 | green | PRODUCT03 | -// | 4 | green | PRODUCT01 | -// | 2 | red | PRODUCT02 | -// | 3 | red | PRODUCT03 | -// | 1 | red | PRODUCT01 | -// | 8 | blue | PRODUCT02 | -// | 7 | blue | PRODUCT01 | -// +----+-------+-----------+ -// 8 rows diff --git a/examples/topics/partitioning/reading/output.txt b/examples/topics/partitioning/reading/output.txt new file mode 100644 index 000000000..eda133b8e --- /dev/null +++ b/examples/topics/partitioning/reading/output.txt @@ -0,0 +1,72 @@ ++----+-------+-----------+ +| id | color | sku | ++----+-------+-----------+ +| 5 | green | PRODUCT02 | ++----+-------+-----------+ +Partitions: + - color=green + - sku=PRODUCT02 +1 rows ++----+-------+-----------+ +| id | color | sku | ++----+-------+-----------+ +| 6 | green | PRODUCT03 | ++----+-------+-----------+ +Partitions: + - color=green + - sku=PRODUCT03 +1 rows ++----+-------+-----------+ +| id | color | sku | ++----+-------+-----------+ +| 4 | green | PRODUCT01 | ++----+-------+-----------+ +Partitions: + - color=green + - sku=PRODUCT01 +1 rows ++----+-------+-----------+ +| id | color | sku | ++----+-------+-----------+ +| 2 | red | PRODUCT02 | ++----+-------+-----------+ +Partitions: + - color=red + - sku=PRODUCT02 +1 rows ++----+-------+-----------+ +| id | color | sku | ++----+-------+-----------+ +| 3 | red | PRODUCT03 | ++----+-------+-----------+ +Partitions: + - color=red + - sku=PRODUCT03 +1 rows ++----+-------+-----------+ +| id | color | sku | ++----+-------+-----------+ +| 1 | red | PRODUCT01 | ++----+-------+-----------+ +Partitions: + - color=red + - sku=PRODUCT01 +1 rows ++----+-------+-----------+ +| id | color | sku | ++----+-------+-----------+ +| 8 | blue | PRODUCT02 | ++----+-------+-----------+ +Partitions: + - color=blue + - sku=PRODUCT02 +1 rows ++----+-------+-----------+ +| id | color | sku | ++----+-------+-----------+ +| 7 | blue | PRODUCT01 | ++----+-------+-----------+ +Partitions: + - color=blue + - sku=PRODUCT01 +1 rows diff --git a/examples/topics/schema/display/code.php b/examples/topics/schema/display/code.php index 5630f6252..cfe2d5893 100644 --- a/examples/topics/schema/display/code.php +++ b/examples/topics/schema/display/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, from_array, to_output}; +use function Flow\ETL\DSL\{data_frame, from_array, to_stream}; use Flow\ETL\Loader\StreamLoader\Output; require __DIR__ . '/../../../autoload.php'; @@ -15,5 +15,5 @@ ['id' => 3, 'name' => 'Product 3', 'active' => true], ])) ->collect() - ->write(to_output(false, Output::schema)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false, output: Output::schema)) ->run(); diff --git a/examples/topics/schema/display/output.txt b/examples/topics/schema/display/output.txt new file mode 100644 index 000000000..4697cec67 --- /dev/null +++ b/examples/topics/schema/display/output.txt @@ -0,0 +1,8 @@ +schema +|-- id: integer +|-- name: string +|-- active: boolean +|-- tags: list +|-- address: structure +| |-- city: string +| |-- country: string diff --git a/examples/topics/schema/inferring/code.php b/examples/topics/schema/inferring/code.php index fd12fb987..a3d119abb 100644 --- a/examples/topics/schema/inferring/code.php +++ b/examples/topics/schema/inferring/code.php @@ -3,13 +3,13 @@ declare(strict_types=1); use function Flow\ETL\Adapter\CSV\from_csv; -use function Flow\ETL\DSL\{df, schema_from_json, schema_to_json, to_output}; +use function Flow\ETL\DSL\{data_frame, schema_from_json, schema_to_json, to_stream}; use Flow\ETL\Loader\StreamLoader\Output; require __DIR__ . '/../../../autoload.php'; if (!\file_exists(__DIR__ . '/output/schema.json')) { - $schema = df() + $schema = data_frame() ->read(from_csv(__DIR__ . '/input/dataset.csv')) ->limit(100) // Limiting the number of rows to read will speed up the process but might bring less accurate results ->autoCast() @@ -22,8 +22,8 @@ } // Reading schemaless data formats with predefined schema can significantly improve performance -df() +data_frame() ->read(from_csv(__DIR__ . '/input/dataset.csv', schema: $schema)) ->collect() - ->write(to_output(truncate: false, output: Output::rows_and_schema)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false, output: Output::rows_and_schema)) ->run(); diff --git a/examples/topics/schema/inferring/output.txt b/examples/topics/schema/inferring/output.txt new file mode 100644 index 000000000..967572c3d --- /dev/null +++ b/examples/topics/schema/inferring/output.txt @@ -0,0 +1,36 @@ ++-------+-----------------+------------------------------+--------------------------------+-------------------------------+----------------------------------------------+---------+----------------------------------+---------------------+ +| Index | Organization Id | Name | Website | Country | Description | Founded | Industry | Number of employees | ++-------+-----------------+------------------------------+--------------------------------+-------------------------------+----------------------------------------------+---------+----------------------------------+---------------------+ +| 1 | 8cC6B5992C0309c | Acevedo LLC | https://www.donovan.com/ | Holy See (Vatican City State) | Multi-channeled bottom-line core | 2019 | Graphic Design / Web Design | 7070 | +| 2 | ec094061FeaF7Bc | Walls-Mcdonald | http://arias-willis.net/ | Lithuania | Compatible encompassing groupware | 2005 | Utilities | 8156 | +| 3 | DAcC5dbc58946A7 | Gregory PLC | http://www.lynch-hoover.net/ | Tokelau | Multi-channeled intangible help-desk | 2019 | Leisure / Travel | 6121 | +| 4 | 8Dd7beDa37FbeD0 | Byrd, Patterson and Knox | https://www.james-velez.net/ | Netherlands | Pre-emptive national function | 1982 | Furniture | 3494 | +| 5 | a3b5c54AEC163e4 | Mcdowell-Hopkins | http://fuentes.com/ | Mayotte | Cloned bifurcated solution | 2016 | Online Publishing | 36 | +| 6 | fDfEBeFDaEb59Af | Hayden and Sons | https://www.shaw-mooney.info/ | Belize | Persistent mobile task-force | 1978 | Insurance | 7010 | +| 7 | 752ef90Eae1f7f5 | Castro LLC | http://wilkinson.com/ | Jamaica | Advanced value-added definition | 2008 | Outsourcing / Offshoring | 2526 | +| 8 | B1D4c5CA34f9992 | Barajas, Baird and Shaw | http://www.jordan-harvey.com/ | United States of America | Stand-alone bandwidth-monitored algorithm | 2000 | Wholesale | 4478 | +| 9 | Cfa1a44106faD4B | Lucas, Galloway and Benjamin | http://silva.info/ | Western Sahara | Persevering leadingedge ability | 1990 | Retail Industry | 8223 | +| 10 | C08fcf292AB17DF | Barker, Hubbard and Bennett | http://www.allen.biz/ | Mauritania | Decentralized fault-tolerant functionalities | 2014 | Museums / Institutions | 7716 | +| 11 | 94B9bEedc626820 | Underwood-Mitchell | https://www.leonard.com/ | Italy | Compatible dynamic support | 1992 | Fine Art | 4564 | +| 12 | FE42dEd40f5DfD8 | Lester, Ochoa and Franco | http://www.munoz.com/ | Timor-Leste | Vision-oriented dynamic conglomeration | 2014 | Motion Pictures / Film | 8075 | +| 13 | 1F861fAbeDdCFea | Arias, Jackson and Hester | https://hardin-thompson.com/ | Algeria | Switchable maximized synergy | 1980 | Utilities | 1319 | +| 14 | 456de7dE1ab18ca | Riggs and Sons | http://klein-benton.info/ | Czech Republic | Object-based discrete orchestration | 2012 | Law Enforcement | 4946 | +| 15 | 457bcfFF18A7DD2 | Stanley LLC | https://bowman.com/ | Eritrea | Self-enabling 24/7 groupware | 1984 | Executive Office | 4980 | +| 16 | 5B5ea5aea34dc5F | Page-Ware | http://lam-soto.com/ | Togo | Realigned mobile groupware | 1991 | Entertainment / Movie Production | 1307 | +| 17 | A66F35C298Dfd82 | Garner, Melton and Burgess | https://mathews-knox.com/ | Guinea-Bissau | Automated 5thgeneration complexity | 2003 | E - Learning | 9038 | +| 18 | EdAC2EF13734E0B | Andersen-Fuentes | http://www.mann.com/ | Oman | Ameliorated coherent database | 1991 | Textiles | 6436 | +| 19 | dD1612190b24B12 | Ford-Rice | https://peterson-irwin.com/ | Turks and Caicos Islands | Sharable intangible leverage | 1971 | Computer / Network Security | 3038 | +| 20 | 992CAdffccEebEa | Collins-Figueroa | http://www.holt-bartlett.info/ | Mongolia | Realigned multi-state installation | 1985 | Aviation / Aerospace | 9420 | ++-------+-----------------+------------------------------+--------------------------------+-------------------------------+----------------------------------------------+---------+----------------------------------+---------------------+ +20 rows + +schema +|-- Index: integer +|-- Organization Id: string +|-- Name: string +|-- Website: string +|-- Country: string +|-- Description: string +|-- Founded: integer +|-- Industry: string +|-- Number of employees: integer diff --git a/examples/topics/schema/validate/code.php b/examples/topics/schema/validate/code.php index d8592eb96..c84ec3271 100644 --- a/examples/topics/schema/validate/code.php +++ b/examples/topics/schema/validate/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{bool_schema, df, from_array, int_schema, schema, str_schema, to_output}; +use function Flow\ETL\DSL\{bool_schema, data_frame, from_array, int_schema, schema, str_schema, to_stream}; use Flow\ETL\Loader\StreamLoader\Output; use Flow\ETL\Row\Schema\Metadata; @@ -14,12 +14,13 @@ bool_schema('active', $nullable = false, Metadata::empty()->add('key', 'value')), ); -df() +data_frame() ->read(from_array([ ['id' => 1, 'name' => 'Product 1', 'active' => true], ['id' => 2, 'name' => 'Product 2', 'active' => false], ['id' => 3, 'name' => 'Product 3', 'active' => true], ])) ->validate($schema) - ->write(to_output(false, Output::rows_and_schema)) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false, output: Output::rows_and_schema)) ->run(); diff --git a/examples/topics/schema/validate/output.txt b/examples/topics/schema/validate/output.txt new file mode 100644 index 000000000..2edcab617 --- /dev/null +++ b/examples/topics/schema/validate/output.txt @@ -0,0 +1,13 @@ ++----+-----------+--------+ +| id | name | active | ++----+-----------+--------+ +| 1 | Product 1 | true | +| 2 | Product 2 | false | +| 3 | Product 3 | true | ++----+-----------+--------+ +3 rows + +schema +|-- id: integer +|-- name: string +|-- active: boolean diff --git a/examples/topics/transformations/array_expand/code.php b/examples/topics/transformations/array_expand/code.php index 1eca2d734..592ec57bc 100644 --- a/examples/topics/transformations/array_expand/code.php +++ b/examples/topics/transformations/array_expand/code.php @@ -2,15 +2,24 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{array_entry, array_expand, df, from_rows, int_entry, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{array_entry, + array_expand, + data_frame, + from_rows, + int_entry, + ref, + row, + rows, + to_output, + to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->read(from_rows(rows( row(int_entry('id', 1), array_entry('array', ['a' => 1, 'b' => 2, 'c' => 3])), ))) ->write(to_output(false)) ->withEntry('expanded', array_expand(ref('array'))) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/transformations/array_expand/output.txt b/examples/topics/transformations/array_expand/output.txt new file mode 100644 index 000000000..ccb027ada --- /dev/null +++ b/examples/topics/transformations/array_expand/output.txt @@ -0,0 +1,8 @@ ++----+---------------------+----------+ +| id | array | expanded | ++----+---------------------+----------+ +| 1 | {"a":1,"b":2,"c":3} | 1 | +| 1 | {"a":1,"b":2,"c":3} | 2 | +| 1 | {"a":1,"b":2,"c":3} | 3 | ++----+---------------------+----------+ +3 rows diff --git a/examples/topics/transformations/array_unpack/code.php b/examples/topics/transformations/array_unpack/code.php index 77cd889a1..6da4276b6 100644 --- a/examples/topics/transformations/array_unpack/code.php +++ b/examples/topics/transformations/array_unpack/code.php @@ -2,16 +2,16 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{array_entry, df, from_rows, int_entry, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{array_entry, data_frame, from_rows, int_entry, ref, row, rows, to_output, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->read(from_rows(rows( row(int_entry('id', 1), array_entry('array', ['a' => 1, 'b' => 2, 'c' => 3])), row(int_entry('id', 2), array_entry('array', ['d' => 4, 'e' => 5, 'f' => 6])), ))) ->write(to_output(false)) ->withEntry('unpacked', ref('array')->unpack()) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/transformations/array_unpack/output.txt b/examples/topics/transformations/array_unpack/output.txt new file mode 100644 index 000000000..ddc50249b --- /dev/null +++ b/examples/topics/transformations/array_unpack/output.txt @@ -0,0 +1,7 @@ ++----+---------------------+------------+------------+------------+------------+------------+------------+ +| id | array | unpacked.a | unpacked.b | unpacked.c | unpacked.d | unpacked.e | unpacked.f | ++----+---------------------+------------+------------+------------+------------+------------+------------+ +| 1 | {"a":1,"b":2,"c":3} | 1 | 2 | 3 | | | | +| 2 | {"d":4,"e":5,"f":6} | | | | 4 | 5 | 6 | ++----+---------------------+------------+------------+------------+------------+------------+------------+ +2 rows diff --git a/examples/topics/transformations/filter_divide/code.php b/examples/topics/transformations/filter_divide/code.php index dfdf9e158..cc89cf4d2 100644 --- a/examples/topics/transformations/filter_divide/code.php +++ b/examples/topics/transformations/filter_divide/code.php @@ -2,16 +2,16 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, from_rows, int_entry, lit, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{data_frame, from_rows, int_entry, lit, ref, row, rows, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->read(from_rows(rows( row(int_entry('a', 100), int_entry('b', 100)), row(int_entry('a', 100), int_entry('b', 200)) ))) - ->filter(ref('b')->divide(lit(2))->equals(lit('a'))) + ->filter(ref('b')->divide(lit(2))->same(ref('a'))) ->withEntry('new_b', ref('b')->multiply(lit(2))->multiply(lit(5))) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/transformations/filter_divide/output.txt b/examples/topics/transformations/filter_divide/output.txt new file mode 100644 index 000000000..6fe6a565b --- /dev/null +++ b/examples/topics/transformations/filter_divide/output.txt @@ -0,0 +1,6 @@ ++-----+-----+-------+ +| a | b | new_b | ++-----+-----+-------+ +| 100 | 200 | 2000 | ++-----+-----+-------+ +1 rows diff --git a/examples/topics/transformations/filter_mod/code.php b/examples/topics/transformations/filter_mod/code.php index 7f14230f6..b180de534 100644 --- a/examples/topics/transformations/filter_mod/code.php +++ b/examples/topics/transformations/filter_mod/code.php @@ -2,15 +2,15 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, from_rows, int_entry, lit, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{data_frame, from_rows, int_entry, lit, ref, row, rows, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->read(from_rows(rows( row(int_entry('a', 4), int_entry('b', 5)), row(int_entry('a', 3), int_entry('b', 6)) ))) ->filter(ref('b')->mod(lit(2))->equals(lit(0))) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/transformations/filter_mod/output.txt b/examples/topics/transformations/filter_mod/output.txt new file mode 100644 index 000000000..50ae40c9b --- /dev/null +++ b/examples/topics/transformations/filter_mod/output.txt @@ -0,0 +1,6 @@ ++---+---+ +| a | b | ++---+---+ +| 3 | 6 | ++---+---+ +1 rows diff --git a/examples/topics/transformations/literals/code.php b/examples/topics/transformations/literals/code.php index a11cac9a4..29efff405 100644 --- a/examples/topics/transformations/literals/code.php +++ b/examples/topics/transformations/literals/code.php @@ -2,14 +2,14 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, from_rows, lit, row, rows, str_entry, to_output}; +use function Flow\ETL\DSL\{data_frame, from_rows, lit, row, rows, str_entry, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->read(from_rows(rows( row(str_entry('name', 'Norbert')) ))) ->withEntry('number', lit(1)) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/transformations/literals/output.txt b/examples/topics/transformations/literals/output.txt new file mode 100644 index 000000000..82b3bd047 --- /dev/null +++ b/examples/topics/transformations/literals/output.txt @@ -0,0 +1,6 @@ ++---------+--------+ +| name | number | ++---------+--------+ +| Norbert | 1 | ++---------+--------+ +1 rows diff --git a/examples/topics/transformations/math/code.php b/examples/topics/transformations/math/code.php index 36eeece6a..f8cb759d8 100644 --- a/examples/topics/transformations/math/code.php +++ b/examples/topics/transformations/math/code.php @@ -2,16 +2,16 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, from_rows, int_entry, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{data_frame, from_rows, int_entry, ref, row, rows, to_output, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->read(from_rows(rows( row(int_entry('a', 100), int_entry('b', 200)) ))) ->write(to_output(false)) ->withEntry('c', ref('a')->plus(ref('b'))) ->withEntry('d', ref('b')->minus(ref('a'))) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/transformations/math/output.txt b/examples/topics/transformations/math/output.txt new file mode 100644 index 000000000..7cec54cad --- /dev/null +++ b/examples/topics/transformations/math/output.txt @@ -0,0 +1,6 @@ ++-----+-----+-----+-----+ +| a | b | c | d | ++-----+-----+-----+-----+ +| 100 | 200 | 300 | 100 | ++-----+-----+-----+-----+ +1 rows diff --git a/examples/topics/transformations/size/code.php b/examples/topics/transformations/size/code.php index cf2409fcb..01dca631d 100644 --- a/examples/topics/transformations/size/code.php +++ b/examples/topics/transformations/size/code.php @@ -2,14 +2,14 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{array_entry, df, from_rows, int_entry, ref, row, rows, to_output}; +use function Flow\ETL\DSL\{array_entry, data_frame, from_rows, int_entry, ref, row, rows, to_stream}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->read(from_rows(rows( row(int_entry('id', 1), array_entry('array', ['a' => 1, 'b' => 2, 'c' => 3])), ))) ->withEntry('array_size', ref('array')->size()) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/transformations/size/output.txt b/examples/topics/transformations/size/output.txt new file mode 100644 index 000000000..a58732078 --- /dev/null +++ b/examples/topics/transformations/size/output.txt @@ -0,0 +1,6 @@ ++----+---------------------+------------+ +| id | array | array_size | ++----+---------------------+------------+ +| 1 | {"a":1,"b":2,"c":3} | 3 | ++----+---------------------+------------+ +1 rows diff --git a/examples/topics/transformations/sort/code.php b/examples/topics/transformations/sort/code.php index 87e1a8cea..daf5988cc 100644 --- a/examples/topics/transformations/sort/code.php +++ b/examples/topics/transformations/sort/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, from_sequence_number, ref, to_output}; +use function Flow\ETL\DSL\{data_frame, from_sequence_number, ref, to_stream}; require __DIR__ . '/../../../autoload.php'; @@ -10,5 +10,5 @@ ->read(from_sequence_number('id', 0, 10)) ->sortBy(ref('id')->desc()) ->collect() - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/transformations/sort/output.txt b/examples/topics/transformations/sort/output.txt new file mode 100644 index 000000000..c54f35f1c --- /dev/null +++ b/examples/topics/transformations/sort/output.txt @@ -0,0 +1,16 @@ ++----+ +| id | ++----+ +| 10 | +| 9 | +| 8 | +| 7 | +| 6 | +| 5 | +| 4 | +| 3 | +| 2 | +| 1 | +| 0 | ++----+ +11 rows diff --git a/examples/topics/transformations/when_null/code.php b/examples/topics/transformations/when_null/code.php index d15f03f12..77b0a78b2 100644 --- a/examples/topics/transformations/when_null/code.php +++ b/examples/topics/transformations/when_null/code.php @@ -2,11 +2,11 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, from_rows, int_entry, lit, null_entry, ref, row, rows, to_output, when}; +use function Flow\ETL\DSL\{data_frame, from_rows, int_entry, lit, null_entry, ref, row, rows, to_stream, when}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->read(from_rows(rows( row(int_entry('id', 1), int_entry('value', 1)), row(int_entry('id', 2), int_entry('value', 1)), @@ -18,5 +18,5 @@ 'value', when(ref('value')->isNull(), then: lit(0)) ) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/transformations/when_null/output.txt b/examples/topics/transformations/when_null/output.txt new file mode 100644 index 000000000..27ae90a2c --- /dev/null +++ b/examples/topics/transformations/when_null/output.txt @@ -0,0 +1,10 @@ ++----+-------+ +| id | value | ++----+-------+ +| 1 | false | +| 2 | false | +| 3 | 0 | +| 4 | false | +| 5 | 0 | ++----+-------+ +5 rows diff --git a/examples/topics/transformations/when_odd/code.php b/examples/topics/transformations/when_odd/code.php index 37b2bafe1..a3efaa493 100644 --- a/examples/topics/transformations/when_odd/code.php +++ b/examples/topics/transformations/when_odd/code.php @@ -2,11 +2,11 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{df, from_sequence_number, lit, ref, to_output, when}; +use function Flow\ETL\DSL\{data_frame, from_sequence_number, lit, ref, to_stream, when}; require __DIR__ . '/../../../autoload.php'; -df() +data_frame() ->read(from_sequence_number('number', 1, 100)) ->collect() ->withEntry( @@ -17,5 +17,5 @@ else: lit('even') ) ) - ->write(to_output(false)) + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/transformations/when_odd/output.txt b/examples/topics/transformations/when_odd/output.txt new file mode 100644 index 000000000..fe59e3b65 --- /dev/null +++ b/examples/topics/transformations/when_odd/output.txt @@ -0,0 +1,105 @@ ++--------+------+ +| number | type | ++--------+------+ +| 1 | odd | +| 2 | even | +| 3 | odd | +| 4 | even | +| 5 | odd | +| 6 | even | +| 7 | odd | +| 8 | even | +| 9 | odd | +| 10 | even | +| 11 | odd | +| 12 | even | +| 13 | odd | +| 14 | even | +| 15 | odd | +| 16 | even | +| 17 | odd | +| 18 | even | +| 19 | odd | +| 20 | even | +| 21 | odd | +| 22 | even | +| 23 | odd | +| 24 | even | +| 25 | odd | +| 26 | even | +| 27 | odd | +| 28 | even | +| 29 | odd | +| 30 | even | +| 31 | odd | +| 32 | even | +| 33 | odd | +| 34 | even | +| 35 | odd | +| 36 | even | +| 37 | odd | +| 38 | even | +| 39 | odd | +| 40 | even | +| 41 | odd | +| 42 | even | +| 43 | odd | +| 44 | even | +| 45 | odd | +| 46 | even | +| 47 | odd | +| 48 | even | +| 49 | odd | +| 50 | even | +| 51 | odd | +| 52 | even | +| 53 | odd | +| 54 | even | +| 55 | odd | +| 56 | even | +| 57 | odd | +| 58 | even | +| 59 | odd | +| 60 | even | +| 61 | odd | +| 62 | even | +| 63 | odd | +| 64 | even | +| 65 | odd | +| 66 | even | +| 67 | odd | +| 68 | even | +| 69 | odd | +| 70 | even | +| 71 | odd | +| 72 | even | +| 73 | odd | +| 74 | even | +| 75 | odd | +| 76 | even | +| 77 | odd | +| 78 | even | +| 79 | odd | +| 80 | even | +| 81 | odd | +| 82 | even | +| 83 | odd | +| 84 | even | +| 85 | odd | +| 86 | even | +| 87 | odd | +| 88 | even | +| 89 | odd | +| 90 | even | +| 91 | odd | +| 92 | even | +| 93 | odd | +| 94 | even | +| 95 | odd | +| 96 | even | +| 97 | odd | +| 98 | even | +| 99 | odd | +| 100 | even | ++--------+------+ +100 rows diff --git a/examples/topics/window_functions/dens_rank/code.php b/examples/topics/window_functions/dens_rank/code.php index 945a276a3..4b4022037 100644 --- a/examples/topics/window_functions/dens_rank/code.php +++ b/examples/topics/window_functions/dens_rank/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, dense_rank, from_array, ref, to_output, window}; +use function Flow\ETL\DSL\{data_frame, dense_rank, from_array, ref, to_stream, window}; require __DIR__ . '/../../../autoload.php'; @@ -19,5 +19,6 @@ ) ->withEntry('rank', dense_rank()->over(window()->partitionBy(ref('department'))->orderBy(ref('salary')->desc()))) ->sortBy(ref('department'), ref('rank')) - ->write(to_output(false)) + ->collect() + ->write(to_stream(__DIR__ . '/output.txt', truncate: false)) ->run(); diff --git a/examples/topics/window_functions/dens_rank/output.txt b/examples/topics/window_functions/dens_rank/output.txt new file mode 100644 index 000000000..88af42ba0 --- /dev/null +++ b/examples/topics/window_functions/dens_rank/output.txt @@ -0,0 +1,11 @@ ++----+--------+------------+--------+------+ +| id | name | department | salary | rank | ++----+--------+------------+--------+------+ +| 5 | Jane | Finances | 14000 | 1 | +| 3 | Tomas | Finances | 11000 | 2 | +| 4 | John | Finances | 9000 | 3 | +| 6 | Janet | Finances | 4000 | 4 | +| 1 | Greg | IT | 6000 | 1 | +| 2 | Michal | IT | 5000 | 2 | ++----+--------+------------+--------+------+ +6 rows diff --git a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/DbalLoaderTest.php b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/DbalLoaderTest.php index fc14ea831..4de207d68 100644 --- a/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/DbalLoaderTest.php +++ b/src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests/Integration/DbalLoaderTest.php @@ -73,10 +73,10 @@ public function test_inserts_multiple_rows_at_once() : void (new Flow()) ->read( from_array([ - ['id' => 1, 'name' => 'Name One', 'description' => 'Description One'], - ['id' => 2, 'name' => 'Name Two', 'description' => 'Description Two'], - ['id' => 3, 'name' => 'Name Three', 'description' => 'Description Three'], - ]) + ['id' => 1, 'name' => 'Name One', 'description' => 'Description One'], + ['id' => 2, 'name' => 'Name Two', 'description' => 'Description Two'], + ['id' => 3, 'name' => 'Name Three', 'description' => 'Description Three'], + ]) ) ->load($loader) ->run(); @@ -213,10 +213,10 @@ public function test_inserts_new_rows_or_updates_already_existed_based_on_primar (new Flow())->extract( from_array([ - ['id' => 2, 'name' => 'New Name Two', 'description' => 'New Description Two'], - ['id' => 3, 'name' => 'New Name Three', 'description' => 'New Description Three'], - ['id' => 4, 'name' => 'New Name Four', 'description' => 'New Description Three'], - ]) + ['id' => 2, 'name' => 'New Name Two', 'description' => 'New Description Two'], + ['id' => 3, 'name' => 'New Name Three', 'description' => 'New Description Three'], + ['id' => 4, 'name' => 'New Name Four', 'description' => 'New Description Three'], + ]) ) ->load(to_dbal_table_insert($this->connectionParams(), $table, ['constraint' => 'flow_doctrine_bulk_test_pkey'])) ->run(); @@ -264,7 +264,7 @@ public function test_update_multiple_rows_at_once() : void ['id' => 1, 'name' => 'Changed Name One', 'description' => 'Description One'], ['id' => 2, 'name' => 'Name Two', 'description' => 'Description Two'], ['id' => 3, 'name' => 'Changed Name Three', 'description' => 'Description Three'], - ]) + ]) ) ->load($updateLoader) ->run(); diff --git a/src/core/etl/src/Flow/ETL/DSL/functions.php b/src/core/etl/src/Flow/ETL/DSL/functions.php index 6d940d367..50362e040 100644 --- a/src/core/etl/src/Flow/ETL/DSL/functions.php +++ b/src/core/etl/src/Flow/ETL/DSL/functions.php @@ -143,7 +143,7 @@ function to_stdout(int|bool $truncate = 20, Output $output = Output::rows, Forma function to_stream(string $uri, int|bool $truncate = 20, Output $output = Output::rows, string $mode = 'w', Formatter $formatter = new Formatter\AsciiTableFormatter(), SchemaFormatter $schemaFormatter = new ASCIISchemaFormatter()) : StreamLoader { - return new StreamLoader($uri, Mode::from($mode), $truncate, $output, $formatter, $schemaFormatter); + return new StreamLoader($uri, Mode::from($mode), $truncate, $output, $formatter, $schemaFormatter, StreamLoader\Type::custom); } function to_transformation(Transformer $transformer, Loader $loader) : TransformerLoader diff --git a/src/core/etl/src/Flow/ETL/Loader/StreamLoader.php b/src/core/etl/src/Flow/ETL/Loader/StreamLoader.php index 85c3bda80..c71889fcb 100644 --- a/src/core/etl/src/Flow/ETL/Loader/StreamLoader.php +++ b/src/core/etl/src/Flow/ETL/Loader/StreamLoader.php @@ -9,10 +9,15 @@ use Flow\ETL\Loader\StreamLoader\Output; use Flow\ETL\Row\Schema\Formatter\ASCIISchemaFormatter; use Flow\ETL\Row\Schema\SchemaFormatter; -use Flow\ETL\{FlowContext, Formatter, Loader, Rows}; +use Flow\ETL\{FlowContext, Formatter, Loader, Loader\StreamLoader\Type, Rows}; -final class StreamLoader implements Loader +final class StreamLoader implements Closure, Loader { + /** + * @var null|resource + */ + private $stream; + /** * @param string $url all protocols supported by PHP are allowed https://www.php.net/manual/en/wrappers.php * @param Mode $mode only writing modes explained in https://www.php.net/manual/en/function.fopen.php are supported @@ -25,36 +30,35 @@ public function __construct( private readonly int|bool $truncate = 20, private readonly Output $output = Output::rows, private readonly Formatter $formatter = new Formatter\AsciiTableFormatter(), - private readonly SchemaFormatter $schemaFormatter = new ASCIISchemaFormatter() + private readonly SchemaFormatter $schemaFormatter = new ASCIISchemaFormatter(), + private readonly Type $type = Type::custom, ) { + $this->stream = null; } public static function output(int|bool $truncate = 20, Output $output = Output::rows, Formatter $formatter = new Formatter\AsciiTableFormatter(), SchemaFormatter $schemaFormatter = new ASCIISchemaFormatter()) : self { - return new self('php://output', Mode::WRITE, $truncate, $output, $formatter, $schemaFormatter); + return new self('php://output', Mode::WRITE, $truncate, $output, $formatter, $schemaFormatter, Type::output); } public static function stderr(int|bool $truncate = 20, Output $output = Output::rows, Formatter $formatter = new Formatter\AsciiTableFormatter(), SchemaFormatter $schemaFormatter = new ASCIISchemaFormatter()) : self { - return new self('php://stderr', Mode::WRITE, $truncate, $output, $formatter, $schemaFormatter); + return new self('php://stderr', Mode::WRITE, $truncate, $output, $formatter, $schemaFormatter, Type::stderr); } public static function stdout(int|bool $truncate = 20, Output $output = Output::rows, Formatter $formatter = new Formatter\AsciiTableFormatter(), SchemaFormatter $schemaFormatter = new ASCIISchemaFormatter()) : self { - return new self('php://stdout', Mode::WRITE, $truncate, $output, $formatter, $schemaFormatter); + return new self('php://stdout', Mode::WRITE, $truncate, $output, $formatter, $schemaFormatter, Type::stdout); } - public function load(Rows $rows, FlowContext $context) : void + public function closure(FlowContext $context) : void { - try { - $stream = @\fopen($this->url, $this->mode->value); - } catch (\Throwable $e) { - throw new RuntimeException("Can't open stream for url: {$this->url} in mode: {$this->mode->value}. Reason: " . $e->getMessage(), (int) $e->getCode(), $e); - } + $this->closeStream(); + } - if ($stream === false) { - throw new RuntimeException("Can't open stream for url: {$this->url} in mode: {$this->mode->value}"); - } + public function load(Rows $rows, FlowContext $context) : void + { + $stream = $this->getStream(); \fwrite( $stream, @@ -65,6 +69,45 @@ public function load(Rows $rows, FlowContext $context) : void } ); - \fclose($stream); + match ($this->type) { + Type::output => $this->closeStream(), + Type::stderr => $this->closeStream(), + Type::stdout => $this->closeStream(), + Type::custom => null, + }; + } + + private function closeStream() : void + { + if ($this->stream !== null) { + /** + * @psalm-suppress InvalidPropertyAssignmentValue + */ + \fclose($this->stream); + $this->stream = null; + } + } + + /** + * @return resource + */ + private function getStream() + { + if ($this->stream !== null) { + return $this->stream; + } + + try { + /** @phpstan-ignore-next-line */ + $this->stream = @\fopen($this->url, $this->mode->value); + } catch (\Throwable $e) { + throw new RuntimeException("Can't open stream for url: {$this->url} in mode: {$this->mode->value}. Reason: " . $e->getMessage(), (int) $e->getCode(), $e); + } + + if ($this->stream === false) { + throw new RuntimeException("Can't open stream for url: {$this->url} in mode: {$this->mode->value}"); + } + + return $this->stream; } } diff --git a/src/core/etl/src/Flow/ETL/Loader/StreamLoader/Type.php b/src/core/etl/src/Flow/ETL/Loader/StreamLoader/Type.php new file mode 100644 index 000000000..b341b6840 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Loader/StreamLoader/Type.php @@ -0,0 +1,13 @@ + $i % 2 === 0 ? \array_map(static fn ($i) => [ - 'id' => $faker->numberBetween(0, Consts::PHP_INT32_MAX), - 'name' => $faker->text(10), - ], \range(1, \random_int(2, 10))) + 'id' => $faker->numberBetween(0, Consts::PHP_INT32_MAX), + 'name' => $faker->text(10), + ], \range(1, \random_int(2, 10))) : null, ], ]; diff --git a/web/landing/assets/controllers/syntax_highlight_controller.js b/web/landing/assets/controllers/syntax_highlight_controller.js index 470733f4b..3ed025399 100644 --- a/web/landing/assets/controllers/syntax_highlight_controller.js +++ b/web/landing/assets/controllers/syntax_highlight_controller.js @@ -1,6 +1,7 @@ import {Controller} from '@hotwired/stimulus'; import 'highlight.js/styles/github-dark.min.css'; import php from 'highlight.js/lib/languages/php'; +import shell from 'highlight.js/lib/languages/shell'; import hljs from 'highlight.js/lib/core'; /* stimulusFetch: 'lazy' */ @@ -9,6 +10,7 @@ export default class extends Controller initialize() { hljs.registerLanguage('php', php); + hljs.registerLanguage('shell', shell); } connect() diff --git a/web/landing/importmap.php b/web/landing/importmap.php index 9576792be..309d0a551 100644 --- a/web/landing/importmap.php +++ b/web/landing/importmap.php @@ -44,16 +44,10 @@ 'clipboard' => [ 'version' => '2.0.11', ], - - /** - * On mobile there is a collapsible menu that uses relatively new popover attribute, - * but it's not yet available in a firefox browser: https://caniuse.com/?search=popover. - * This polyfill will make it work there. - * - * Once it's available, run 'bin/console importmap:remove @oddbird/popover-polyfill' - * and remove import from 'landing/assets/app.js'. - */ '@oddbird/popover-polyfill' => [ 'version' => '0.3.8', ], + 'highlight.js/lib/languages/shell' => [ + 'version' => '11.9.0', + ], ]; diff --git a/web/landing/src/Flow/Website/Controller/DefaultController.php b/web/landing/src/Flow/Website/Controller/DefaultController.php index 4bde5fa0f..f01deb347 100644 --- a/web/landing/src/Flow/Website/Controller/DefaultController.php +++ b/web/landing/src/Flow/Website/Controller/DefaultController.php @@ -32,6 +32,7 @@ public function example(string $topic, string $example) : Response 'currentTopic' => $topic, 'currentExample' => $example, 'code' => $this->examples->code($currentTopic, $currentExample), + 'output' => $this->examples->output($currentTopic, $currentExample), ]); } @@ -58,6 +59,7 @@ public function topic(string $topic) : Response 'currentTopic' => $currentTopic, 'currentExample' => $currentExample, 'code' => $this->examples->code($currentTopic, $currentExample), + 'output' => $this->examples->output($currentTopic, $currentExample), ]); } } diff --git a/web/landing/src/Flow/Website/Service/Examples.php b/web/landing/src/Flow/Website/Service/Examples.php index 96953865f..c6cef8a40 100644 --- a/web/landing/src/Flow/Website/Service/Examples.php +++ b/web/landing/src/Flow/Website/Service/Examples.php @@ -41,6 +41,17 @@ public function examples(string $topic) : array return $examples; } + public function output(string $topic, string $example) : ?string + { + $path = \sprintf('%s/topics/%s/%s/output.txt', \realpath($this->examplesPath), $topic, $example); + + if (false === \file_exists($path)) { + return null; + } + + return \file_get_contents($path); + } + /** * @return string[] */ diff --git a/web/landing/templates/example/index.html.twig b/web/landing/templates/example/index.html.twig index d6187a553..a42181208 100644 --- a/web/landing/templates/example/index.html.twig +++ b/web/landing/templates/example/index.html.twig @@ -48,6 +48,22 @@ {% endapply %} + + {% if output %} +

Output

+
+ {% apply spaceless %} +
+                    
+                    
+                        {{- output | escape('html') -}}
+                    
+                
+ {% endapply %} +
+ {% endif %} {% endblock %}