Skip to content

Commit c775d54

Browse files
norberttechnorberttech
authored
Added database upsert example (#1371)
1 parent bae474e commit c775d54

File tree

6 files changed

+153
-5
lines changed

6 files changed

+153
-5
lines changed

‎examples/topics/data_writing/database_upsert/code.php

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
use function Flow\ETL\Adapter\Doctrine\{from_dbal_query, to_dbal_table_insert};
6+
use function Flow\ETL\DSL\{data_frame, from_array, to_stream};
7+
use Doctrine\DBAL\DriverManager;
8+
use Doctrine\DBAL\Schema\{Column, Table, UniqueConstraint};
9+
use Doctrine\DBAL\Types\{Type, Types};
10+
11+
require __DIR__ . '/../../../autoload.php';
12+
13+
require __DIR__ . '/generate_static_orders.php';
14+
15+
$connection = DriverManager::getConnection([
16+
'path' => __DIR__ . '/output/orders.db',
17+
'driver' => 'pdo_sqlite',
18+
]);
19+
20+
$schemaManager = $connection->createSchemaManager();
21+
22+
if (!$schemaManager->tablesExist(['orders'])) {
23+
$schemaManager->createTable(new Table(
24+
$table = 'orders',
25+
[
26+
new Column('order_id', Type::getType(Types::GUID), ['notnull' => true]),
27+
new Column('created_at', Type::getType(Types::DATETIME_IMMUTABLE), ['notnull' => true]),
28+
new Column('updated_at', Type::getType(Types::DATETIME_IMMUTABLE), ['notnull' => false]),
29+
new Column('discount', Type::getType(Types::FLOAT), ['notnull' => false]),
30+
new Column('email', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]),
31+
new Column('customer', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]),
32+
new Column('address', Type::getType(Types::JSON), ['notnull' => true]),
33+
new Column('notes', Type::getType(Types::JSON), ['notnull' => true]),
34+
new Column('items', Type::getType(Types::JSON), ['notnull' => true]),
35+
],
36+
uniqueConstraints: [
37+
new UniqueConstraint('orders_order_id', ['order_id']),
38+
]
39+
));
40+
}
41+
42+
$orderIds = [
43+
'c0a43894-0102-4a4e-9fcd-393ef9e4f16a',
44+
'83fd51a4-9bd1-4b40-8f6e-6a7cc940bb5a',
45+
'7c65db1a-410f-4e91-8aeb-66fb3f1665f7',
46+
'5af1d56c-a9f7-411e-8738-865942d6c40f',
47+
'3a3ae1a9-debd-425a-8f9d-63c3315bc483',
48+
'27d8ee4d-94cc-47fa-bc14-209a4ab2eb45',
49+
'cc4fd722-1407-4781-9ad4-fa53966060af',
50+
'718360e1-c4c9-40f4-84e2-6f7898788883',
51+
'ea7c731c-ce3b-40bb-bbf8-79f1c717b6ca',
52+
'17b0d6c5-dd8f-4d5a-ae06-1df15b67c82c',
53+
];
54+
55+
data_frame()
56+
->read(from_array(generateStaticOrders($orderIds)))
57+
->write(
58+
to_dbal_table_insert(
59+
DriverManager::getConnection([
60+
'path' => __DIR__ . '/output/orders.db',
61+
'driver' => 'pdo_sqlite',
62+
]),
63+
'orders',
64+
[
65+
'conflict_columns' => ['order_id'],
66+
]
67+
)
68+
)
69+
// second insert that normally would fail due to Integrity constraint violation
70+
->write(
71+
to_dbal_table_insert(
72+
DriverManager::getConnection([
73+
'path' => __DIR__ . '/output/orders.db',
74+
'driver' => 'pdo_sqlite',
75+
]),
76+
'orders',
77+
[
78+
'conflict_columns' => ['order_id'],
79+
]
80+
)
81+
)
82+
->run();
83+
84+
data_frame()
85+
->read(
86+
from_dbal_query(
87+
DriverManager::getConnection([
88+
'path' => __DIR__ . '/output/orders.db',
89+
'driver' => 'pdo_sqlite',
90+
]),
91+
'SELECT COUNT(*) as total_rows FROM orders'
92+
)
93+
)
94+
->write(to_stream(__DIR__ . '/output.txt', truncate: false))
95+
->run();

‎examples/topics/data_writing/database_upsert/generate_static_orders.php

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* @param array<string> $ids
7+
*/
8+
function generateStaticOrders(array $ids) : Generator
9+
{
10+
$faker = Faker\Factory::create();
11+
12+
$skus = [
13+
['sku' => 'SKU_0001', 'name' => 'Product 1', 'price' => $faker->randomFloat(2, 0, 500)],
14+
['sku' => 'SKU_0002', 'name' => 'Product 2', 'price' => $faker->randomFloat(2, 0, 500)],
15+
['sku' => 'SKU_0003', 'name' => 'Product 3', 'price' => $faker->randomFloat(2, 0, 500)],
16+
['sku' => 'SKU_0004', 'name' => 'Product 4', 'price' => $faker->randomFloat(2, 0, 500)],
17+
['sku' => 'SKU_0005', 'name' => 'Product 5', 'price' => $faker->randomFloat(2, 0, 500)],
18+
];
19+
20+
foreach ($ids as $i => $id) {
21+
yield [
22+
'order_id' => $id,
23+
'created_at' => $faker->dateTimeThisYear,
24+
'updated_at' => \random_int(0, 1) === 1 ? $faker->dateTimeThisMonth : null,
25+
'discount' => \random_int(0, 1) === 1 ? $faker->randomFloat(2, 0, 50) : null,
26+
'email' => $faker->email,
27+
'customer' => $faker->firstName . ' ' . $faker->lastName,
28+
'address' => [
29+
'street' => $faker->streetAddress,
30+
'city' => $faker->city,
31+
'zip' => $faker->postcode,
32+
'country' => $faker->country,
33+
],
34+
'notes' => \array_map(
35+
static fn ($i) => $faker->sentence,
36+
\range(1, $faker->numberBetween(1, 5))
37+
),
38+
'items' => \array_map(
39+
static fn (int $index) => [
40+
'sku' => $skus[$skuIndex = $faker->numberBetween(1, 4)]['sku'],
41+
'quantity' => $faker->numberBetween(1, 10),
42+
'price' => $skus[$skuIndex]['price'],
43+
],
44+
\range(1, $faker->numberBetween(1, 4))
45+
),
46+
];
47+
}
48+
}

‎examples/topics/data_writing/database_upsert/output.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
+------------+
2+
| total_rows |
3+
+------------+
4+
| 10 |
5+
+------------+
6+
1 rows

‎examples/topics/data_writing/database_upsert/output/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
orders.db

‎src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/functions.php

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,7 @@
88
use Doctrine\DBAL\Types\Type as DbalType;
99
use Doctrine\DBAL\{ArrayParameterType as DbalArrayType, Connection, ParameterType as DbalParameterType};
1010
use Flow\ETL\Exception\InvalidArgumentException;
11-
use Flow\ETL\{Attribute\DocumentationDSL,
12-
Attribute\Module,
13-
Attribute\Type as DSLType
14-
};
11+
use Flow\ETL\{Attribute\DocumentationDSL, Attribute\DocumentationExample, Attribute\Module, Attribute\Type as DSLType};
1512

1613
/**
1714
* @param array<string, mixed>|Connection $connection
@@ -176,6 +173,7 @@ function dbal_from_query(
176173
* @throws InvalidArgumentException
177174
*/
178175
#[DocumentationDSL(module: Module::DOCTRINE, type: DSLType::LOADER)]
176+
#[DocumentationExample(topic: 'data_writing', example: 'database_upsert')]
179177
function to_dbal_table_insert(
180178
array|Connection $connection,
181179
string $table,

‎web/landing/resources/dsl.json

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)