Skip to content

Commit 51d33a7

Browse files
norberttechnorberttech
authored
Added missing logical types (#926)
* Added missing logical types * CS fixes * Fixed typo in the code
1 parent c20a191 commit 51d33a7

File tree

57 files changed

+970
-334
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+970
-334
lines changed

‎src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroLoader.php

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@
88
use Flow\ETL\FlowContext;
99
use Flow\ETL\Loader;
1010
use Flow\ETL\Loader\Closure;
11+
use Flow\ETL\PHP\Type\Logical\DateTimeType;
1112
use Flow\ETL\PHP\Type\Logical\ListType;
12-
use Flow\ETL\PHP\Type\Native\ObjectType;
13+
use Flow\ETL\PHP\Type\Logical\UuidType;
1314
use Flow\ETL\Row;
1415
use Flow\ETL\Row\Schema;
1516
use Flow\ETL\Rows;
@@ -78,28 +79,26 @@ private function listEntryToValues(Row\Entry\ListEntry $entry) : array
7879
$listType = $entry->definition()->metadata()->get(Schema\FlowMetadata::METADATA_LIST_ENTRY_TYPE);
7980
$listElement = $listType->element();
8081

81-
if ($listElement->type() instanceof ObjectType) {
82-
if (\is_a($listElement->type()->class, Row\Entry\Type\Uuid::class, true)) {
83-
/** @var array<string> $data */
84-
$data = [];
82+
if ($listElement->type() instanceof UuidType) {
83+
/** @var array<string> $data */
84+
$data = [];
8585

86-
foreach ($entry->value() as $value) {
87-
$data[] = $value->toString();
88-
}
89-
90-
return $data;
86+
foreach ($entry->value() as $value) {
87+
$data[] = $value->toString();
9188
}
9289

93-
if (\is_a($listElement->type()->class, \DateTimeInterface::class, true)) {
94-
/** @var array<int> $data */
95-
$data = [];
90+
return $data;
91+
}
9692

97-
foreach ($entry->value() as $value) {
98-
$data[] = (int) $value->format('Uu');
99-
}
93+
if ($listElement->type() instanceof DateTimeType) {
94+
/** @var array<int> $data */
95+
$data = [];
10096

101-
return $data;
97+
foreach ($entry->value() as $value) {
98+
$data[] = (int) $value->format('Uu');
10299
}
100+
101+
return $data;
103102
}
104103

105104
return $entry->value();

‎src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/SchemaConverter.php

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55
namespace Flow\ETL\Adapter\Avro\FlixTech;
66

77
use Flow\ETL\Exception\RuntimeException;
8+
use Flow\ETL\PHP\Type\Logical\DateTimeType;
89
use Flow\ETL\PHP\Type\Logical\ListType;
910
use Flow\ETL\PHP\Type\Logical\MapType;
1011
use Flow\ETL\PHP\Type\Logical\Structure\StructureElement;
1112
use Flow\ETL\PHP\Type\Logical\StructureType;
13+
use Flow\ETL\PHP\Type\Logical\UuidType;
1214
use Flow\ETL\PHP\Type\Native\ArrayType;
1315
use Flow\ETL\PHP\Type\Native\ObjectType;
1416
use Flow\ETL\PHP\Type\Native\ScalarType;
@@ -61,10 +63,8 @@ private function convert(Definition $definition) : array
6163
};
6264
}
6365

64-
if ($listElement->type() instanceof ObjectType) {
65-
if (\is_a($listElement->type()->class, \DateTimeInterface::class, true)) {
66-
return ['name' => $definition->entry()->name(), 'type' => ['type' => 'array', 'items' => 'long', \AvroSchema::LOGICAL_TYPE_ATTR => 'timestamp-micros']];
67-
}
66+
if ($listElement->type() instanceof DateTimeType) {
67+
return ['name' => $definition->entry()->name(), 'type' => ['type' => 'array', 'items' => 'long', \AvroSchema::LOGICAL_TYPE_ATTR => 'timestamp-micros']];
6868
}
6969

7070
throw new RuntimeException("List of {$listElement->toString()} is not supported yet supported.");
@@ -118,7 +118,7 @@ private function convert(Definition $definition) : array
118118
}
119119

120120
$avroType = match ($type) {
121-
Entry\StringEntry::class, Entry\JsonEntry::class, Entry\UuidEntry::class => ['name' => $definition->entry()->name(), 'type' => \AvroSchema::STRING_TYPE],
121+
Entry\StringEntry::class, Entry\JsonEntry::class, Entry\UuidEntry::class, Entry\XMLEntry::class, Entry\XMLNodeEntry::class => ['name' => $definition->entry()->name(), 'type' => \AvroSchema::STRING_TYPE],
122122
Entry\EnumEntry::class => [
123123
'name' => $definition->entry()->name(),
124124
'type' => [
@@ -172,15 +172,15 @@ private function structureElementToArvo(StructureElement $element) : array
172172
throw new RuntimeException("ArrayEntry entry can't be saved in Avro file, try convert it to ListEntry, MapEntry or StructEntry");
173173
}
174174

175-
if ($elementType instanceof ObjectType) {
176-
if (\in_array($elementType->class, [\DateTimeImmutable::class, \DateTimeInterface::class, \DateTime::class], true)) {
177-
return ['name' => $element->name(), 'type' => 'long', \AvroSchema::LOGICAL_TYPE_ATTR => 'timestamp-micros'];
178-
}
175+
if ($elementType instanceof DateTimeType) {
176+
return ['name' => $element->name(), 'type' => 'long', \AvroSchema::LOGICAL_TYPE_ATTR => 'timestamp-micros'];
177+
}
179178

180-
if ($elementType->class === Entry\Type\Uuid::class) {
181-
return ['name' => $element->name(), 'type' => \AvroSchema::STRING_TYPE];
182-
}
179+
if ($elementType instanceof UuidType) {
180+
return ['name' => $element->name(), 'type' => \AvroSchema::STRING_TYPE];
181+
}
183182

183+
if ($elementType instanceof ObjectType) {
184184
throw new RuntimeException($elementType->class . ' is not supported.');
185185
}
186186

‎src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Integration/AvroTest.php

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@
2222
use function Flow\ETL\DSL\struct_element;
2323
use function Flow\ETL\DSL\struct_entry;
2424
use function Flow\ETL\DSL\struct_type;
25+
use function Flow\ETL\DSL\type_datetime;
2526
use function Flow\ETL\DSL\type_float;
2627
use function Flow\ETL\DSL\type_list;
27-
use function Flow\ETL\DSL\type_object;
2828
use function Flow\ETL\DSL\type_string;
2929
use Flow\ETL\Adapter\Avro\FlixTech\AvroExtractor;
3030
use Flow\ETL\Config;
@@ -77,7 +77,7 @@ public function test_safe_writing_and_reading_avro_with_all_supported_types() :
7777
json_object_entry('json_object', ['id' => 1, 'name' => 'test']),
7878
json_entry('json', [['id' => 1, 'name' => 'test'], ['id' => 2, 'name' => 'test']]),
7979
list_entry('list_of_strings', ['a', 'b', 'c'], type_list(type_string())),
80-
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_object(\DateTimeImmutable::class)))
80+
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_datetime()))
8181
);
8282
}, \range(1, 100))
8383
)
@@ -155,7 +155,7 @@ public function test_writing_and_reading_avro_with_all_supported_types() : void
155155
json_object_entry('json_object', ['id' => 1, 'name' => 'test']),
156156
json_entry('json', [['id' => 1, 'name' => 'test'], ['id' => 2, 'name' => 'test']]),
157157
list_entry('list_of_strings', ['a', 'b', 'c'], type_list(type_string())),
158-
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_object(\DateTimeImmutable::class))),
158+
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_datetime())),
159159
struct_entry(
160160
'address',
161161
[
@@ -165,19 +165,19 @@ public function test_writing_and_reading_avro_with_all_supported_types() : void
165165
'country' => 'country_' . $i,
166166
'location' => ['lat' => 1.5, 'lon' => 1.5],
167167
],
168-
struct_type(
168+
struct_type([
169169
struct_element('street', type_string()),
170170
struct_element('city', type_string()),
171171
struct_element('zip', type_string()),
172172
struct_element('country', type_string()),
173173
struct_element(
174174
'location',
175-
struct_type(
175+
struct_type([
176176
struct_element('lat', type_float()),
177177
struct_element('lon', type_float()),
178-
)
179-
)
180-
),
178+
])
179+
),
180+
]),
181181
),
182182
);
183183
}, \range(1, 100))
@@ -219,7 +219,7 @@ public function test_writing_twice_to_the_same_location() : void
219219
json_object_entry('json_object', ['id' => 1, 'name' => 'test']),
220220
json_entry('json', [['id' => 1, 'name' => 'test'], ['id' => 2, 'name' => 'test']]),
221221
list_entry('list_of_strings', ['a', 'b', 'c'], type_list(type_string())),
222-
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_object(\DateTimeImmutable::class)))
222+
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_datetime()))
223223
);
224224
}, \range(1, 100))
225225
)
@@ -240,7 +240,7 @@ public function test_writing_twice_to_the_same_location() : void
240240
json_object_entry('json_object', ['id' => 1, 'name' => 'test']),
241241
json_entry('json', [['id' => 1, 'name' => 'test'], ['id' => 2, 'name' => 'test']]),
242242
list_entry('list_of_strings', ['a', 'b', 'c'], type_list(type_string())),
243-
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_object(\DateTimeImmutable::class)))
243+
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_datetime()))
244244
);
245245
}, \range(1, 100))
246246
)
@@ -277,7 +277,7 @@ public function test_writing_twice_to_the_same_location_with_ignore_mode() : voi
277277
json_object_entry('json_object', ['id' => 1, 'name' => 'test']),
278278
json_entry('json', [['id' => 1, 'name' => 'test'], ['id' => 2, 'name' => 'test']]),
279279
list_entry('list_of_strings', ['a', 'b', 'c'], type_list(type_string())),
280-
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_object(\DateTimeImmutable::class)))
280+
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_datetime()))
281281
);
282282
}, \range(1, 100))
283283
)
@@ -298,7 +298,7 @@ public function test_writing_twice_to_the_same_location_with_ignore_mode() : voi
298298
json_object_entry('json_object', ['id' => 1, 'name' => 'test']),
299299
json_entry('json', [['id' => 1, 'name' => 'test'], ['id' => 2, 'name' => 'test']]),
300300
list_entry('list_of_strings', ['a', 'b', 'c'], type_list(type_string())),
301-
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_object(\DateTimeImmutable::class)))
301+
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_datetime()))
302302
);
303303
}, \range(1, 100))
304304
)
@@ -336,7 +336,7 @@ public function test_writing_twice_to_the_same_location_with_overwrite_mode() :
336336
json_object_entry('json_object', ['id' => 1, 'name' => 'test']),
337337
json_entry('json', [['id' => 1, 'name' => 'test'], ['id' => 2, 'name' => 'test']]),
338338
list_entry('list_of_strings', ['a', 'b', 'c'], type_list(type_string())),
339-
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_object(\DateTimeImmutable::class)))
339+
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_datetime()))
340340
);
341341
}, \range(1, 100))
342342
)
@@ -357,7 +357,7 @@ public function test_writing_twice_to_the_same_location_with_overwrite_mode() :
357357
json_object_entry('json_object', ['id' => 1, 'name' => 'test']),
358358
json_entry('json', [['id' => 1, 'name' => 'test'], ['id' => 2, 'name' => 'test']]),
359359
list_entry('list_of_strings', ['a', 'b', 'c'], type_list(type_string())),
360-
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_object(\DateTimeImmutable::class)))
360+
list_entry('list_of_datetimes', [new \DateTimeImmutable(), new \DateTimeImmutable(), new \DateTimeImmutable()], type_list(type_datetime()))
361361
);
362362
}, \range(1, 100))
363363
)

‎src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/SchemaConverterTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ public function test_convert_etl_entries_to_avro_json() : void
9292
Schema\Definition::dateTime('datetime'),
9393
Schema\Definition::json('json'),
9494
Schema\Definition::list('list', new ListType(ListElement::string())),
95-
Schema\Definition::structure('structure', new StructureType(new StructureElement('a', type_string()))),
95+
Schema\Definition::structure('structure', new StructureType([new StructureElement('a', type_string())])),
9696
Schema\Definition::map('map', new MapType(MapKey::string(), MapValue::integer()))
9797
))
9898
);

0 commit comments

Comments
 (0)