diff --git a/docs/components/core/schema.md b/docs/components/core/schema.md index 930dd5e22..870bd335d 100644 --- a/docs/components/core/schema.md +++ b/docs/components/core/schema.md @@ -21,16 +21,6 @@ There is more than one way to validate the schema, built in strategies are defin By default, ETL is initializing `StrictValidator`, but it's possible to override it by passing second argument to `DataFrame::validate()` method. -### Schema Constraints - -- [all](../../../src/core/etl/src/Flow/ETL/Row/Schema/Constraint/All.php) -- [any](../../../src/core/etl/src/Flow/ETL/Row/Schema/Constraint/Any.php) -- [is instance of](../../../src/core/etl/src/Flow/ETL/Row/Schema/Constraint/IsInstanceOf.php) -- [list type](../../../src/core/etl/src/Flow/ETL/Row/Schema/Constraint/ListType.php) -- [not empty](../../../src/core/etl/src/Flow/ETL/Row/Schema/Constraint/NotEmpty.php) -- [same as](../../../src/core/etl/src/Flow/ETL/Row/Schema/Constraint/SameAs.php) -- [void](../../../src/core/etl/src/Flow/ETL/Row/Schema/Constraint/VoidConstraint.php) - ## Example - schema validation ```php @@ -46,7 +36,7 @@ data_frame() schema( int_schema('id', $nullable = false), str_schema('name', $nullable = true), - bool_schema('active', $nullable = false, new SameAs(true), Metadata::empty()->add('key', 'value')), + bool_schema('active', $nullable = false, Metadata::empty()->add('key', 'value')), ) ) ->write(to_output(false, Output::rows_and_schema)) diff --git a/src/core/etl/src/Flow/ETL/DSL/functions.php b/src/core/etl/src/Flow/ETL/DSL/functions.php index 0f5b6bbec..ea37f6324 100644 --- a/src/core/etl/src/Flow/ETL/DSL/functions.php +++ b/src/core/etl/src/Flow/ETL/DSL/functions.php @@ -376,6 +376,14 @@ function structure_type(array $elements, bool $nullable = false) : StructureType return new StructureType($elements, $nullable); } +/** + * @param array $elements + */ +function type_structure(array $elements, bool $nullable = false) : StructureType +{ + return new StructureType($elements, $nullable); +} + function struct_element(string $name, Type $type) : StructureElement { return new StructureElement($name, $type); @@ -1007,6 +1015,16 @@ function schema(Definition ...$definitions) : Schema return new Schema(...$definitions); } +function schema_to_json(Schema $schema) : string +{ + return \json_encode($schema->normalize(), JSON_THROW_ON_ERROR); +} + +function schema_from_json(string $schema) : Schema +{ + return Schema::fromArray(\json_decode($schema, true, 512, JSON_THROW_ON_ERROR)); +} + function int_schema(string $name, bool $nullable = false, ?Schema\Metadata $metadata = null) : Definition { return Definition::integer($name, $nullable, $metadata); @@ -1085,9 +1103,14 @@ function struct_schema(string $name, StructureType $type, ?Schema\Metadata $meta return Definition::structure($name, $type, $metadata); } +function structure_schema(string $name, StructureType $type, ?Schema\Metadata $metadata = null) : Definition +{ + return Definition::structure($name, $type, $metadata); +} + function uuid_schema(string $name, bool $nullable = false, ?Schema\Metadata $metadata = null) : Definition { - return Definition::uuid($name, \uuid_type($nullable), $metadata); + return Definition::uuid($name, $nullable, $metadata); } function execution_context(?Config $config = null) : FlowContext diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/DateTimeType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/DateTimeType.php index ed4f73e23..585040485 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/DateTimeType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/DateTimeType.php @@ -14,6 +14,11 @@ public function __construct(private readonly bool $nullable = false) { } + public static function fromArray(array $data) : self + { + return new self($data['nullable'] ?? false); + } + public function isEqual(Type $type) : bool { return $type instanceof self; @@ -42,6 +47,14 @@ public function merge(Type $type) : self return new self($this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'datetime', + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/JsonType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/JsonType.php index de3d56ae3..4e8eaa172 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/JsonType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/JsonType.php @@ -15,6 +15,11 @@ public function __construct(private readonly bool $nullable) { } + public static function fromArray(array $data) : self + { + return new self($data['nullable'] ?? false); + } + public function isEqual(Type $type) : bool { return $type instanceof self; @@ -47,6 +52,14 @@ public function merge(Type $type) : self return new self($this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'json', + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/List/ListElement.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/List/ListElement.php index 6d7425516..de95debf8 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/List/ListElement.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/List/ListElement.php @@ -12,14 +12,16 @@ use function Flow\ETL\DSL\type_uuid; use function Flow\ETL\DSL\type_xml; use function Flow\ETL\DSL\type_xml_node; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\PHP\Type\Logical\ListType; use Flow\ETL\PHP\Type\Logical\MapType; use Flow\ETL\PHP\Type\Logical\StructureType; use Flow\ETL\PHP\Type\Type; +use Flow\ETL\PHP\Type\TypeFactory; final class ListElement { - public function __construct(private readonly Type $value) + public function __construct(private readonly Type $type) { } @@ -38,6 +40,15 @@ public static function float() : self return new self(type_float(false)); } + public static function fromArray(array $data) : self + { + if (!\array_key_exists('type', $data)) { + throw new InvalidArgumentException("Missing 'type' key in list element definition"); + } + + return new self(TypeFactory::fromArray($data['type'])); + } + public static function fromType(Type $type) : self { return new self($type); @@ -102,21 +113,28 @@ public static function xml_node(bool $nullable = false) : self public function isEqual(mixed $value) : bool { - return $this->value->isEqual($value); + return $this->type->isEqual($value); } public function isValid(mixed $value) : bool { - return $this->value->isValid($value); + return $this->type->isValid($value); + } + + public function normalize() : array + { + return [ + 'type' => $this->type->normalize(), + ]; } public function toString() : string { - return $this->value->toString(); + return $this->type->toString(); } public function type() : Type { - return $this->value; + return $this->type; } } diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/ListType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/ListType.php index e9e1c6698..84e52f208 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/ListType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/ListType.php @@ -13,6 +13,11 @@ public function __construct(private readonly ListElement $element, private reado { } + public static function fromArray(array $data) : self + { + return new self(ListElement::fromArray($data['element']), $data['nullable'] ?? false); + } + public function element() : ListElement { return $this->element; @@ -64,6 +69,15 @@ public function merge(Type $type) : self return new self($this->element, $this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'list', + 'element' => $this->element->normalize(), + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Map/MapKey.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Map/MapKey.php index 2a054d9d9..a189c6c9a 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Map/MapKey.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Map/MapKey.php @@ -6,8 +6,10 @@ use function Flow\ETL\DSL\type_int; use function Flow\ETL\DSL\type_string; use function Flow\ETL\DSL\type_uuid; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\PHP\Type\Logical\LogicalType; use Flow\ETL\PHP\Type\Native\ScalarType; +use Flow\ETL\PHP\Type\TypeFactory; final class MapKey { @@ -20,6 +22,21 @@ public static function datetime() : self return new self(type_datetime(false)); } + public static function fromArray(array $data) : self + { + if (!\array_key_exists('type', $data)) { + throw new InvalidArgumentException('Missing "type" key in ' . self::class . ' fromArray()'); + } + + $keyType = TypeFactory::fromArray($data['type']); + + if (!$keyType instanceof ScalarType && !$keyType instanceof LogicalType) { + throw new InvalidArgumentException('Invalid "type" key in ' . self::class . ' fromArray()'); + } + + return new self($keyType); + } + public static function fromType(ScalarType|LogicalType $type) : self { return new self($type); @@ -50,6 +67,13 @@ public function isValid(mixed $value) : bool return $this->value->isValid($value); } + public function normalize() : array + { + return [ + 'type' => $this->value->normalize(), + ]; + } + public function toString() : string { return $this->value->toString(); diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Map/MapValue.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Map/MapValue.php index 9ebb76e73..b662c1aa9 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Map/MapValue.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Map/MapValue.php @@ -10,9 +10,11 @@ use function Flow\ETL\DSL\type_string; use function Flow\ETL\DSL\type_xml; use function Flow\ETL\DSL\type_xml_node; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\PHP\Type\Logical\ListType; use Flow\ETL\PHP\Type\Logical\MapType; use Flow\ETL\PHP\Type\Type; +use Flow\ETL\PHP\Type\TypeFactory; final class MapValue { @@ -35,6 +37,15 @@ public static function float() : self return new self(type_float()); } + public static function fromArray(array $value) : self + { + if (!\array_key_exists('type', $value)) { + throw new InvalidArgumentException('Missing "type" key in ' . self::class . ' fromArray()'); + } + + return new self(TypeFactory::fromArray($value['type'])); + } + public static function fromType(Type $type) : self { return new self($type); @@ -93,6 +104,13 @@ public function isValid(mixed $value) : bool return $this->value->isValid($value); } + public function normalize() : array + { + return [ + 'type' => $this->value->normalize(), + ]; + } + public function toString() : string { return $this->value->toString(); diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/MapType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/MapType.php index d9db30b0e..a4e6f4759 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/MapType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/MapType.php @@ -14,6 +14,11 @@ public function __construct(private readonly MapKey $key, private readonly MapVa { } + public static function fromArray(array $data) : self + { + return new self(MapKey::fromArray($data['key']), MapValue::fromArray($data['value']), $data['nullable'] ?? false); + } + public function isEqual(Type $type) : bool { if (!$type instanceof self) { @@ -69,6 +74,16 @@ public function merge(Type $type) : self return new self($this->key, $this->value, $this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'map', + 'key' => $this->key->normalize(), + 'value' => $this->value->normalize(), + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Structure/StructureElement.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Structure/StructureElement.php index c2a2f95d8..1196ef3dc 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Structure/StructureElement.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/Structure/StructureElement.php @@ -5,6 +5,7 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\PHP\Type\Type; +use Flow\ETL\PHP\Type\TypeFactory; final class StructureElement { @@ -15,6 +16,19 @@ public function __construct(private readonly string $name, private readonly Type } } + public static function fromArray(array $element) : self + { + if (!\array_key_exists('name', $element)) { + throw InvalidArgumentException::because('Structure element must have a name'); + } + + if (!\array_key_exists('type', $element)) { + throw InvalidArgumentException::because('Structure element must have a type'); + } + + return new self($element['name'], TypeFactory::fromArray($element['type'])); + } + public function isEqual(self $element) : bool { return $this->name === $element->name && $this->type->isEqual($element->type()); @@ -44,6 +58,14 @@ public function name() : string return $this->name; } + public function normalize() : array + { + return [ + 'name' => $this->name, + 'type' => $this->type->normalize(), + ]; + } + public function toString() : string { return $this->name . ': ' . $this->type->toString(); diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/StructureType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/StructureType.php index 3ccd2e659..c7dc6e189 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/StructureType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/StructureType.php @@ -27,6 +27,21 @@ public function __construct(array $elements, private readonly bool $nullable = f $this->elements = $elements; } + public static function fromArray(array $data) : self + { + if (!\array_key_exists('elements', $data)) { + throw InvalidArgumentException::because('Structure must receive at least one element.'); + } + + $elements = []; + + foreach ($data['elements'] as $element) { + $elements[] = StructureElement::fromArray($element); + } + + return new self($elements, $data['nullable'] ?? false); + } + /** * @return array */ @@ -125,6 +140,15 @@ public function merge(Type $type) : self return new self(\array_values($elements), $this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'structure', + 'elements' => \array_map(fn (StructureElement $element) => $element->normalize(), $this->elements), + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/UuidType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/UuidType.php index 2631b8b11..de53feb75 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/UuidType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/UuidType.php @@ -15,6 +15,11 @@ public function __construct(private readonly bool $nullable = false) { } + public static function fromArray(array $data) : Type + { + return new self($data['nullable'] ?? false); + } + public function isEqual(Type $type) : bool { return $type instanceof self; @@ -51,6 +56,14 @@ public function merge(Type $type) : self return new self($this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'uuid', + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/XMLNodeType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/XMLNodeType.php index 6b2934df8..0110deb51 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/XMLNodeType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/XMLNodeType.php @@ -14,6 +14,11 @@ public function __construct(private readonly bool $nullable) { } + public static function fromArray(array $data) : self + { + return new self($data['nullable'] ?? false); + } + public function isEqual(Type $type) : bool { return $type instanceof self; @@ -46,6 +51,14 @@ public function merge(Type $type) : self return new self($this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'xml_node', + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/XMLType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/XMLType.php index 1c02a69f4..149d03607 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/XMLType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/XMLType.php @@ -14,6 +14,11 @@ public function __construct(private readonly bool $nullable) { } + public static function fromArray(array $data) : self + { + return new self($data['nullable'] ?? false); + } + public function isEqual(Type $type) : bool { return $type instanceof self; @@ -46,6 +51,14 @@ public function merge(Type $type) : self return new self($this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'xml', + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Native/ArrayType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Native/ArrayType.php index b621e427f..011a3d9a2 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Native/ArrayType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Native/ArrayType.php @@ -17,6 +17,11 @@ public static function empty() : self return new self(true); } + public static function fromArray(array $data) : self + { + return new self($data['empty'] ?? false, $data['nullable'] ?? false); + } + public function isEqual(Type $type) : bool { return $type instanceof self && $this->empty === $type->empty; @@ -45,6 +50,15 @@ public function merge(Type $type) : self return new self($this->empty || $type->empty, $this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'array', + 'empty' => $this->empty, + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Native/CallableType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Native/CallableType.php index c4e14e629..808617338 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Native/CallableType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Native/CallableType.php @@ -13,6 +13,11 @@ public function __construct(private readonly bool $nullable) } + public static function fromArray(array $data) : self + { + return new self($data['nullable'] ?? false); + } + public function isEqual(Type $type) : bool { return $type instanceof self && $this->nullable === $type->nullable; @@ -41,6 +46,14 @@ public function merge(Type $type) : self return new self($this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'callable', + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Native/EnumType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Native/EnumType.php index e4d625368..4b7d2236d 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Native/EnumType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Native/EnumType.php @@ -19,6 +19,17 @@ public function __construct(public readonly string $class, private readonly bool } } + public static function fromArray(array $data) : self + { + if (!\array_key_exists('class', $data)) { + throw new InvalidArgumentException("Missing 'class' key in enum type definition"); + } + + $nullable = $data['nullable'] ?? false; + + return new self($data['class'], $nullable); + } + /** * @param class-string<\UnitEnum> $class */ @@ -55,6 +66,15 @@ public function merge(Type $type) : self return new self($this->class, $this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'enum', + 'class' => $this->class, + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Native/NullType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Native/NullType.php index e8871d115..915ac0e3f 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Native/NullType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Native/NullType.php @@ -7,6 +7,11 @@ final class NullType implements NativeType { + public static function fromArray(array $data) : self + { + return new self(); + } + public function isEqual(Type $type) : bool { return $type instanceof self; @@ -28,6 +33,13 @@ public function merge(Type $type) : self return $type->makeNullable(true); } + public function normalize() : array + { + return [ + 'type' => 'null', + ]; + } + public function nullable() : bool { return true; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Native/ObjectType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Native/ObjectType.php index ac6e2769e..44b8a573d 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Native/ObjectType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Native/ObjectType.php @@ -19,6 +19,17 @@ public function __construct(public readonly string $class, private readonly bool } } + public static function fromArray(array $data) : self + { + if (!\array_key_exists('class', $data)) { + throw new InvalidArgumentException("Missing 'class' key in object type definition"); + } + + $nullable = $data['nullable'] ?? false; + + return new self($data['class'], $nullable); + } + public function isEqual(Type $type) : bool { return $type instanceof self && $this->class === $type->class && $this->nullable === $type->nullable; @@ -51,6 +62,15 @@ public function merge(Type $type) : self return new self($this->class, $this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'object', + 'class' => $this->class, + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Native/ResourceType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Native/ResourceType.php index 862cf005a..241d4455c 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Native/ResourceType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Native/ResourceType.php @@ -13,6 +13,11 @@ public function __construct(private readonly bool $nullable) } + public static function fromArray(array $data) : self + { + return new self($data['nullable'] ?? false); + } + public function isEqual(Type $type) : bool { return $type instanceof self && $this->nullable === $type->nullable; @@ -41,6 +46,14 @@ public function merge(Type $type) : self return new self($this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'resource', + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Native/ScalarType.php b/src/core/etl/src/Flow/ETL/PHP/Type/Native/ScalarType.php index 0fdb11aa8..c35d674f5 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Native/ScalarType.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Native/ScalarType.php @@ -34,6 +34,23 @@ public static function float(bool $nullable = false) : self return new self(self::FLOAT, $nullable); } + public static function fromArray(array $data) : self + { + if (!\array_key_exists('scalar_type', $data)) { + throw new InvalidArgumentException("Missing 'scalar_type' key in scalar type definition"); + } + + $nullable = $data['nullable'] ?? false; + + return match ($data['scalar_type']) { + 'boolean' => self::boolean($nullable), + 'float' => self::float($nullable), + 'integer' => self::integer($nullable), + 'string' => self::string($nullable), + default => throw new InvalidArgumentException("Unknown scalar type '{$data['scalar_type']}'"), + }; + } + public static function integer(bool $nullable = false) : self { return new self(self::INTEGER, $nullable); @@ -105,6 +122,15 @@ public function merge(Type $type) : self return new self($this->type, $this->nullable || $type->nullable()); } + public function normalize() : array + { + return [ + 'type' => 'scalar', + 'scalar_type' => $this->type, + 'nullable' => $this->nullable, + ]; + } + public function nullable() : bool { return $this->nullable; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Type.php b/src/core/etl/src/Flow/ETL/PHP/Type/Type.php index a437fdcad..b91b5ed30 100644 --- a/src/core/etl/src/Flow/ETL/PHP/Type/Type.php +++ b/src/core/etl/src/Flow/ETL/PHP/Type/Type.php @@ -6,6 +6,8 @@ interface Type { + public static function fromArray(array $data) : self; + public function isEqual(self $type) : bool; public function isValid(mixed $value) : bool; @@ -14,6 +16,8 @@ public function makeNullable(bool $nullable) : self; public function merge(self $type) : self; + public function normalize() : array; + public function nullable() : bool; public function toString() : string; diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/TypeFactory.php b/src/core/etl/src/Flow/ETL/PHP/Type/TypeFactory.php new file mode 100644 index 000000000..fdfba65b1 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/PHP/Type/TypeFactory.php @@ -0,0 +1,51 @@ + ScalarType::fromArray($data), + 'callable' => CallableType::fromArray($data), + 'array' => ArrayType::fromArray($data), + 'enum' => EnumType::fromArray($data), + 'null' => NullType::fromArray($data), + 'object' => ObjectType::fromArray($data), + 'resource' => ResourceType::fromArray($data), + 'datetime' => DateTimeType::fromArray($data), + 'json' => JsonType::fromArray($data), + 'uuid' => UuidType::fromArray($data), + 'list' => ListType::fromArray($data), + 'map' => MapType::fromArray($data), + 'structure' => StructureType::fromArray($data), + 'xml_node' => XMLNodeType::fromArray($data), + 'xml' => XMLType::fromArray($data), + default => throw new InvalidArgumentException("Unknown type '{$data['type']}'"), + }; + } +} diff --git a/src/core/etl/src/Flow/ETL/Row/Schema.php b/src/core/etl/src/Flow/ETL/Row/Schema.php index ca11f8d99..49488f10d 100644 --- a/src/core/etl/src/Flow/ETL/Row/Schema.php +++ b/src/core/etl/src/Flow/ETL/Row/Schema.php @@ -29,6 +29,21 @@ public function __construct(Definition ...$definitions) $this->definitions = $uniqueDefinitions; } + public static function fromArray(array $definitions) : self + { + $schema = []; + + foreach ($definitions as $definition) { + if (!\is_array($definition)) { + throw new InvalidArgumentException('Schema definition must be an array'); + } + + $schema[] = Definition::fromArray($definition); + } + + return new self(...$schema); + } + public function count() : int { return \count($this->definitions); @@ -110,6 +125,17 @@ public function merge(self $schema) : self return new self(...\array_values($newDefinitions)); } + public function normalize() : array + { + $definitions = []; + + foreach ($this->definitions as $definition) { + $definitions[] = $definition->normalize(); + } + + return $definitions; + } + public function nullable() : self { $definitions = []; diff --git a/src/core/etl/src/Flow/ETL/Row/Schema/Definition.php b/src/core/etl/src/Flow/ETL/Row/Schema/Definition.php index 342ccaa33..d864e26ed 100644 --- a/src/core/etl/src/Flow/ETL/Row/Schema/Definition.php +++ b/src/core/etl/src/Flow/ETL/Row/Schema/Definition.php @@ -24,6 +24,7 @@ use Flow\ETL\PHP\Type\Logical\StructureType; use Flow\ETL\PHP\Type\Native\ObjectType; use Flow\ETL\PHP\Type\Type; +use Flow\ETL\PHP\Type\TypeFactory; use Flow\ETL\Row\Entry; use Flow\ETL\Row\Entry\ArrayEntry; use Flow\ETL\Row\Entry\BooleanEntry; @@ -104,6 +105,49 @@ public static function float(string|Reference $entry, bool $nullable = false, ?M return new self($entry, FloatEntry::class, type_float($nullable), $metadata); } + public static function fromArray(array $definition) : self + { + if (!\array_key_exists('ref', $definition)) { + throw new InvalidArgumentException('Schema definition must contain "ref" key'); + } + + if (!\array_key_exists('type', $definition)) { + throw new InvalidArgumentException('Schema definition must contain "type" key'); + } + + if (!\is_array($definition['type'])) { + throw new InvalidArgumentException('Schema definition "type" must be an array, got: ' . \json_encode($definition['type'])); + } + + return new self( + $definition['ref'], + match ($definition['type']['type']) { + 'array' => ArrayEntry::class, + 'scalar' => match ($definition['type']['scalar_type']) { + 'boolean' => BooleanEntry::class, + 'float' => FloatEntry::class, + 'integer' => IntegerEntry::class, + 'string' => StringEntry::class, + default => throw new InvalidArgumentException(\sprintf('Unknown scalar type "%s"', \json_encode($definition['type']['scalar_type']))), + }, + 'datetime' => DateTimeEntry::class, + 'enum' => EnumEntry::class, + 'json' => JsonEntry::class, + 'list' => ListEntry::class, + 'map' => MapEntry::class, + 'null' => NullEntry::class, + 'object' => ObjectEntry::class, + 'structure' => StructureEntry::class, + 'uuid' => UuidEntry::class, + 'xml' => XMLEntry::class, + 'xml_node' => XMLNodeEntry::class, + default => throw new InvalidArgumentException(\sprintf('Unknown entry type "%s"', \json_encode($definition['type']))), + }, + TypeFactory::fromArray($definition['type']), + Metadata::fromArray($definition['metadata'] ?? []) + ); + } + public static function integer(string|Reference $entry, bool $nullable = false, ?Metadata $metadata = null) : self { return new self($entry, IntegerEntry::class, type_int($nullable), $metadata); @@ -305,6 +349,15 @@ public function metadata() : Metadata return $this->metadata; } + public function normalize() : array + { + return [ + 'ref' => $this->ref->name(), + 'type' => $this->type->normalize(), + 'metadata' => $this->metadata->normalize(), + ]; + } + public function nullable() : self { return new self($this->ref, $this->entryClass, $this->type->makeNullable(true), $this->metadata); diff --git a/src/core/etl/src/Flow/ETL/Row/Schema/Metadata.php b/src/core/etl/src/Flow/ETL/Row/Schema/Metadata.php index b67f3dda9..3605d54ca 100644 --- a/src/core/etl/src/Flow/ETL/Row/Schema/Metadata.php +++ b/src/core/etl/src/Flow/ETL/Row/Schema/Metadata.php @@ -30,9 +30,17 @@ public static function empty() : self return new self([]); } + /** + * @param array|bool|float|int|string> $map + */ + public static function fromArray(array $map) : self + { + return new self($map); + } + /** * @param string $key - * @param array|bool|float|int|object|string $value + * @param array|bool|float|int|string $value * * @return $this */ @@ -43,7 +51,7 @@ public static function with(string $key, int|string|bool|float|array $value) : s /** * @param string $key - * @param array|bool|float|int|object|string $value + * @param array|bool|float|int|string $value * * @return $this */ @@ -78,6 +86,14 @@ public function merge(self $metadata) : self return new self(\array_merge($this->map, $metadata->map)); } + /** + * @return array|bool|float|int|string> + */ + public function normalize() : array + { + return $this->map; + } + public function remove(string $key) : self { $map = []; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/PHP/Type/Fixtures/SomeEnum.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/PHP/Type/Fixtures/SomeEnum.php new file mode 100644 index 000000000..c09c55bc3 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/PHP/Type/Fixtures/SomeEnum.php @@ -0,0 +1,12 @@ +assertEquals($string, TypeFactory::fromArray($string->normalize())); + $integer = type_integer(); + $this->assertEquals($integer, TypeFactory::fromArray($integer->normalize())); + $boolean = type_boolean(); + $this->assertEquals($boolean, TypeFactory::fromArray($boolean->normalize())); + $float = type_float(); + $this->assertEquals($float, TypeFactory::fromArray($float->normalize())); + } + + public function test_normalizing_and_creating_array_type() : void + { + $array = type_array(); + $this->assertEquals($array, TypeFactory::fromArray($array->normalize())); + } + + public function test_normalizing_and_creating_callable_type() : void + { + $callable = type_callable(); + $this->assertEquals($callable, TypeFactory::fromArray($callable->normalize())); + } + + public function test_normalizing_and_creating_datetime_type() : void + { + $datetime = type_datetime(); + $this->assertEquals($datetime, TypeFactory::fromArray($datetime->normalize())); + } + + public function test_normalizing_and_creating_enum_type() : void + { + $enum = type_enum(SomeEnum::class); + $this->assertEquals($enum, TypeFactory::fromArray($enum->normalize())); + } + + public function test_normalizing_and_creating_json_type() : void + { + $json = type_json(); + $this->assertEquals($json, TypeFactory::fromArray($json->normalize())); + } + + public function test_normalizing_and_creating_list_type() : void + { + $list = type_list(type_string()); + $this->assertEquals($list, TypeFactory::fromArray($list->normalize())); + } + + public function test_normalizing_and_creating_map_type() : void + { + $map = type_map(type_string(), type_integer()); + $this->assertEquals($map, TypeFactory::fromArray($map->normalize())); + } + + public function test_normalizing_and_creating_null_type() : void + { + $null = type_null(); + $this->assertEquals($null, TypeFactory::fromArray($null->normalize())); + } + + public function test_normalizing_and_creating_object_type() : void + { + $object = type_object(\stdClass::class); + $this->assertEquals($object, TypeFactory::fromArray($object->normalize())); + } + + public function test_normalizing_and_creating_resource_type() : void + { + $resource = type_resource(); + $this->assertEquals($resource, TypeFactory::fromArray($resource->normalize())); + } + + public function test_normalizing_and_creating_structure_type() : void + { + $structure = type_structure( + [ + structure_element('name', type_string()), + structure_element('age', type_integer()), + structure_element('list', type_list(type_string())), + structure_element('map', type_map(type_string(), type_integer())), + structure_element('object', type_object(\stdClass::class)), + ] + ); + + $this->assertEquals($structure, TypeFactory::fromArray($structure->normalize())); + } + + public function test_normalizing_and_creating_uuid_type() : void + { + $uuid = type_uuid(); + $this->assertEquals($uuid, TypeFactory::fromArray($uuid->normalize())); + } + + public function test_normalizing_and_creating_xml_node_type() : void + { + $xmlNode = type_xml_node(); + $this->assertEquals($xmlNode, TypeFactory::fromArray($xmlNode->normalize())); + } + + public function test_normalizing_and_creating_xml_type() : void + { + $xml = type_xml(); + $this->assertEquals($xml, TypeFactory::fromArray($xml->normalize())); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/Schema/DefinitionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/Schema/DefinitionTest.php index c6dbe6f6b..3aa4fc7c9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/Schema/DefinitionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/Schema/DefinitionTest.php @@ -9,6 +9,7 @@ use function Flow\ETL\DSL\str_entry; use function Flow\ETL\DSL\struct_element; use function Flow\ETL\DSL\struct_entry; +use function Flow\ETL\DSL\struct_schema; use function Flow\ETL\DSL\struct_type; use function Flow\ETL\DSL\type_datetime; use function Flow\ETL\DSL\type_float; @@ -16,12 +17,14 @@ use function Flow\ETL\DSL\type_list; use function Flow\ETL\DSL\type_map; use function Flow\ETL\DSL\type_string; +use function Flow\ETL\DSL\type_structure; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\PHP\Type\Logical\List\ListElement; use Flow\ETL\PHP\Type\Logical\ListType; use Flow\ETL\PHP\Type\Logical\StructureType; use Flow\ETL\Row\Schema\Definition; +use Flow\ETL\Row\Schema\Metadata; use PHPUnit\Framework\TestCase; final class DefinitionTest extends TestCase @@ -189,6 +192,31 @@ public function test_merging_two_same_maps() : void ); } + public function test_normalize_and_from_array() : void + { + $definition = struct_schema( + 'structure', + type_structure( + [ + struct_element('street', type_string()), + struct_element('city', type_string()), + struct_element('location', type_structure( + [ + struct_element('lat', type_float()), + struct_element('lng', type_float()), + ] + )), + ] + ), + Metadata::with('description', 'some_random_description')->add('priority', 1) + ); + + $this->assertEquals( + $definition, + Definition::fromArray($definition->normalize()) + ); + } + public function test_not_matches_when_not_nullable_name_matches_but_null_given() : void { $def = Definition::integer('test', $nullable = false); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/SchemaTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/SchemaTest.php index cfc231ed6..2faee6f96 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/SchemaTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/SchemaTest.php @@ -4,6 +4,22 @@ namespace Flow\ETL\Tests\Unit\Row; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\json_schema; +use function Flow\ETL\DSL\list_schema; +use function Flow\ETL\DSL\map_schema; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\schema_from_json; +use function Flow\ETL\DSL\schema_to_json; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\struct_element; +use function Flow\ETL\DSL\structure_schema; +use function Flow\ETL\DSL\type_int; +use function Flow\ETL\DSL\type_list; +use function Flow\ETL\DSL\type_map; +use function Flow\ETL\DSL\type_string; +use function Flow\ETL\DSL\type_structure; +use function Flow\ETL\DSL\uuid_schema; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Row\EntryReference; use Flow\ETL\Row\Schema; @@ -31,6 +47,30 @@ public function test_allowing_only_unique_definitions_case_insensitive() : void $this->assertEquals([EntryReference::init('id'), EntryReference::init('Id')], $schema->entries()); } + public function test_creating_schema_from_corrupted_json() : void + { + $this->expectException(\JsonException::class); + $this->expectExceptionMessage('Syntax error'); + + schema_from_json('{"ref": "id", "type": {"type": "scalar", "scalar_type": "integer", "nullable": false}, "metadata": []'); + } + + public function test_creating_schema_from_invalid_json_format() : void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Schema definition must be an array'); + + schema_from_json('{"ref": "id", "type": {"type": "scalar", "scalar_type": "integer", "nullable": false}, "metadata": []}'); + } + + public function test_creating_schema_from_invalid_json_format_at_definition_level() : void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Schema definition "type" must be an array, got: "test"'); + + schema_from_json('[{"ref": "id", "type": "test", "metadata": []}]'); + } + public function test_making_whole_schema_nullable() : void { $schema = new Schema( @@ -98,6 +138,27 @@ public function test_merge_with_empty_schema() : void ); } + public function test_normalizing_and_recreating_schema() : void + { + $schema = schema( + int_schema('id'), + str_schema('str', true), + uuid_schema('uuid'), + json_schema('json', true), + map_schema('map', type_map(type_string(), type_int())), + list_schema('list', type_list(type_int())), + structure_schema('struct', type_structure([ + struct_element('street', type_string()), + struct_element('city', type_string()), + ])) + ); + + $this->assertEquals( + $schema, + Schema::fromArray($schema->normalize()) + ); + } + public function test_removing_elements_from_schema() : void { $schema = new Schema( @@ -112,4 +173,130 @@ public function test_removing_elements_from_schema() : void $schema->without('name', 'tags') ); } + + public function test_schema_to_from_json() : void + { + $schema = schema( + int_schema('id'), + str_schema('str', true), + uuid_schema('uuid'), + json_schema('json', true), + map_schema('map', type_map(type_string(), type_int())), + list_schema('list', type_list(type_int())), + structure_schema('struct', type_structure([ + struct_element('street', type_string()), + struct_element('city', type_string()), + ])) + ); + + $this->assertSame( + <<<'JSON' +[ + { + "ref": "id", + "type": { + "type": "scalar", + "scalar_type": "integer", + "nullable": false + }, + "metadata": [] + }, + { + "ref": "str", + "type": { + "type": "scalar", + "scalar_type": "string", + "nullable": true + }, + "metadata": [] + }, + { + "ref": "uuid", + "type": { + "type": "uuid", + "nullable": false + }, + "metadata": [] + }, + { + "ref": "json", + "type": { + "type": "json", + "nullable": true + }, + "metadata": [] + }, + { + "ref": "map", + "type": { + "type": "map", + "key": { + "type": { + "type": "scalar", + "scalar_type": "string", + "nullable": false + } + }, + "value": { + "type": { + "type": "scalar", + "scalar_type": "integer", + "nullable": false + } + }, + "nullable": false + }, + "metadata": [] + }, + { + "ref": "list", + "type": { + "type": "list", + "element": { + "type": { + "type": "scalar", + "scalar_type": "integer", + "nullable": false + } + }, + "nullable": false + }, + "metadata": [] + }, + { + "ref": "struct", + "type": { + "type": "structure", + "elements": [ + { + "name": "street", + "type": { + "type": "scalar", + "scalar_type": "string", + "nullable": false + } + }, + { + "name": "city", + "type": { + "type": "scalar", + "scalar_type": "string", + "nullable": false + } + } + ], + "nullable": false + }, + "metadata": [] + } +] +JSON, + \json_encode($schema->normalize(), JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT) + ); + + $this->assertEquals( + $schema, + schema_from_json(schema_to_json($schema)) + ); + } }