From af68770e94ce7094b67eaf9fe3d338992546f6da Mon Sep 17 00:00:00 2001 From: kevin-m-knight-gs Date: Thu, 19 Dec 2024 11:46:42 -0500 Subject: [PATCH] Add concrete element serialization --- .../m3/navigation/PrimitiveUtilities.java | 24 + .../pure/m3/navigation/property/Property.java | 13 +- .../element/ConcreteElementSerializer.java | 167 ++++ .../ConcreteElementSerializerExtension.java | 27 + .../element/DeserializedConcreteElement.java | 24 + .../compiler/element/DeserializedElement.java | 28 + .../compiler/element/PropertyValues.java | 26 + .../compiler/element/Reference.java | 112 +++ .../element/SerializationContext.java | 48 ++ .../serialization/compiler/element/Value.java | 384 +++++++++ .../compiler/element/ValueOrReference.java | 24 + .../element/ValueOrReferenceConsumer.java | 182 ++++ .../element/ValueOrReferenceVisitor.java | 83 ++ .../compiler/element/v1/BaseV1.java | 804 ++++++++++++++++++ .../v1/ConcreteElementSerializerV1.java | 43 + .../compiler/element/v1/DeserializerV1.java | 354 ++++++++ .../compiler/element/v1/SerializerV1.java | 347 ++++++++ ...element.ConcreteElementSerializerExtension | 1 + ...estConcreteElementSerializerExtension.java | 267 ++++++ .../v1/TestConcreteElementSerializerV1.java | 27 + .../v1/TestPrimitiveSerialization.java | 257 ++++++ 21 files changed, 3239 insertions(+), 3 deletions(-) create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ConcreteElementSerializer.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ConcreteElementSerializerExtension.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/DeserializedConcreteElement.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/DeserializedElement.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/PropertyValues.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/Reference.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/SerializationContext.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/Value.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReference.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReferenceConsumer.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReferenceVisitor.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/BaseV1.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/ConcreteElementSerializerV1.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/DeserializerV1.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/SerializerV1.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.compiler.element.ConcreteElementSerializerExtension create mode 100644 legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/AbstractTestConcreteElementSerializerExtension.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/TestConcreteElementSerializerV1.java create mode 100644 legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/TestPrimitiveSerialization.java diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/navigation/PrimitiveUtilities.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/navigation/PrimitiveUtilities.java index b1e4b6975d..d924485c27 100644 --- a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/navigation/PrimitiveUtilities.java +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/navigation/PrimitiveUtilities.java @@ -20,12 +20,16 @@ import org.finos.legend.pure.m4.ModelRepository; import org.finos.legend.pure.m4.coreinstance.CoreInstance; import org.finos.legend.pure.m4.coreinstance.primitive.BooleanCoreInstance; +import org.finos.legend.pure.m4.coreinstance.primitive.ByteCoreInstance; import org.finos.legend.pure.m4.coreinstance.primitive.DateCoreInstance; import org.finos.legend.pure.m4.coreinstance.primitive.DecimalCoreInstance; import org.finos.legend.pure.m4.coreinstance.primitive.FloatCoreInstance; import org.finos.legend.pure.m4.coreinstance.primitive.IntegerCoreInstance; +import org.finos.legend.pure.m4.coreinstance.primitive.StrictTimeCoreInstance; import org.finos.legend.pure.m4.coreinstance.primitive.date.DateFunctions; import org.finos.legend.pure.m4.coreinstance.primitive.date.PureDate; +import org.finos.legend.pure.m4.coreinstance.primitive.strictTime.PureStrictTime; +import org.finos.legend.pure.m4.coreinstance.primitive.strictTime.StrictTimeFunctions; import java.math.BigDecimal; import java.math.BigInteger; @@ -52,6 +56,16 @@ public static boolean getBooleanValue(CoreInstance instance, boolean defaultIfNu return (instance == null) ? defaultIfNull : getBooleanValue(instance); } + public static byte getByteValue(CoreInstance instance) + { + return (instance instanceof ByteCoreInstance) ? ((ByteCoreInstance) instance).getValue() : Byte.parseByte(instance.getName()); + } + + public static byte getByteValue(CoreInstance instance, byte defaultIfNull) + { + return (instance == null) ? defaultIfNull : getByteValue(instance); + } + public static PureDate getDateValue(CoreInstance instance) { return (instance instanceof DateCoreInstance) ? ((DateCoreInstance) instance).getValue() : DateFunctions.parsePureDate(instance.getName()); @@ -122,6 +136,16 @@ public static Number getIntegerValue(CoreInstance instance, BigInteger defaultIf return (instance == null) ? defaultIfNull : getIntegerValue(instance); } + public static PureStrictTime getStrictTimeValue(CoreInstance instance) + { + return (instance instanceof StrictTimeCoreInstance) ? ((StrictTimeCoreInstance) instance).getValue() : StrictTimeFunctions.parsePureStrictTime(instance.getName()); + } + + public static PureStrictTime getStrictTimeValue(CoreInstance instance, PureStrictTime defaultIfNull) + { + return (instance == null) ? defaultIfNull : getStrictTimeValue(instance); + } + public static String getStringValue(CoreInstance instance) { return instance.getName(); diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/navigation/property/Property.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/navigation/property/Property.java index e41c33c963..36479cd6eb 100644 --- a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/navigation/property/Property.java +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/navigation/property/Property.java @@ -114,12 +114,19 @@ public static String getPropertyName(CoreInstance property) return PrimitiveUtilities.getStringValue(property.getValueForMetaPropertyToOne(M3Properties.name)); } + public static CoreInstance getPropertySourceType(CoreInstance property, ProcessorSupport processorSupport) + { + CoreInstance classifierGenericType = property.getValueForMetaPropertyToOne(M3Properties.classifierGenericType); + ListIterable typeArguments = classifierGenericType.getValueForMetaPropertyToMany(M3Properties.typeArguments); + return Instance.getValueForMetaPropertyToOneResolved(typeArguments.get(0), M3Properties.rawType, processorSupport); + } + public static ListIterable calculatePropertyPath(CoreInstance property, ProcessorSupport processorSupport) { // Example: [Root, children, core, children, Any, properties, classifierGenericType] String propertyName = getPropertyName(property); - CoreInstance sourceType = Instance.getValueForMetaPropertyToOneResolved(Instance.getValueForMetaPropertyToManyResolved(Instance.getValueForMetaPropertyToOneResolved(property, M3Properties.classifierGenericType, processorSupport), M3Properties.typeArguments, processorSupport).get(0), M3Properties.rawType, processorSupport); + CoreInstance sourceType = getPropertySourceType(property, processorSupport); String propertiesProperty = getPropertiesProperty(sourceType, property); if (propertiesProperty == null) @@ -153,8 +160,8 @@ public static CoreInstance resolvePropertyReturnMultiplicity(CoreInstance source return multiplicity; } - CoreInstance propertyOwnerRawType = Instance.getValueForMetaPropertyToOneResolved(Instance.getValueForMetaPropertyToManyResolved(Instance.getValueForMetaPropertyToOneResolved(property, M3Properties.classifierGenericType, processorSupport), M3Properties.typeArguments, processorSupport).get(0), M3Properties.rawType, processorSupport); - GenericTypeWithXArguments p = GenericType.resolveClassMultiplicityParameterUsingInheritance(sourceGenericType, propertyOwnerRawType, processorSupport); + CoreInstance propertySourceType = getPropertySourceType(property, processorSupport); + GenericTypeWithXArguments p = GenericType.resolveClassMultiplicityParameterUsingInheritance(sourceGenericType, propertySourceType, processorSupport); return p.getArgumentsByParameterName().get(Multiplicity.getMultiplicityParameter(multiplicity)); } diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ConcreteElementSerializer.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ConcreteElementSerializer.java new file mode 100644 index 0000000000..6827e417d1 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ConcreteElementSerializer.java @@ -0,0 +1,167 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +import org.finos.legend.pure.m3.navigation.ProcessorSupport; +import org.finos.legend.pure.m3.serialization.compiler.ExtensibleSerializer; +import org.finos.legend.pure.m3.serialization.compiler.reference.ReferenceIdProvider; +import org.finos.legend.pure.m3.serialization.compiler.reference.ReferenceIds; +import org.finos.legend.pure.m3.serialization.compiler.strings.StringIndexer; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.m4.serialization.Reader; +import org.finos.legend.pure.m4.serialization.Writer; + +import java.util.Arrays; +import java.util.Objects; + +public class ConcreteElementSerializer extends ExtensibleSerializer +{ + private static final long LEGEND_ENTITY_SIGNATURE = Long.parseLong("LegendEntity", 36); + + private final ReferenceIds referenceIds; + private final StringIndexer stringIndexer; + private final ProcessorSupport processorSupport; + + private ConcreteElementSerializer(Iterable extensions, int defaultVersion, StringIndexer stringIndexer, ReferenceIds referenceIds, ProcessorSupport processorSupport) + { + super(extensions, defaultVersion); + this.stringIndexer = stringIndexer; + this.referenceIds = referenceIds; + this.processorSupport = processorSupport; + } + + public void serialize(Writer writer, CoreInstance element) + { + serialize(writer, element, getDefaultExtension(), this.referenceIds.provider()); + } + + public void serialize(Writer writer, CoreInstance element, int serializerVersion, int referenceIdVersion) + { + ConcreteElementSerializerExtension serializerExtension = getExtension(serializerVersion); + ReferenceIdProvider referenceIdProvider = this.referenceIds.provider(referenceIdVersion); + serialize(writer, element, serializerExtension, referenceIdProvider); + } + + private void serialize(Writer writer, CoreInstance element, ConcreteElementSerializerExtension serializerExtension, ReferenceIdProvider referenceIdProvider) + { + writer.writeLong(LEGEND_ENTITY_SIGNATURE); + writer.writeInt(serializerExtension.version()); + writer.writeInt(referenceIdProvider.version()); + serializerExtension.serialize(writer, element, new SerializationContext(this.stringIndexer, referenceIdProvider, this.processorSupport)); + } + + public DeserializedConcreteElement deserialize(Reader reader) + { + long signature = reader.readLong(); + if (signature != LEGEND_ENTITY_SIGNATURE) + { + throw new IllegalArgumentException("Invalid file format: not a Legend concrete element file"); + } + int version = reader.readInt(); + ConcreteElementSerializerExtension extension = getExtension(version); + int referenceIdVersion = reader.readInt(); + return extension.deserialize(reader, new SerializationContext(this.stringIndexer, this.referenceIds.provider(referenceIdVersion), this.processorSupport)); + } + + public static Builder builder(ProcessorSupport processorSupport) + { + return new Builder(processorSupport); + } + + public static class Builder extends AbstractBuilder + { + private StringIndexer stringIndexer; + private ReferenceIds referenceIds; + private final ProcessorSupport processorSupport; + + private Builder(ProcessorSupport processorSupport) + { + this.processorSupport = Objects.requireNonNull(processorSupport); + } + + public Builder withExtension(ConcreteElementSerializerExtension extension) + { + addExtension(extension); + return this; + } + + public Builder withExtensions(Iterable extensions) + { + addExtensions(extensions); + return this; + } + + public Builder withExtensions(ConcreteElementSerializerExtension... extensions) + { + return withExtensions(Arrays.asList(extensions)); + } + + public Builder withLoadedExtensions(ClassLoader classLoader) + { + loadExtensions(classLoader); + return this; + } + + public Builder withLoadedExtensions() + { + loadExtensions(); + return this; + } + + public Builder withDefaultVersion(int defaultVersion) + { + setDefaultVersion(defaultVersion); + return this; + } + + public Builder withStringIndexer(StringIndexer stringIndexer) + { + this.stringIndexer = stringIndexer; + return this; + } + + public Builder withReferenceIds(ReferenceIds referenceIds) + { + this.referenceIds = referenceIds; + return this; + } + + @Override + protected ConcreteElementSerializer build(Iterable extensions, int defaultVersion) + { + return new ConcreteElementSerializer(extensions, defaultVersion, resolveStringIndexer(), resolveReferenceIds(), this.processorSupport); + } + + private ReferenceIds resolveReferenceIds() + { + // If reference ids has not been specified, create one + return (this.referenceIds == null) ? + ReferenceIds.builder(this.processorSupport).withAvailableExtensions().build() : + this.referenceIds; + } + + private StringIndexer resolveStringIndexer() + { + // if string indexer has not been specified, use the default + return (this.stringIndexer == null) ? StringIndexer.defaultStringIndexer() : this.stringIndexer; + } + + @Override + protected Class getExtensionClass() + { + return ConcreteElementSerializerExtension.class; + } + } +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ConcreteElementSerializerExtension.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ConcreteElementSerializerExtension.java new file mode 100644 index 0000000000..beb862421e --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ConcreteElementSerializerExtension.java @@ -0,0 +1,27 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +import org.finos.legend.pure.m3.serialization.compiler.SerializerExtension; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.m4.serialization.Reader; +import org.finos.legend.pure.m4.serialization.Writer; + +public interface ConcreteElementSerializerExtension extends SerializerExtension +{ + void serialize(Writer writer, CoreInstance element, SerializationContext serializationContext); + + DeserializedConcreteElement deserialize(Reader reader, SerializationContext serializationContext); +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/DeserializedConcreteElement.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/DeserializedConcreteElement.java new file mode 100644 index 0000000000..87f341b6f8 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/DeserializedConcreteElement.java @@ -0,0 +1,24 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +public interface DeserializedConcreteElement extends DeserializedElement +{ + String getPath(); + + DeserializedElement getInternalElement(int id); + + int getReferenceIdVersion(); +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/DeserializedElement.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/DeserializedElement.java new file mode 100644 index 0000000000..eb96a57c26 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/DeserializedElement.java @@ -0,0 +1,28 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +import org.finos.legend.pure.m4.coreinstance.SourceInformation; + +public interface DeserializedElement +{ + String getName(); + + String getClassifierReferenceId(); + + SourceInformation getSourceInformation(); + + Iterable getPropertyValues(); +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/PropertyValues.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/PropertyValues.java new file mode 100644 index 0000000000..4dacbb4e71 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/PropertyValues.java @@ -0,0 +1,26 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +import org.eclipse.collections.api.list.ListIterable; + +public interface PropertyValues +{ + String getPropertyName(); + + ListIterable getRealKey(); + + ListIterable getValues(); +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/Reference.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/Reference.java new file mode 100644 index 0000000000..c66a1bc0cc --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/Reference.java @@ -0,0 +1,112 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +import java.util.Objects; + +public abstract class Reference extends ValueOrReference +{ + private Reference() + { + } + + public static ExternalReference newExternalReference(String id) + { + return new ExternalReference(id); + } + + public static InternalReference newInternalReference(int id) + { + return new InternalReference(id); + } + + public static class ExternalReference extends Reference + { + private final String id; + + private ExternalReference(String id) + { + this.id = Objects.requireNonNull(id); + } + + @Override + public boolean equals(Object other) + { + return (this == other) || ((other instanceof ExternalReference) && this.id.equals(((ExternalReference) other).id)); + } + + @Override + public int hashCode() + { + return this.id.hashCode(); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{id=" + this.id + "}"; + } + + public String getId() + { + return this.id; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } + + public static class InternalReference extends Reference + { + private final int id; + + private InternalReference(int id) + { + this.id = id; + } + + @Override + public boolean equals(Object other) + { + return (this == other) || ((other instanceof InternalReference) && (this.id == ((InternalReference) other).id)); + } + + @Override + public int hashCode() + { + return this.id; + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{id=" + this.id + "}"; + } + + public int getId() + { + return this.id; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/SerializationContext.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/SerializationContext.java new file mode 100644 index 0000000000..822cbb453b --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/SerializationContext.java @@ -0,0 +1,48 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +import org.finos.legend.pure.m3.navigation.ProcessorSupport; +import org.finos.legend.pure.m3.serialization.compiler.reference.ReferenceIdProvider; +import org.finos.legend.pure.m3.serialization.compiler.strings.StringIndexer; + +public class SerializationContext +{ + private final StringIndexer stringIndexer; + private final ReferenceIdProvider referenceIdProvider; + private final ProcessorSupport processorSupport; + + SerializationContext(StringIndexer stringIndexer, ReferenceIdProvider referenceIdProvider, ProcessorSupport processorSupport) + { + this.stringIndexer = stringIndexer; + this.referenceIdProvider = referenceIdProvider; + this.processorSupport = processorSupport; + } + + public StringIndexer getStringIndexer() + { + return this.stringIndexer; + } + + public ReferenceIdProvider getReferenceIdProvider() + { + return this.referenceIdProvider; + } + + public ProcessorSupport getProcessorSupport() + { + return this.processorSupport; + } +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/Value.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/Value.java new file mode 100644 index 0000000000..d9750f61c0 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/Value.java @@ -0,0 +1,384 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +import org.finos.legend.pure.m3.navigation.M3Paths; +import org.finos.legend.pure.m4.coreinstance.primitive.date.PureDate; +import org.finos.legend.pure.m4.coreinstance.primitive.strictTime.PureStrictTime; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Objects; + +public abstract class Value extends ValueOrReference +{ + private final T value; + + private Value(T value) + { + this.value = Objects.requireNonNull(value, "value may not be null"); + } + + @Override + public boolean equals(Object other) + { + if (this == other) + { + return true; + } + if (getClass() != other.getClass()) + { + return false; + } + Value that = (Value) other; + return this.getClassifierPath().equals(that.getClassifierPath()) && + this.value.equals(that.value); + } + + @Override + public int hashCode() + { + return getClassifierPath().hashCode() + 31 * this.value.hashCode(); + } + + @Override + public String toString() + { + return "Value{classifier=" + getClassifierPath() + " value=" + getValue() + "}"; + } + + public abstract String getClassifierPath(); + + public T getValue() + { + return this.value; + } + + public static BooleanValue newBooleanValue(boolean value) + { + return value ? BooleanValue.TRUE : BooleanValue.FALSE; + } + + public static ByteValue newByteValue(byte b) + { + return new ByteValue(b); + } + + public static DateValue newDateValue(PureDate value) + { + return new DateValue(value); + } + + public static DateTimeValue newDateTimeValue(PureDate value) + { + return new DateTimeValue(value); + } + + public static StrictDateValue newStrictDateValue(PureDate value) + { + return new StrictDateValue(value); + } + + public static LatestDateValue newLatestDateValue() + { + return LatestDateValue.INSTANCE; + } + + public static DecimalValue newDecimalValue(String value) + { + int lastIndex = value.length() - 1; + char lastChar = value.charAt(lastIndex); + boolean endsWithD = (lastChar == 'd') || (lastChar == 'D'); + return newDecimalValue(new BigDecimal(endsWithD ? value.substring(0, lastIndex) : value)); + } + + public static DecimalValue newDecimalValue(BigDecimal value) + { + return new DecimalValue(value); + } + + public static FloatValue newFloatValue(String value) + { + int lastIndex = value.length() - 1; + char lastChar = value.charAt(lastIndex); + boolean endsWithF = (lastChar == 'f') || (lastChar == 'F'); + return newFloatValue(new BigDecimal(endsWithF ? value.substring(0, lastIndex) : value)); + } + + public static FloatValue newFloatValue(float value) + { + return newFloatValue(BigDecimal.valueOf(value)); + } + + public static FloatValue newFloatValue(double value) + { + return newFloatValue(BigDecimal.valueOf(value)); + } + + public static FloatValue newFloatValue(BigDecimal value) + { + return new FloatValue(value); + } + + public static IntegerValue newIntegerValue(int value) + { + return new IntegerValue(value); + } + + public static IntegerValue newIntegerValue(long value) + { + return new IntegerValue(value); + } + + public static IntegerValue newIntegerValue(BigInteger value) + { + return new IntegerValue(value); + } + + public static StrictTimeValue newStrictTimeValue(PureStrictTime value) + { + return new StrictTimeValue(value); + } + + public static StringValue newStringValue(String value) + { + return new StringValue(value); + } + + public static class BooleanValue extends Value + { + private static final BooleanValue TRUE = new BooleanValue(true); + private static final BooleanValue FALSE = new BooleanValue(false); + + private BooleanValue(boolean value) + { + super(value); + } + + @Override + public String getClassifierPath() + { + return M3Paths.Boolean; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } + + public static class ByteValue extends Value + { + private ByteValue(Byte value) + { + super(value); + } + + @Override + public String getClassifierPath() + { + return M3Paths.Byte; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } + + public static class DateValue extends Value + { + private DateValue(PureDate value) + { + super(value); + } + + @Override + public String getClassifierPath() + { + return M3Paths.Date; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } + + public static class DateTimeValue extends Value + { + private DateTimeValue(PureDate value) + { + super(value); + } + + @Override + public String getClassifierPath() + { + return M3Paths.DateTime; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } + + public static class StrictDateValue extends Value + { + private StrictDateValue(PureDate value) + { + super(value); + } + + @Override + public String getClassifierPath() + { + return M3Paths.StrictDate; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } + + public static class LatestDateValue extends Value + { + private static final LatestDateValue INSTANCE = new LatestDateValue(); + + private LatestDateValue() + { + super(null); + } + + @Override + public String getClassifierPath() + { + return M3Paths.LatestDate; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } + + public static class DecimalValue extends Value + { + private DecimalValue(BigDecimal value) + { + super(value); + } + + @Override + public String getClassifierPath() + { + return M3Paths.Decimal; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } + + public static class FloatValue extends Value + { + private FloatValue(BigDecimal value) + { + super(value); + } + + @Override + public String getClassifierPath() + { + return M3Paths.Float; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } + + public static class IntegerValue extends Value + { + private IntegerValue(Number value) + { + super(value); + } + + @Override + public String getClassifierPath() + { + return M3Paths.Integer; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } + + public static class StrictTimeValue extends Value + { + private StrictTimeValue(PureStrictTime value) + { + super(value); + } + + @Override + public String getClassifierPath() + { + return M3Paths.StrictTime; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } + + public static class StringValue extends Value + { + private StringValue(String value) + { + super(value); + } + + @Override + public String getClassifierPath() + { + return M3Paths.String; + } + + @Override + public V visit(ValueOrReferenceVisitor visitor) + { + return visitor.visit(this); + } + } +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReference.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReference.java new file mode 100644 index 0000000000..afea0bb7aa --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReference.java @@ -0,0 +1,24 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +public abstract class ValueOrReference +{ + ValueOrReference() + { + } + + public abstract V visit(ValueOrReferenceVisitor visitor); +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReferenceConsumer.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReferenceConsumer.java new file mode 100644 index 0000000000..3bbf7a4912 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReferenceConsumer.java @@ -0,0 +1,182 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +import java.util.function.Consumer; + +public abstract class ValueOrReferenceConsumer implements Consumer, ValueOrReferenceVisitor +{ + @Override + public Void visit(Reference.ExternalReference reference) + { + accept(reference); + return null; + } + + @Override + public Void visit(Reference.InternalReference reference) + { + accept(reference); + return null; + } + + @Override + public Void visit(Value.BooleanValue value) + { + accept(value); + return null; + } + + @Override + public Void visit(Value.ByteValue value) + { + accept(value); + return null; + } + + @Override + public Void visit(Value.DateValue value) + { + accept(value); + return null; + } + + @Override + public Void visit(Value.DateTimeValue value) + { + accept(value); + return null; + } + + @Override + public Void visit(Value.StrictDateValue value) + { + accept(value); + return null; + } + + @Override + public Void visit(Value.LatestDateValue value) + { + accept(value); + return null; + } + + @Override + public Void visit(Value.DecimalValue value) + { + accept(value); + return null; + } + + @Override + public Void visit(Value.FloatValue value) + { + accept(value); + return null; + } + + @Override + public Void visit(Value.IntegerValue value) + { + accept(value); + return null; + } + + @Override + public Void visit(Value.StrictTimeValue value) + { + accept(value); + return null; + } + + @Override + public Void visit(Value.StringValue value) + { + accept(value); + return null; + } + + @Override + public void accept(ValueOrReference valueOrReference) + { + valueOrReference.visit(this); + } + + protected void accept(Reference.ExternalReference reference) + { + // do nothing by default + } + + protected void accept(Reference.InternalReference reference) + { + // do nothing by default + } + + protected void accept(Value.BooleanValue value) + { + // do nothing by default + } + + protected void accept(Value.ByteValue value) + { + // do nothing by default + } + + protected void accept(Value.DateValue value) + { + // do nothing by default + } + + protected void accept(Value.DateTimeValue value) + { + // do nothing by default + } + + protected void accept(Value.StrictDateValue value) + { + // do nothing by default + } + + protected void accept(Value.LatestDateValue value) + { + // do nothing by default + } + + protected void accept(Value.DecimalValue value) + { + // do nothing by default + } + + protected void accept(Value.FloatValue value) + { + // do nothing by default + } + + protected void accept(Value.IntegerValue value) + { + // do nothing by default + } + + protected void accept(Value.StrictTimeValue value) + { + // do nothing by default + } + + protected void accept(Value.StringValue value) + { + // do nothing by default + } +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReferenceVisitor.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReferenceVisitor.java new file mode 100644 index 0000000000..59e7f78428 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/ValueOrReferenceVisitor.java @@ -0,0 +1,83 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +public interface ValueOrReferenceVisitor +{ + default T visit(Reference.ExternalReference reference) + { + throw new UnsupportedOperationException(); + } + + default T visit(Reference.InternalReference reference) + { + throw new UnsupportedOperationException(); + } + + default T visit(Value.BooleanValue value) + { + throw new UnsupportedOperationException(); + } + + default T visit(Value.ByteValue value) + { + throw new UnsupportedOperationException(); + } + + default T visit(Value.DateValue value) + { + throw new UnsupportedOperationException(); + } + + default T visit(Value.DateTimeValue value) + { + throw new UnsupportedOperationException(); + } + + default T visit(Value.StrictDateValue value) + { + throw new UnsupportedOperationException(); + } + + default T visit(Value.LatestDateValue value) + { + throw new UnsupportedOperationException(); + } + + default T visit(Value.DecimalValue value) + { + throw new UnsupportedOperationException(); + } + + default T visit(Value.FloatValue value) + { + throw new UnsupportedOperationException(); + } + + default T visit(Value.IntegerValue value) + { + throw new UnsupportedOperationException(); + } + + default T visit(Value.StrictTimeValue value) + { + throw new UnsupportedOperationException(); + } + + default T visit(Value.StringValue value) + { + throw new UnsupportedOperationException(); + } +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/BaseV1.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/BaseV1.java new file mode 100644 index 0000000000..0e50cc08de --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/BaseV1.java @@ -0,0 +1,804 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element.v1; + +import org.finos.legend.pure.m3.navigation.PrimitiveUtilities; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.m4.coreinstance.primitive.date.DateFunctions; +import org.finos.legend.pure.m4.coreinstance.primitive.date.PureDate; +import org.finos.legend.pure.m4.coreinstance.primitive.strictTime.PureStrictTime; +import org.finos.legend.pure.m4.coreinstance.primitive.strictTime.StrictTimeFunctions; +import org.finos.legend.pure.m4.serialization.Reader; +import org.finos.legend.pure.m4.serialization.Writer; + +import java.math.BigInteger; +import java.util.Arrays; + +class BaseV1 +{ + static final int VALUE_PRESENT_MASK = 0b1000_0000; + static final int VALUE_PRESENT = 0b1000_0000; + static final int VALUE_NOT_PRESENT = 0b0000_0000; + + static final int INTEGRAL_WIDTH_MASK = 0b0000_0011; + static final int BYTE_WIDTH = 0b0000_0000; + static final int SHORT_WIDTH = 0b0000_0001; + static final int INT_WIDTH = 0b0000_0010; + static final int LONG_WIDTH = 0b0000_0011; + + static final int NEGATIVE_MASK = 0b0000_0001; + static final int NON_NEGATIVE = 0b0000_0000; + static final int NEGATIVE = 0b0000_0001; + + static final int NODE_TYPE_MASK = 0b1110_0000; + static final int INTERNAL_REFERENCE = 0b0000_0000; + static final int EXTERNAL_REFERENCE = 0b1000_0000; + static final int BOOLEAN = 0b0100_0000; + static final int BYTE = 0b0010_0000; + static final int DATE = 0b1100_0000; + static final int NUMBER = 0b1010_0000; + static final int STRICT_TIME = 0b0110_0000; + static final int STRING = 0b1110_0000; + + static final int BOOLEAN_MASK = 0b0000_0001; + static final int BOOLEAN_FALSE = 0b0000_0000; + static final int BOOLEAN_TRUE = 0b0000_0001; + + static final int NUMBER_TYPE_MASK = 0b0001_1000; + static final int INTEGER_TYPE = 0b0000_0000; + static final int BIG_INTEGER_TYPE = 0b0000_1000; + static final int FLOAT_TYPE = 0b0001_0000; + static final int DECIMAL_TYPE = 0b0001_1000; + + static final int DECIMAL_POINT_MASK = 0b0000_0010; + static final int HAS_DECIMAL_POINT = 0b0000_0010; + static final int HAS_NO_DECIMAL_POINT = 0b0000_0000; + + static final int DATE_TYPE_MASK = 0b0001_1000; + static final int DATE_TYPE = 0b0000_0000; + static final int STRICT_DATE_TYPE = 0b0000_1000; + static final int DATE_TIME_TYPE = 0b0001_0000; + static final int LATEST_DATE_TYPE = 0b0001_1000; + + static final int DATE_WIDTH_MASK = 0b0000_0111; + static final int YEAR_DATE_WIDTH = 0b0000_0000; + static final int MONTH_DATE_WIDTH = 0b0000_0001; + static final int DAY_DATE_WIDTH = 0b0000_0010; + static final int HOUR_DATE_WIDTH = 0b0000_0100; + static final int MINUTE_DATE_WIDTH = 0b0000_0011; + static final int SECOND_DATE_WIDTH = 0b0000_0110; + static final int SUBSECOND_DATE_WIDTH = 0b0000_0101; + + static final int DIGIT_STRING_MASK = 0b1100_0000; + static final int EMPTY_DIGIT_STRING = 0b1000_0000; + static final int SINGLE_DIGIT_STRING = 0b0100_0000; + static final int REPEATED_DIGIT_STRING = 0b1100_0000; + + // Value present + + static boolean isValuePresent(int code) + { + switch (code & VALUE_PRESENT_MASK) + { + case VALUE_PRESENT: + { + return true; + } + case VALUE_NOT_PRESENT: + { + return false; + } + default: + { + throw new RuntimeException(String.format("Unknown value present code: %02x", code & VALUE_PRESENT_MASK)); + } + } + } + + // Boolean + + static void serializeBoolean(Writer writer, CoreInstance booleanInstance) + { + serializeBoolean(writer, PrimitiveUtilities.getBooleanValue(booleanInstance)); + } + + static void serializeBoolean(Writer writer, boolean value) + { + writer.writeByte((byte) (BOOLEAN | (value ? BOOLEAN_TRUE : BOOLEAN_FALSE))); + } + + static boolean deserializeBoolean(int code) + { + if ((code & NODE_TYPE_MASK) != BOOLEAN) + { + throw new RuntimeException(String.format("Not a Boolean: %02x", code & NODE_TYPE_MASK)); + } + switch (code & BOOLEAN_MASK) + { + case BOOLEAN_FALSE: + { + return false; + } + case BOOLEAN_TRUE: + { + return true; + } + default: + { + throw new RuntimeException(String.format("Unknown Boolean value code: %02x", code & BOOLEAN_MASK)); + } + } + } + + // Byte + + static void serializeByte(Writer writer, CoreInstance byteInstance) + { + serializeByte(writer, PrimitiveUtilities.getByteValue(byteInstance)); + } + + static void serializeByte(Writer writer, byte value) + { + writer.writeByte((byte) BYTE); + writer.writeByte(value); + } + + static byte deserializeByte(Reader reader) + { + return reader.readByte(); + } + + // Decimal/Float + + static void serializeDecimal(Writer writer, CoreInstance decimalInstance) + { + serializeFloatOrDecimal(writer, DECIMAL_TYPE, decimalInstance.getName()); + } + + static void serializeFloat(Writer writer, CoreInstance floatInstance) + { + serializeFloatOrDecimal(writer, FLOAT_TYPE, floatInstance.getName()); + } + + private static void serializeFloatOrDecimal(Writer writer, int type, String name) + { + boolean negative = name.charAt(0) == '-'; + int decimalIndex = name.indexOf('.'); + boolean hasDecimal = decimalIndex != -1; + writer.writeByte((byte) (NUMBER | type | getHasDecimalPoint(hasDecimal) | getNegative(negative))); + if (hasDecimal) + { + writeDigitString(writer, name.substring(negative ? 1 : 0, decimalIndex)); + writeDigitString(writer, name.substring(decimalIndex + 1)); + } + else + { + writeDigitString(writer, name.substring(negative ? 1 : 0)); + } + } + + static String deserializeDecimal(Reader reader, int code) + { + return deserializeFloatOrDecimal(reader, code); + } + + static String deserializeFloat(Reader reader, int code) + { + return deserializeFloatOrDecimal(reader, code); + } + + private static String deserializeFloatOrDecimal(Reader reader, int code) + { + StringBuilder builder = new StringBuilder(); + if (isNegative(code)) + { + builder.append('-'); + } + builder.append(readDigitString(reader)); + if (hasDecimalPoint(code)) + { + builder.append('.').append(readDigitString(reader)); + } + return builder.toString(); + } + + // Integer + + static void serializeInteger(Writer writer, CoreInstance integerInstance) + { + Number value = PrimitiveUtilities.getIntegerValue(integerInstance); + if (value instanceof BigInteger) + { + boolean negative = ((BigInteger) value).signum() < 0; + writer.writeByte((byte) (NUMBER | BIG_INTEGER_TYPE | getNegative(negative))); + String name = integerInstance.getName(); + writeDigitString(writer, negative ? name.substring(1) : name); + } + else + { + long l = value.longValue(); + int width = getLongWidth(l); + writer.writeByte((byte) (NUMBER | INTEGER_TYPE | width)); + writeLongOfWidth(writer, l, width); + } + } + + static Number deserializeInteger(Reader reader, int code) + { + switch (code & NUMBER_TYPE_MASK) + { + case INTEGER_TYPE: + { + if ((code & INTEGRAL_WIDTH_MASK) == LONG_WIDTH) + { + return deserializeOrdinaryIntegerAsLong(reader, code); + } + return deserializeOrdinaryIntegerAsInt(reader, code); + } + case BIG_INTEGER_TYPE: + { + return deserializeBigInteger(reader, code); + } + default: + { + throw new RuntimeException(String.format("Unknown Integer type code: %02x", code & BOOLEAN_MASK)); + } + } + } + + static long deserializeOrdinaryIntegerAsLong(Reader reader, int code) + { + return readLongOfWidth(reader, code); + } + + static int deserializeOrdinaryIntegerAsInt(Reader reader, int code) + { + return readIntOfWidth(reader, code); + } + + static BigInteger deserializeBigInteger(Reader reader, int code) + { + boolean negative = isNegative(code); + String string = readDigitString(reader); + return new BigInteger(negative ? ('-' + string) : string); + } + + static int getIntWidth(int... ints) + { + int type = BYTE_WIDTH; + for (int i : ints) + { + switch (getIntWidth(i)) + { + case INT_WIDTH: + { + return INT_WIDTH; + } + case SHORT_WIDTH: + { + type = SHORT_WIDTH; + } + } + } + return type; + } + + static int getLongWidth(long... longs) + { + int type = BYTE_WIDTH; + for (long l : longs) + { + switch (getLongWidth(l)) + { + case LONG_WIDTH: + { + return LONG_WIDTH; + } + case INT_WIDTH: + { + type = INT_WIDTH; + break; + } + case SHORT_WIDTH: + { + if (type == BYTE_WIDTH) + { + type = SHORT_WIDTH; + } + } + } + } + return type; + } + + static int getIntWidth(int i) + { + return (i < 0) ? + (i >= Byte.MIN_VALUE) ? BYTE_WIDTH : ((i >= Short.MIN_VALUE) ? SHORT_WIDTH : INT_WIDTH) : + (i <= Byte.MAX_VALUE) ? BYTE_WIDTH : ((i <= Short.MAX_VALUE) ? SHORT_WIDTH : INT_WIDTH); + } + + static int getLongWidth(long l) + { + return (l < 0) ? + (l >= Byte.MIN_VALUE) ? BYTE_WIDTH : ((l >= Short.MIN_VALUE) ? SHORT_WIDTH : ((l >= Integer.MIN_VALUE) ? INT_WIDTH : LONG_WIDTH)) : + (l <= Byte.MAX_VALUE) ? BYTE_WIDTH : ((l <= Short.MAX_VALUE) ? SHORT_WIDTH : ((l <= Integer.MAX_VALUE) ? INT_WIDTH : LONG_WIDTH)); + + } + + static void writeIntOfWidth(Writer writer, int i, int widthCode) + { + switch (widthCode & INTEGRAL_WIDTH_MASK) + { + case BYTE_WIDTH: + { + writer.writeByte((byte) i); + return; + } + case SHORT_WIDTH: + { + writer.writeShort((short) i); + return; + } + case INT_WIDTH: + { + writer.writeInt(i); + return; + } + default: + { + throw new RuntimeException(String.format("Unknown long width code: %02x", widthCode & INTEGRAL_WIDTH_MASK)); + } + } + } + + static void writeLongOfWidth(Writer writer, long l, int intType) + { + switch (intType & INTEGRAL_WIDTH_MASK) + { + case BYTE_WIDTH: + { + writer.writeByte((byte) l); + return; + } + case SHORT_WIDTH: + { + writer.writeShort((short) l); + return; + } + case INT_WIDTH: + { + writer.writeInt((int) l); + return; + } + case LONG_WIDTH: + { + writer.writeLong(l); + return; + } + default: + { + throw new RuntimeException(String.format("Unknown long width code: %02x", intType & INTEGRAL_WIDTH_MASK)); + } + } + } + + static int readIntOfWidth(Reader reader, int widthCode) + { + switch (widthCode & INTEGRAL_WIDTH_MASK) + { + case BYTE_WIDTH: + { + return reader.readByte(); + } + case SHORT_WIDTH: + { + return reader.readShort(); + } + case INT_WIDTH: + { + return reader.readInt(); + } + default: + { + throw new RuntimeException(String.format("Unknown int width code: %02x", widthCode & INTEGRAL_WIDTH_MASK)); + } + } + } + + static long readLongOfWidth(Reader reader, int widthCode) + { + switch (widthCode & INTEGRAL_WIDTH_MASK) + { + case BYTE_WIDTH: + { + return reader.readByte(); + } + case SHORT_WIDTH: + { + return reader.readShort(); + } + case INT_WIDTH: + { + return reader.readInt(); + } + case LONG_WIDTH: + { + return reader.readLong(); + } + default: + { + throw new RuntimeException(String.format("Unknown long width code: %02x", widthCode & INTEGRAL_WIDTH_MASK)); + } + } + } + + static int getNegative(boolean negative) + { + return negative ? NEGATIVE : NON_NEGATIVE; + } + + static boolean isNegative(int parityCode) + { + return (parityCode & NEGATIVE_MASK) == NEGATIVE; + } + + + static int getHasDecimalPoint(boolean hasDecimalPoint) + { + return hasDecimalPoint ? HAS_DECIMAL_POINT : HAS_NO_DECIMAL_POINT; + } + + static boolean hasDecimalPoint(int code) + { + return (code & DECIMAL_POINT_MASK) == HAS_DECIMAL_POINT; + } + + + private static void writeDigitString(Writer writer, String string) + { + int len = string.length(); + if (len == 0) + { + writer.writeByte((byte) EMPTY_DIGIT_STRING); + return; + } + if (len == 1) + { + writer.writeByte((byte) SINGLE_DIGIT_STRING); + writer.writeByte((byte) string.charAt(0)); + return; + } + + int lenWidth = getIntWidth(len); + char firstChar = string.charAt(0); + if (string.chars().allMatch(c -> c == firstChar)) + { + writer.writeByte((byte) (REPEATED_DIGIT_STRING | lenWidth)); + writeIntOfWidth(writer, len, lenWidth); + writer.writeByte((byte) firstChar); + return; + } + + writer.writeByte((byte) lenWidth); + writeIntOfWidth(writer, len, lenWidth); + int index = 0; + while (index < len) + { + int i = string.charAt(index++) - '0'; + if (index < len) + { + i = (10 * i) + (string.charAt(index++) - '0'); + } + writer.writeByte((byte) i); + } + } + + private static String readDigitString(Reader reader) + { + int code = reader.readByte(); + switch (code & DIGIT_STRING_MASK) + { + case EMPTY_DIGIT_STRING: + { + return ""; + } + case SINGLE_DIGIT_STRING: + { + return String.valueOf((char) reader.readByte()); + } + case REPEATED_DIGIT_STRING: + { + int len = readIntOfWidth(reader, code); + char c = (char) reader.readByte(); + char[] chars = new char[len]; + Arrays.fill(chars, c); + return String.valueOf(chars); + } + default: + { + int len = readIntOfWidth(reader, code); + StringBuilder builder = new StringBuilder(len); + int remaining = len; + while (remaining > 1) + { + byte b = reader.readByte(); + char c1; + char c2; + if (b < 10) + { + c1 = '0'; + c2 = (char) ('0' + b); + } + else + { + c1 = (char) ('0' + (b / 10)); + c2 = (char) ('0' + (b % 10)); + } + builder.append(c1).append(c2); + remaining -= 2; + } + if (remaining == 1) + { + byte b = reader.readByte(); + builder.append((char) ('0' + b)); + } + return builder.toString(); + } + } + } + + // Date/Time + + static void serializeDate(Writer writer, CoreInstance instance) + { + serializePureDate(writer, instance, DATE_TYPE); + } + + static void serializeDateTime(Writer writer, CoreInstance instance) + { + serializePureDate(writer, instance, DATE_TIME_TYPE); + } + + static void serializeStrictDate(Writer writer, CoreInstance instance) + { + serializePureDate(writer, instance, STRICT_DATE_TYPE); + } + + static void serializeLatestDate(Writer writer, CoreInstance instance) + { + serializeLatestDate(writer); + } + + static void serializeLatestDate(Writer writer) + { + writer.writeByte((byte) (DATE | LATEST_DATE_TYPE)); + } + + static void serializeStrictTime(Writer writer, CoreInstance instance) + { + PureStrictTime value = PrimitiveUtilities.getStrictTimeValue(instance); + writer.writeByte((byte) (STRICT_TIME | getStrictTimeWidth(value))); + writer.writeByte((byte) value.getHour()); + writer.writeByte((byte) value.getMinute()); + if (value.hasSecond()) + { + writer.writeByte((byte) value.getSecond()); + if (value.hasSubsecond()) + { + writeDigitString(writer, value.getSubsecond()); + } + } + } + + static PureDate deserializeDate(Reader reader, int code) + { + switch (code & DATE_TYPE_MASK) + { + case LATEST_DATE_TYPE: + { + return null; + } + case DATE_TYPE: + case DATE_TIME_TYPE: + case STRICT_DATE_TYPE: + { + int year = reader.readInt(); + switch (code & DATE_WIDTH_MASK) + { + case YEAR_DATE_WIDTH: + { + return DateFunctions.newPureDate(year); + } + case MONTH_DATE_WIDTH: + { + int month = reader.readByte(); + return DateFunctions.newPureDate(year, month); + } + case DAY_DATE_WIDTH: + { + int month = reader.readByte(); + int day = reader.readByte(); + return DateFunctions.newPureDate(year, month, day); + } + case HOUR_DATE_WIDTH: + { + int month = reader.readByte(); + int day = reader.readByte(); + int hour = reader.readByte(); + return DateFunctions.newPureDate(year, month, day, hour); + } + case MINUTE_DATE_WIDTH: + { + int month = reader.readByte(); + int day = reader.readByte(); + int hour = reader.readByte(); + int minute = reader.readByte(); + return DateFunctions.newPureDate(year, month, day, hour, minute); + } + case SECOND_DATE_WIDTH: + { + int month = reader.readByte(); + int day = reader.readByte(); + int hour = reader.readByte(); + int minute = reader.readByte(); + int second = reader.readByte(); + return DateFunctions.newPureDate(year, month, day, hour, minute, second); + } + case SUBSECOND_DATE_WIDTH: + { + int month = reader.readByte(); + int day = reader.readByte(); + int hour = reader.readByte(); + int minute = reader.readByte(); + int second = reader.readByte(); + String subsecond = readDigitString(reader); + return DateFunctions.newPureDate(year, month, day, hour, minute, second, subsecond); + } + default: + { + throw new RuntimeException(String.format("Unknown Date width code: %02x", code & DATE_WIDTH_MASK)); + } + } + } + default: + { + throw new RuntimeException(String.format("Unknown Date type code: %02x", code & DATE_TYPE_MASK)); + } + } + } + + static PureStrictTime deserializeStrictTime(Reader reader, int code) + { + switch (code & DATE_WIDTH_MASK) + { + case MINUTE_DATE_WIDTH: + { + int hour = reader.readByte(); + int minute = reader.readByte(); + return StrictTimeFunctions.newPureStrictTime(hour, minute); + } + case SECOND_DATE_WIDTH: + { + int hour = reader.readByte(); + int minute = reader.readByte(); + int second = reader.readByte(); + return StrictTimeFunctions.newPureStrictTime(hour, minute, second); + } + case SUBSECOND_DATE_WIDTH: + { + int hour = reader.readByte(); + int minute = reader.readByte(); + int second = reader.readByte(); + String subsecond = readDigitString(reader); + return StrictTimeFunctions.newPureStrictTime(hour, minute, second, subsecond); + } + default: + { + throw new RuntimeException(String.format("Unknown StrictTime width code: %02x", code & DATE_WIDTH_MASK)); + } + } + } + + private static void serializePureDate(Writer writer, CoreInstance instance, int dateTypeCode) + { + serializePureDate(writer, PrimitiveUtilities.getDateValue(instance), dateTypeCode); + } + + private static void serializePureDate(Writer writer, PureDate date, int dateTypeCode) + { + writer.writeByte((byte) (DATE | dateTypeCode | getDateWidth(date))); + writer.writeInt(date.getYear()); + if (date.hasMonth()) + { + writer.writeByte((byte) date.getMonth()); + if (date.hasDay()) + { + writer.writeByte((byte) date.getDay()); + if (date.hasHour()) + { + writer.writeByte((byte) date.getHour()); + if (date.hasMinute()) + { + writer.writeByte((byte) date.getMinute()); + if (date.hasSecond()) + { + writer.writeByte((byte) date.getSecond()); + if (date.hasSubsecond()) + { + writeDigitString(writer, date.getSubsecond()); + } + } + } + } + } + } + } + + private static int getDateWidth(PureDate date) + { + if (date.hasSubsecond()) + { + return SUBSECOND_DATE_WIDTH; + } + if (date.hasSecond()) + { + return SECOND_DATE_WIDTH; + } + if (date.hasMinute()) + { + return MINUTE_DATE_WIDTH; + } + if (date.hasHour()) + { + return HOUR_DATE_WIDTH; + } + if (date.hasDay()) + { + return DAY_DATE_WIDTH; + } + if (date.hasMonth()) + { + return MONTH_DATE_WIDTH; + } + return YEAR_DATE_WIDTH; + } + + private static int getStrictTimeWidth(PureStrictTime time) + { + if (time.hasSubsecond()) + { + return SUBSECOND_DATE_WIDTH; + } + if (time.hasSecond()) + { + return SECOND_DATE_WIDTH; + } + return MINUTE_DATE_WIDTH; + } + + // String + + static void serializeString(Writer writer, CoreInstance instance) + { + writer.writeByte((byte) STRING); + writer.writeString(PrimitiveUtilities.getStringValue(instance)); + } + + static String deserializeString(Reader reader) + { + return reader.readString(); + } +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/ConcreteElementSerializerV1.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/ConcreteElementSerializerV1.java new file mode 100644 index 0000000000..edfb354418 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/ConcreteElementSerializerV1.java @@ -0,0 +1,43 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element.v1; + +import org.finos.legend.pure.m3.serialization.compiler.element.ConcreteElementSerializerExtension; +import org.finos.legend.pure.m3.serialization.compiler.element.DeserializedConcreteElement; +import org.finos.legend.pure.m3.serialization.compiler.element.SerializationContext; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.m4.serialization.Reader; +import org.finos.legend.pure.m4.serialization.Writer; + +public class ConcreteElementSerializerV1 implements ConcreteElementSerializerExtension +{ + @Override + public int version() + { + return 1; + } + + @Override + public void serialize(Writer writer, CoreInstance element, SerializationContext serializationContext) + { + new SerializerV1(element, serializationContext).serialize(writer); + } + + @Override + public DeserializedConcreteElement deserialize(Reader reader, SerializationContext serializationContext) + { + return new DeserializerV1(serializationContext).deserialize(reader); + } +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/DeserializerV1.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/DeserializerV1.java new file mode 100644 index 0000000000..e3ea7b8dd4 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/DeserializerV1.java @@ -0,0 +1,354 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element.v1; + +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.ListIterable; +import org.eclipse.collections.api.list.MutableList; +import org.finos.legend.pure.m3.navigation.M3Properties; +import org.finos.legend.pure.m3.navigation._package._Package; +import org.finos.legend.pure.m3.serialization.compiler.element.DeserializedConcreteElement; +import org.finos.legend.pure.m3.serialization.compiler.element.DeserializedElement; +import org.finos.legend.pure.m3.serialization.compiler.element.PropertyValues; +import org.finos.legend.pure.m3.serialization.compiler.element.Reference; +import org.finos.legend.pure.m3.serialization.compiler.element.SerializationContext; +import org.finos.legend.pure.m3.serialization.compiler.element.Value; +import org.finos.legend.pure.m3.serialization.compiler.element.ValueOrReference; +import org.finos.legend.pure.m4.coreinstance.SourceInformation; +import org.finos.legend.pure.m4.coreinstance.primitive.date.PureDate; +import org.finos.legend.pure.m4.coreinstance.primitive.strictTime.PureStrictTime; +import org.finos.legend.pure.m4.serialization.Reader; + +import java.math.BigInteger; + +public class DeserializerV1 extends BaseV1 +{ + private final SerializationContext serializationContext; + + DeserializerV1(SerializationContext serializationContext) + { + this.serializationContext = serializationContext; + } + + DeserializedConcreteElement deserialize(Reader reader) + { + int referenceIdVersion = this.serializationContext.getReferenceIdProvider().version(); + Reader stringIndexedReader = this.serializationContext.getStringIndexer().readStringIndex(reader); + String path = stringIndexedReader.readString(); + InternalNode[] nodes = deserializeNodes(stringIndexedReader); + return new DeserializedConcreteElement() + { + @Override + public String getPath() + { + return path; + } + + @Override + public DeserializedElement getInternalElement(int id) + { + return nodes[id]; + } + + @Override + public int getReferenceIdVersion() + { + return referenceIdVersion; + } + + @Override + public String getName() + { + return nodes[0].getName(); + } + + @Override + public String getClassifierReferenceId() + { + return nodes[0].getClassifierReferenceId(); + } + + @Override + public SourceInformation getSourceInformation() + { + return nodes[0].getSourceInformation(); + } + + @Override + public Iterable getPropertyValues() + { + return nodes[0].getPropertyValues(); + } + }; + } + + private InternalNode[] deserializeNodes(Reader reader) + { + String sourceId = reader.readString(); + int nodeCount = reader.readInt(); + int internalIdWidth = getIntWidth(nodeCount); + InternalNode[] nodes = new InternalNode[nodeCount]; + for (int i = 0; i < nodeCount; i++) + { + nodes[i] = deserializeNode(reader, sourceId, internalIdWidth); + } + return nodes; + } + + private InternalNode deserializeNode(Reader reader, String sourceId, int internalIdWidth) + { + String name = readName(reader); + String classifierId = readClassifier(reader); + SourceInformation sourceInfo = readSourceInfo(reader, sourceId); + int propertyCount = reader.readInt(); + MutableList propertiesWithValues = Lists.mutable.ofInitialCapacity(propertyCount); + for (int i = 0; i < propertyCount; i++) + { + propertiesWithValues.add(readPropertyWithValues(reader, internalIdWidth)); + } + return new InternalNode(name, classifierId, sourceInfo, propertiesWithValues); + } + + private String readName(Reader reader) + { + boolean hasName = reader.readBoolean(); + return hasName ? reader.readString() : null; + } + + private String readClassifier(Reader reader) + { + return reader.readString(); + } + + private SourceInformation readSourceInfo(Reader reader, String sourceId) + { + int code = reader.readByte(); + switch (code & VALUE_PRESENT_MASK) + { + case VALUE_NOT_PRESENT: + { + return null; + } + case VALUE_PRESENT: + { + int startLine = readIntOfWidth(reader, code); + int startCol = readIntOfWidth(reader, code); + int line = readIntOfWidth(reader, code); + int col = readIntOfWidth(reader, code); + int endLine = readIntOfWidth(reader, code); + int endCol = readIntOfWidth(reader, code); + return new SourceInformation(sourceId, startLine, startCol, line, col, endLine, endCol); + } + default: + { + throw new RuntimeException(String.format("Unknown value present code: %02x", code & NODE_TYPE_MASK)); + } + } + } + + private PropertyWithValues readPropertyWithValues(Reader reader, int internalIdWidth) + { + String name = reader.readString(); + String sourceType = reader.readString(); + int valueCount = reader.readInt(); + MutableList values = Lists.mutable.ofInitialCapacity(valueCount); + for (int i = 0; i < valueCount; i++) + { + values.add(readPropertyValue(reader, internalIdWidth)); + } + return new PropertyWithValues(name, sourceType, values); + } + + private ValueOrReference readPropertyValue(Reader reader, int internalIdWidth) + { + int code = reader.readByte(); + switch (code & NODE_TYPE_MASK) + { + case INTERNAL_REFERENCE: + { + int id = readIntOfWidth(reader, internalIdWidth); + return Reference.newInternalReference(id); + } + case EXTERNAL_REFERENCE: + { + String id = reader.readString(); + return Reference.newExternalReference(id); + } + case BOOLEAN: + { + boolean value = deserializeBoolean(code); + return Value.newBooleanValue(value); + } + case BYTE: + { + byte value = deserializeByte(reader); + return Value.newByteValue(value); + } + case DATE: + { + switch (code & DATE_TYPE_MASK) + { + case DATE_TYPE: + { + PureDate value = deserializeDate(reader, code); + return Value.newDateValue(value); + } + case DATE_TIME_TYPE: + { + PureDate value = deserializeDate(reader, code); + return Value.newDateTimeValue(value); + } + case STRICT_DATE_TYPE: + { + PureDate value = deserializeDate(reader, code); + return Value.newStrictDateValue(value); + } + case LATEST_DATE_TYPE: + { + return Value.newLatestDateValue(); + } + default: + { + throw new RuntimeException(String.format("Unknown date type code: %02x", code & DATE_TYPE_MASK)); + } + } + } + case NUMBER: + { + switch (code & NUMBER_TYPE_MASK) + { + case DECIMAL_TYPE: + { + String value = deserializeDecimal(reader, code); + return Value.newDecimalValue(value); + } + case FLOAT_TYPE: + { + String value = deserializeFloat(reader, code); + return Value.newFloatValue(value); + } + case INTEGER_TYPE: + { + if (((code & INTEGRAL_WIDTH_MASK) == LONG_WIDTH)) + { + long value = deserializeOrdinaryIntegerAsLong(reader, code); + return Value.newIntegerValue(value); + } + else + { + int value = deserializeOrdinaryIntegerAsInt(reader, code); + return Value.newIntegerValue(value); + } + } + case BIG_INTEGER_TYPE: + { + BigInteger value = deserializeBigInteger(reader, code); + return Value.newIntegerValue(value); + } + default: + { + throw new RuntimeException(String.format("Unknown number type code: %02x", code & NUMBER_TYPE_MASK)); + } + } + } + case STRICT_TIME: + { + PureStrictTime value = deserializeStrictTime(reader, code); + return Value.newStrictTimeValue(value); + } + case STRING: + { + String value = deserializeString(reader); + return Value.newStringValue(value); + } + default: + { + throw new RuntimeException(String.format("Unknown node type code: %02x", code & NODE_TYPE_MASK)); + } + } + } + + private static class InternalNode implements DeserializedElement + { + private final String name; + private final String classifierId; + private final SourceInformation sourceInfo; + private final ListIterable propertiesWithValues; + + private InternalNode(String name, String classifierId, SourceInformation sourceInfo, ListIterable propertiesWithValues) + { + this.name = name; + this.classifierId = classifierId; + this.sourceInfo = sourceInfo; + this.propertiesWithValues = propertiesWithValues; + } + + @Override + public String getName() + { + return this.name; + } + + @Override + public String getClassifierReferenceId() + { + return this.classifierId; + } + + @Override + public SourceInformation getSourceInformation() + { + return this.sourceInfo; + } + + @Override + public Iterable getPropertyValues() + { + return this.propertiesWithValues; + } + } + + private static class PropertyWithValues implements PropertyValues + { + private final String name; + private final String sourceType; + private final ListIterable values; + + private PropertyWithValues(String name, String sourceType, ListIterable values) + { + this.name = name; + this.sourceType = sourceType; + this.values = values; + } + + @Override + public String getPropertyName() + { + return this.name; + } + + @Override + public ListIterable getRealKey() + { + return _Package.convertM3PathToM4(this.sourceType).with(M3Properties.properties).with(this.name); + } + + @Override + public ListIterable getValues() + { + return this.values; + } + } +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/SerializerV1.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/SerializerV1.java new file mode 100644 index 0000000000..024953e4da --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/SerializerV1.java @@ -0,0 +1,347 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element.v1; + +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.list.ImmutableList; +import org.eclipse.collections.api.list.ListIterable; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.api.map.ImmutableMap; +import org.eclipse.collections.api.map.MapIterable; +import org.eclipse.collections.api.map.MutableMap; +import org.eclipse.collections.api.map.primitive.MutableObjectIntMap; +import org.eclipse.collections.api.map.primitive.ObjectIntMap; +import org.eclipse.collections.api.set.MutableSet; +import org.eclipse.collections.api.tuple.Pair; +import org.eclipse.collections.impl.factory.primitive.ObjectIntMaps; +import org.eclipse.collections.impl.tuple.Tuples; +import org.finos.legend.pure.m3.navigation.M3Paths; +import org.finos.legend.pure.m3.navigation.M3PropertyPaths; +import org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement; +import org.finos.legend.pure.m3.navigation.PrimitiveUtilities; +import org.finos.legend.pure.m3.navigation.ProcessorSupport; +import org.finos.legend.pure.m3.navigation.property.Property; +import org.finos.legend.pure.m3.serialization.compiler.element.SerializationContext; +import org.finos.legend.pure.m4.ModelRepository; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.m4.coreinstance.SourceInformation; +import org.finos.legend.pure.m4.serialization.Writer; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.function.BiConsumer; + +class SerializerV1 extends BaseV1 +{ + private static final ImmutableMap> SKIP_PROPERTY_PATHS = M3PropertyPaths.BACK_REFERENCE_PROPERTY_PATHS + .groupByUniqueKey(ImmutableList::getLast, Maps.mutable.ofInitialCapacity(M3PropertyPaths.BACK_REFERENCE_PROPERTY_PATHS.size())) + .toImmutable(); + + private final CoreInstance element; + private final SerializationContext serializationContext; + private final ProcessorSupport processorSupport; + private final MutableList nodesToSerialize = Lists.mutable.empty(); + private final MutableMap classifierPathCache = Maps.mutable.empty(); + private final MutableMap> propertyInfoCache = Maps.mutable.empty(); + private final MapIterable> primitiveSerializers; + + SerializerV1(CoreInstance element, SerializationContext serializationContext) + { + this.element = element; + this.serializationContext = serializationContext; + this.processorSupport = serializationContext.getProcessorSupport(); + this.primitiveSerializers = Maps.mutable.>ofInitialCapacity(11) + .withKeyValue(this.processorSupport.package_getByUserPath(M3Paths.Boolean), SerializerV1::serializeBoolean) + .withKeyValue(this.processorSupport.package_getByUserPath(M3Paths.Byte), SerializerV1::serializeByte) + .withKeyValue(this.processorSupport.package_getByUserPath(M3Paths.Date), SerializerV1::serializeDate) + .withKeyValue(this.processorSupport.package_getByUserPath(M3Paths.DateTime), SerializerV1::serializeDateTime) + .withKeyValue(this.processorSupport.package_getByUserPath(M3Paths.Decimal), SerializerV1::serializeDecimal) + .withKeyValue(this.processorSupport.package_getByUserPath(M3Paths.Float), SerializerV1::serializeFloat) + .withKeyValue(this.processorSupport.package_getByUserPath(M3Paths.Integer), SerializerV1::serializeInteger) + .withKeyValue(this.processorSupport.package_getByUserPath(M3Paths.LatestDate), SerializerV1::serializeLatestDate) + .withKeyValue(this.processorSupport.package_getByUserPath(M3Paths.StrictDate), SerializerV1::serializeStrictDate) + .withKeyValue(this.processorSupport.package_getByUserPath(M3Paths.StrictTime), SerializerV1::serializeStrictTime) + .withKeyValue(this.processorSupport.package_getByUserPath(M3Paths.String), SerializerV1::serializeString); + } + + void serialize(Writer writer) + { + Writer stringIndexedWriter = collectNodesAndIndexStrings(writer); + stringIndexedWriter.writeString(PackageableElement.getUserPathForPackageableElement(this.element)); + stringIndexedWriter.writeString(this.element.getSourceInformation().getSourceId()); + serializeNodes(stringIndexedWriter); + } + + private Writer collectNodesAndIndexStrings(Writer writer) + { + MutableSet strings = Sets.mutable.empty(); + MutableSet visited = Sets.mutable.empty(); + Deque deque = new ArrayDeque<>(); + deque.add(toSerialize(this.element)); + strings.add(PackageableElement.getUserPathForPackageableElement(this.element)); + strings.add(this.element.getSourceInformation().getSourceId()); + while (!deque.isEmpty()) + { + NodeToSerialize node = deque.pollFirst(); + if (visited.add(node.instance)) + { + this.nodesToSerialize.add(node); + if (!isAnonymousInstance(node.instance)) + { + strings.add(node.instance.getName()); + } + // We treat the classifier as an external reference, even if it's not + strings.add(this.serializationContext.getReferenceIdProvider().getReferenceId(node.classifier)); + getPropertyInfos(node.classifier).forEach(propertyInfo -> + { + if (!propertyInfo.skip) + { + ListIterable values = node.instance.getValueForMetaPropertyToMany(propertyInfo.name); + if (values.notEmpty()) + { + strings.add(propertyInfo.name); + strings.add(propertyInfo.sourceType); + values.forEach(value -> + { + if (isExternal(value)) + { + strings.add(this.serializationContext.getReferenceIdProvider().getReferenceId(value)); + } + else + { + CoreInstance valueClassifier = this.processorSupport.getClassifier(value); + if (!isPrimitiveType(valueClassifier)) + { + deque.addLast(toSerialize(value, valueClassifier)); + } + else if (M3Paths.String.equals(valueClassifier.getName())) + { + strings.add(PrimitiveUtilities.getStringValue(value)); + } + } + }); + } + } + }); + } + } + return this.serializationContext.getStringIndexer().writeStringIndex(writer, strings); + } + + private void serializeNodes(Writer writer) + { + MutableObjectIntMap internalIds = ObjectIntMaps.mutable.ofInitialCapacity(this.nodesToSerialize.size()); + this.nodesToSerialize.forEachWithIndex((node, i) -> internalIds.put(node.instance, i)); + writer.writeInt(this.nodesToSerialize.size()); + int internalIdWidth = getIntWidth(this.nodesToSerialize.size()); + this.nodesToSerialize.forEach(node -> serializeNode(writer, node, internalIds, internalIdWidth)); + } + + private void serializeNode(Writer writer, NodeToSerialize node, ObjectIntMap internalIds, int internalIdWidth) + { + serializeName(writer, node.instance); + serializeClassifier(writer, node.classifier); + serializeSourceInfo(writer, node.instance); + + MutableList>> propertiesWithValues = Lists.mutable.empty(); + getPropertyInfos(node.classifier).forEach(propertyInfo -> + { + if (!propertyInfo.skip) + { + ListIterable values = node.instance.getValueForMetaPropertyToMany(propertyInfo.name); + if (values.notEmpty()) + { + propertiesWithValues.add(Tuples.pair(propertyInfo, values)); + } + } + }); + writer.writeInt(propertiesWithValues.size()); + propertiesWithValues.forEach(pair -> + { + PropertyInfo propertyInfo = pair.getOne(); + ListIterable values = pair.getTwo(); + writer.writeString(propertyInfo.name); + writer.writeString(propertyInfo.sourceType); + writer.writeInt(values.size()); + values.forEach(value -> + { + int internalId = internalIds.getIfAbsent(value, -1); + if (internalId != -1) + { + serializeInternalReference(writer, internalId, internalIdWidth); + } + else if (isExternal(value)) + { + serializeExternalReference(writer, this.serializationContext.getReferenceIdProvider().getReferenceId(value)); + } + else + { + CoreInstance valueClassifier = this.processorSupport.getClassifier(value); + BiConsumer serializer = this.primitiveSerializers.get(valueClassifier); + if (serializer == null) + { + StringBuilder builder = new StringBuilder("Expected primitive value, found instance of "); + PackageableElement.writeUserPathForPackageableElement(builder, valueClassifier); + builder.append(": ").append(value); + throw new IllegalStateException(builder.toString()); + } + serializer.accept(writer, value); + } + }); + }); + } + + private void serializeClassifier(Writer writer, CoreInstance classifier) + { + // We treat the classifier as an external reference, even if it's not + writer.writeString(this.serializationContext.getReferenceIdProvider().getReferenceId(classifier)); + } + + private void serializeName(Writer writer, CoreInstance instance) + { + if (isAnonymousInstance(instance)) + { + writer.writeBoolean(false); + } + else + { + writer.writeBoolean(true); + writer.writeString(instance.getName()); + } + } + + private void serializeSourceInfo(Writer writer, CoreInstance instance) + { + SourceInformation sourceInfo = instance.getSourceInformation(); + if (sourceInfo == null) + { + writer.writeByte((byte) VALUE_NOT_PRESENT); + } + else + { + int intWidth = getIntWidth(sourceInfo.getStartLine(), sourceInfo.getStartColumn(), sourceInfo.getLine(), sourceInfo.getColumn(), sourceInfo.getEndLine(), sourceInfo.getEndColumn()); + writer.writeByte((byte) (VALUE_PRESENT | intWidth)); + writeIntOfWidth(writer, sourceInfo.getStartLine(), intWidth); + writeIntOfWidth(writer, sourceInfo.getStartColumn(), intWidth); + writeIntOfWidth(writer, sourceInfo.getLine(), intWidth); + writeIntOfWidth(writer, sourceInfo.getColumn(), intWidth); + writeIntOfWidth(writer, sourceInfo.getEndLine(), intWidth); + writeIntOfWidth(writer, sourceInfo.getEndColumn(), intWidth); + } + } + + private void serializeExternalReference(Writer writer, String id) + { + writer.writeByte((byte) EXTERNAL_REFERENCE); + writer.writeString(id); + } + + private void serializeInternalReference(Writer writer, int id, int idWidth) + { + writer.writeByte((byte) INTERNAL_REFERENCE); + writeIntOfWidth(writer, id, idWidth); + } + + private NodeToSerialize toSerialize(CoreInstance instance) + { + return toSerialize(instance, this.processorSupport.getClassifier(instance)); + } + + private NodeToSerialize toSerialize(CoreInstance instance, CoreInstance classifier) + { + return new NodeToSerialize(instance, classifier); + } + + private String getClassifierPath(CoreInstance classifier) + { + return this.classifierPathCache.getIfAbsentPutWithKey(classifier, PackageableElement::getUserPathForPackageableElement); + } + + private ListIterable getPropertyInfos(CoreInstance classifier) + { + return this.propertyInfoCache.getIfAbsentPut(classifier, () -> + { + MapIterable propertiesByName = this.processorSupport.class_getSimplePropertiesByName(classifier); + MutableList infos = Lists.mutable.ofInitialCapacity(propertiesByName.size()); + propertiesByName.forEachKeyValue((name, property) -> infos.add(computePropertyInfo(name, property))); + return infos.sortThisBy(pi -> pi.name); + }); + } + + private PropertyInfo computePropertyInfo(String name, CoreInstance property) + { + if (shouldSkipProperty(name, property)) + { + return new PropertyInfo(name, null, true); + } + CoreInstance sourceType = Property.getSourceType(property, this.processorSupport); + return new PropertyInfo(name, PackageableElement.getUserPathForPackageableElement(sourceType), false); + } + + private boolean shouldSkipProperty(String propertyName, CoreInstance property) + { + ImmutableList realKeyToSkip = SKIP_PROPERTY_PATHS.get(propertyName); + return (realKeyToSkip != null) && realKeyToSkip.equals(Property.calculatePropertyPath(property, this.processorSupport)); + } + + private boolean isPrimitiveType(CoreInstance classifier) + { + return this.primitiveSerializers.containsKey(classifier); + } + + private boolean isExternal(CoreInstance instance) + { + SourceInformation sourceInfo = instance.getSourceInformation(); + return (sourceInfo != null) && !this.element.getSourceInformation().subsumes(sourceInfo); + } + + private boolean isAnonymousInstance(CoreInstance instance) + { + return ModelRepository.isAnonymousInstanceName(instance.getName()); + } + + private ProcessorSupport getProcessorSupport() + { + return this.serializationContext.getProcessorSupport(); + } + + private static class NodeToSerialize + { + private final CoreInstance instance; + private final CoreInstance classifier; + + private NodeToSerialize(CoreInstance instance, CoreInstance classifier) + { + this.instance = instance; + this.classifier = classifier; + } + } + + private static class PropertyInfo + { + private final String name; + private final String sourceType; + private final boolean skip; + + private PropertyInfo(String name, String sourceType, boolean skip) + { + this.name = name; + this.sourceType = sourceType; + this.skip = skip; + } + } +} diff --git a/legend-pure-core/legend-pure-m3-core/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.compiler.element.ConcreteElementSerializerExtension b/legend-pure-core/legend-pure-m3-core/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.compiler.element.ConcreteElementSerializerExtension new file mode 100644 index 0000000000..fbf1bcc937 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.compiler.element.ConcreteElementSerializerExtension @@ -0,0 +1 @@ +org.finos.legend.pure.m3.serialization.compiler.element.v1.ConcreteElementSerializerV1 diff --git a/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/AbstractTestConcreteElementSerializerExtension.java b/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/AbstractTestConcreteElementSerializerExtension.java new file mode 100644 index 0000000000..929a9dbdb3 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/AbstractTestConcreteElementSerializerExtension.java @@ -0,0 +1,267 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element; + +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.list.ListIterable; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.api.list.primitive.MutableIntList; +import org.eclipse.collections.api.map.MutableMap; +import org.eclipse.collections.api.map.primitive.MutableIntObjectMap; +import org.eclipse.collections.impl.factory.primitive.IntLists; +import org.eclipse.collections.impl.factory.primitive.IntObjectMaps; +import org.finos.legend.pure.m3.coreinstance.Package; +import org.finos.legend.pure.m3.navigation.M3PropertyPaths; +import org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement; +import org.finos.legend.pure.m3.navigation.PrimitiveUtilities; +import org.finos.legend.pure.m3.navigation.property.Property; +import org.finos.legend.pure.m3.serialization.compiler.reference.AbstractReferenceTest; +import org.finos.legend.pure.m3.serialization.compiler.reference.ReferenceIdProvider; +import org.finos.legend.pure.m3.serialization.compiler.reference.ReferenceIdResolver; +import org.finos.legend.pure.m3.serialization.compiler.reference.ReferenceIds; +import org.finos.legend.pure.m3.tools.PackageTreeIterable; +import org.finos.legend.pure.m4.ModelRepository; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.m4.serialization.binary.BinaryReaders; +import org.finos.legend.pure.m4.serialization.binary.BinaryWriters; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.util.function.Function; + +public abstract class AbstractTestConcreteElementSerializerExtension extends AbstractReferenceTest +{ + private ConcreteElementSerializerExtension extension; + private ConcreteElementSerializer serializer; + private ReferenceIds referenceIds; + private final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + @Before + public void setUpExtension() + { + this.extension = getExtension(); + this.serializer = ConcreteElementSerializer.builder(processorSupport).withExtension(this.extension).build(); + this.referenceIds = ReferenceIds.builder(processorSupport).withAvailableExtensions().build(); + } + + @Test + public void testVersions() + { + int expectedVersion = this.extension.version(); + + Assert.assertEquals(expectedVersion, this.serializer.getDefaultVersion()); + Assert.assertTrue(this.serializer.isVersionAvailable(expectedVersion)); + + MutableIntList versions = IntLists.mutable.empty(); + this.serializer.forEachVersion(versions::add); + Assert.assertEquals(IntLists.mutable.with(expectedVersion), versions); + } + + @Test + public void testSerializeDeserializeAll() + { + repository.getTopLevels().forEach(this::testSerializeDeserialize); + PackageTreeIterable.newRootPackageTreeIterable(repository) + .flatCollect(Package::_children) + .select(c -> c.getSourceInformation() != null) + .forEach(this::testSerializeDeserialize); + } + + private void testSerializeDeserialize(CoreInstance element) + { + String path = PackageableElement.getUserPathForPackageableElement(element); + try + { + this.stream.reset(); + this.serializer.serialize(BinaryWriters.newBinaryWriter(this.stream), element); + } + catch (Exception e) + { + throw new RuntimeException("Error serializing " + path, e); + } + DeserializedConcreteElement deserialized; + try + { + deserialized = this.serializer.deserialize(BinaryReaders.newBinaryReader(this.stream.toByteArray())); + } + catch (Exception e) + { + throw new RuntimeException("Error deserializing " + path, e); + } + Assert.assertEquals(path, deserialized.getPath()); + assertDeserialization(deserialized, IntObjectMaps.mutable.empty().withKeyValue(0, element), path, element, deserialized); + } + + private void assertDeserialization(DeserializedConcreteElement concreteElement, MutableIntObjectMap internalReferences, String path, CoreInstance element, DeserializedElement deserialized) + { + ReferenceIdProvider referenceIdProvider = this.referenceIds.provider(concreteElement.getReferenceIdVersion()); + ReferenceIdResolver referenceIdResolver = this.referenceIds.resolver(concreteElement.getReferenceIdVersion()); + + if (ModelRepository.isAnonymousInstanceName(element.getName())) + { + Assert.assertNull(path, deserialized.getName()); + } + else + { + Assert.assertEquals(path, element.getName(), deserialized.getName()); + } + Assert.assertEquals(path, element.getSourceInformation(), deserialized.getSourceInformation()); + Assert.assertEquals(path, referenceIdProvider.getReferenceId(processorSupport.getClassifier(element)), deserialized.getClassifierReferenceId()); + + MutableMap deserializedPropertyValues = Maps.mutable.empty(); + deserialized.getPropertyValues().forEach(pv -> + { + if ((deserializedPropertyValues.put(pv.getPropertyName(), pv) != null)) + { + Assert.fail("Multiple property values for '" + pv.getPropertyName() + "' for " + path); + } + }); + + MutableList nonEmptyPropertyKeys = Lists.mutable.empty(); + processorSupport.class_getSimpleProperties(processorSupport.getClassifier(element)).forEach(property -> + { + String key = property.getName(); + ListIterable realKey = Property.calculatePropertyPath(property, processorSupport); + ListIterable values = element.getValueForMetaPropertyToMany(key); + if (values.notEmpty() && !M3PropertyPaths.BACK_REFERENCE_PROPERTY_PATHS.contains(realKey)) + { + nonEmptyPropertyKeys.add(key); + PropertyValues pValues = deserializedPropertyValues.get(key); + if (pValues != null) + { + Assert.assertEquals(path + "." + key, realKey, pValues.getRealKey()); + } + } + }); + Assert.assertEquals(path, nonEmptyPropertyKeys.sortThis(), deserializedPropertyValues.keysView().toSortedList()); + + for (String key : nonEmptyPropertyKeys) + { + ListIterable values = element.getValueForMetaPropertyToMany(key); + ListIterable pValues = deserializedPropertyValues.get(key).getValues(); + String keyPath = path + "." + key; + Assert.assertEquals(keyPath, values.size(), pValues.size()); + for (int i = 0, size = values.size(); i < size; i++) + { + String keyPathWithIndex = keyPath + "[" + i + "]"; + CoreInstance value = values.get(i); + pValues.get(i).visit(new ValueOrReferenceConsumer() + { + @Override + protected void accept(Reference.ExternalReference reference) + { + CoreInstance resolved = referenceIdResolver.resolveReference(reference.getId()); + Assert.assertSame(keyPathWithIndex + "=" + reference.getId(), value, resolved); + } + + @Override + protected void accept(Reference.InternalReference reference) + { + int internalId = reference.getId(); + CoreInstance visited = internalReferences.get(internalId); + if (visited == null) + { + DeserializedElement resolved = concreteElement.getInternalElement(internalId); + Assert.assertNotNull(keyPathWithIndex + "=" + internalId, resolved); + internalReferences.put(internalId, value); + assertDeserialization(concreteElement, internalReferences, keyPathWithIndex, value, resolved); + } + else + { + Assert.assertSame(keyPathWithIndex + "=" + internalId, value, visited); + } + } + + @Override + protected void accept(Value.BooleanValue bValue) + { + assertValue(keyPathWithIndex, value, PrimitiveUtilities::getBooleanValue, bValue); + } + + @Override + protected void accept(Value.ByteValue bValue) + { + assertValue(keyPathWithIndex, value, PrimitiveUtilities::getByteValue, bValue); + } + + @Override + protected void accept(Value.DateValue dValue) + { + assertValue(keyPathWithIndex, value, PrimitiveUtilities::getDateValue, dValue); + } + + @Override + protected void accept(Value.DateTimeValue dtValue) + { + assertValue(keyPathWithIndex, value, PrimitiveUtilities::getDateValue, dtValue); + } + + @Override + protected void accept(Value.StrictDateValue sdValue) + { + assertValue(keyPathWithIndex, value, PrimitiveUtilities::getDateValue, sdValue); + } + + @Override + protected void accept(Value.LatestDateValue ldValue) + { + assertValue(keyPathWithIndex, value, v -> null, ldValue); + } + + @Override + protected void accept(Value.DecimalValue dValue) + { + assertValue(keyPathWithIndex, value, PrimitiveUtilities::getDecimalValue, dValue); + } + + @Override + protected void accept(Value.FloatValue fValue) + { + assertValue(keyPathWithIndex, value, PrimitiveUtilities::getFloatValue, fValue); + } + + @Override + protected void accept(Value.IntegerValue iValue) + { + assertValue(keyPathWithIndex, value, PrimitiveUtilities::getIntegerValue, iValue); + } + + @Override + protected void accept(Value.StrictTimeValue stValue) + { + assertValue(keyPathWithIndex, value, PrimitiveUtilities::getStrictTimeValue, stValue); + } + + @Override + protected void accept(Value.StringValue sValue) + { + assertValue(keyPathWithIndex, value, PrimitiveUtilities::getStringValue, sValue); + } + }); + } + } + } + + private void assertValue(String message, CoreInstance expectedInstance, Function valueExtractor, Value actualValue) + { + Assert.assertEquals(message, PackageableElement.getUserPathForPackageableElement(processorSupport.getClassifier(expectedInstance)), actualValue.getClassifierPath()); + Assert.assertEquals(message, valueExtractor.apply(expectedInstance), actualValue.getValue()); + } + + protected abstract ConcreteElementSerializerExtension getExtension(); +} diff --git a/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/TestConcreteElementSerializerV1.java b/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/TestConcreteElementSerializerV1.java new file mode 100644 index 0000000000..930263464e --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/TestConcreteElementSerializerV1.java @@ -0,0 +1,27 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element.v1; + +import org.finos.legend.pure.m3.serialization.compiler.element.AbstractTestConcreteElementSerializerExtension; +import org.finos.legend.pure.m3.serialization.compiler.element.ConcreteElementSerializerExtension; + +public class TestConcreteElementSerializerV1 extends AbstractTestConcreteElementSerializerExtension +{ + @Override + protected ConcreteElementSerializerExtension getExtension() + { + return new ConcreteElementSerializerV1(); + } +} diff --git a/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/TestPrimitiveSerialization.java b/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/TestPrimitiveSerialization.java new file mode 100644 index 0000000000..395a7419b8 --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/serialization/compiler/element/v1/TestPrimitiveSerialization.java @@ -0,0 +1,257 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.serialization.compiler.element.v1; + +import org.finos.legend.pure.m4.ModelRepository; +import org.finos.legend.pure.m4.coreinstance.primitive.date.DateFunctions; +import org.finos.legend.pure.m4.coreinstance.primitive.date.PureDate; +import org.finos.legend.pure.m4.coreinstance.primitive.strictTime.PureStrictTime; +import org.finos.legend.pure.m4.coreinstance.primitive.strictTime.StrictTimeFunctions; +import org.finos.legend.pure.m4.serialization.Reader; +import org.finos.legend.pure.m4.serialization.Writer; +import org.finos.legend.pure.m4.serialization.binary.BinaryReaders; +import org.finos.legend.pure.m4.serialization.binary.BinaryWriters; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +public class TestPrimitiveSerialization +{ + private final ModelRepository modelRepository = new ModelRepository(); + private final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + @Test + public void testBooleanSerialization() + { + byte[] trueBytes = serialize(this.modelRepository.newBooleanCoreInstance(true), BaseV1::serializeBoolean); + Assert.assertEquals(1, trueBytes.length); + Assert.assertEquals(BaseV1.BOOLEAN, trueBytes[0] & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(BaseV1.BOOLEAN_TRUE, trueBytes[0] & BaseV1.BOOLEAN_MASK); + Assert.assertTrue(BaseV1.deserializeBoolean(trueBytes[0])); + + byte[] falseBytes = serialize(this.modelRepository.newBooleanCoreInstance(false), BaseV1::serializeBoolean); + Assert.assertEquals(1, falseBytes.length); + Assert.assertEquals(BaseV1.BOOLEAN, falseBytes[0] & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(BaseV1.BOOLEAN_FALSE, falseBytes[0] & BaseV1.BOOLEAN_MASK); + Assert.assertFalse(BaseV1.deserializeBoolean(falseBytes[0])); + } + + @Test + public void testByteSerialization() + { + for (int i = Byte.MIN_VALUE; i <= Byte.MAX_VALUE; i++) + { + byte b = (byte) i; + String message = Integer.toString(i); + byte[] bytes = serialize(b, BaseV1::serializeByte); + Assert.assertEquals(message, 2, bytes.length); + Reader reader = BinaryReaders.newBinaryReader(bytes); + int code = reader.readByte(); + Assert.assertEquals(BaseV1.BYTE, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(message, b, BaseV1.deserializeByte(reader)); + } + } + + @Test + public void testDecimalSerialization() + { + for (String decimalString : new String[]{"-9876543210123456789098765432101234567890.9876543210123456789098765432101234567890", + BigDecimal.valueOf(Double.MIN_VALUE).toPlainString(), Long.MIN_VALUE + ".0", Integer.MIN_VALUE + ".0", "-3.14159265358979323846264338327950288419716939937510", "-2", "-1.0", "-1.00000", + "0.0", "0.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "0.000000", "1.0", "2", "3.14159265358979323846264338327950288419716939937510", Integer.MAX_VALUE + ".0", Long.MAX_VALUE + ".0", BigDecimal.valueOf(Double.MAX_VALUE).toPlainString() + ".0", + "9876543210123456789098765432101234567890.9876543210123456789098765432101234567890"}) + { + byte[] serialized = serialize(this.modelRepository.newDecimalCoreInstance(decimalString), BaseV1::serializeDecimal); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(BaseV1.NUMBER, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(BaseV1.DECIMAL_TYPE, code & BaseV1.NUMBER_TYPE_MASK); + Assert.assertEquals(decimalString, BaseV1.deserializeDecimal(reader, code)); + } + } + + @Test + public void testFloatSerialization() + { + for (String floatString : new String[]{BigDecimal.valueOf(Double.MIN_VALUE).toPlainString(), Long.MIN_VALUE + ".0", Integer.MIN_VALUE + ".0", "-3.14159265358979", "-1.0", "0.0", + "1.0", "3.14159265358979", Integer.MAX_VALUE + ".0", Long.MAX_VALUE + ".0", BigDecimal.valueOf(Double.MAX_VALUE).toPlainString() + ".0"}) + { + byte[] serialized = serialize(this.modelRepository.newFloatCoreInstance(floatString), BaseV1::serializeFloat); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(BaseV1.NUMBER, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(BaseV1.FLOAT_TYPE, code & BaseV1.NUMBER_TYPE_MASK); + Assert.assertEquals(floatString, BaseV1.deserializeFloat(reader, code)); + } + } + + @Test + public void testIntegerSerialization() + { + for (int i : new int[]{Integer.MIN_VALUE, -1_000_000, -256, -1, 0, 1, 357, 1_000_000, Integer.MAX_VALUE}) + { + byte[] serialized = serialize(this.modelRepository.newIntegerCoreInstance(i), BaseV1::serializeInteger); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(BaseV1.NUMBER, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(BaseV1.INTEGER_TYPE, code & BaseV1.NUMBER_TYPE_MASK); + Assert.assertEquals(i, BaseV1.deserializeInteger(reader, code)); + } + for (long l : new long[]{Long.MIN_VALUE, Integer.MIN_VALUE * 17L, Integer.MAX_VALUE * 17L, Long.MAX_VALUE}) + { + byte[] serialized = serialize(this.modelRepository.newIntegerCoreInstance(l), BaseV1::serializeInteger); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(BaseV1.NUMBER, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(BaseV1.INTEGER_TYPE, code & BaseV1.NUMBER_TYPE_MASK); + Assert.assertEquals(l, BaseV1.deserializeInteger(reader, code)); + } + for (String integerString : new String[]{"-123456789098765432101234567890987654321", "9876543210123456789098765432101234567890"}) + { + byte[] serialized = serialize(this.modelRepository.newIntegerCoreInstance(integerString), BaseV1::serializeInteger); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(BaseV1.NUMBER, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(BaseV1.BIG_INTEGER_TYPE, code & BaseV1.NUMBER_TYPE_MASK); + Assert.assertEquals(new BigInteger(integerString), BaseV1.deserializeInteger(reader, code)); + } + } + + @Test + public void testDateSerialization() + { + for (PureDate date : new PureDate[]{DateFunctions.newPureDate(2024), DateFunctions.newPureDate(1835), DateFunctions.newPureDate(2024, 12), DateFunctions.newPureDate(1220, 4)}) + { + String message = date.toString(); + byte[] serialized = serialize(this.modelRepository.newDateCoreInstance(date), BaseV1::serializeDate); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(message, BaseV1.DATE, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(message, BaseV1.DATE_TYPE, code & BaseV1.DATE_TYPE_MASK); + Assert.assertEquals(message, date.hasMonth() ? BaseV1.MONTH_DATE_WIDTH : BaseV1.YEAR_DATE_WIDTH, code & BaseV1.DATE_WIDTH_MASK); + Assert.assertEquals(message, date, BaseV1.deserializeDate(reader, code)); + } + } + + @Test + public void testDateTimeSerialization() + { + for (PureDate date : new PureDate[]{DateFunctions.newPureDate(2024, 12, 1, 17, 2, 35, "123456789"), DateFunctions.newPureDate(1900, 1, 1, 0), DateFunctions.newPureDate(1835, 4, 7, 0, 1), DateFunctions.newPureDate(2011, 1, 10, 11, 58, 0, "000000000"), DateFunctions.newPureDate(1220, 8, 28, 23, 59, 59, "999999999999999999999999999999999999999999999999999999999")}) + { + String message = date.toString(); + byte[] serialized = serialize(this.modelRepository.newDateCoreInstance(date), BaseV1::serializeDateTime); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(message, BaseV1.DATE, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(message, BaseV1.DATE_TIME_TYPE, code & BaseV1.DATE_TYPE_MASK); + Assert.assertEquals(message, date, BaseV1.deserializeDate(reader, code)); + } + } + + @Test + public void testStrictDateSerialization() + { + for (PureDate date : new PureDate[]{DateFunctions.newPureDate(2024, 12, 1), DateFunctions.newPureDate(1835, 4, 7), DateFunctions.newPureDate(2011, 1, 10), DateFunctions.newPureDate(1220, 8, 28)}) + { + String message = date.toString(); + byte[] serialized = serialize(this.modelRepository.newDateCoreInstance(date), BaseV1::serializeStrictDate); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(message, BaseV1.DATE, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(message, BaseV1.STRICT_DATE_TYPE, code & BaseV1.DATE_TYPE_MASK); + Assert.assertEquals(message, BaseV1.DAY_DATE_WIDTH, code & BaseV1.DATE_WIDTH_MASK); + Assert.assertEquals(message, date, BaseV1.deserializeDate(reader, code)); + } + } + + @Test + public void testLatestDateSerialization() + { + byte[] serialized = serialize(BaseV1::serializeLatestDate); + Assert.assertEquals(1, serialized.length); + int code = serialized[0]; + Assert.assertEquals(BaseV1.DATE, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(BaseV1.LATEST_DATE_TYPE, code & BaseV1.DATE_TYPE_MASK); + Assert.assertNull(BaseV1.deserializeDate(BinaryReaders.newBinaryReader(new byte[0]), code)); + } + + @Test + public void testStrictTimeSerialization() + { + for (int h = 0; h < 24; h++) + { + for (int m = 0; m < 60; m++) + { + for (int s = 0; s < 60; s++) + { + for (String subsecond : new String[]{"0", "000", "000000", "000000000", "123456789", "1", "2", "3", "9876543210123456789"}) + { + PureStrictTime time = StrictTimeFunctions.newPureStrictTime(h, m, s, subsecond); + byte[] serialized = serialize(this.modelRepository.newStrictTimeCoreInstance(time), BaseV1::serializeStrictTime); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(BaseV1.STRICT_TIME, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(BaseV1.SUBSECOND_DATE_WIDTH, code & BaseV1.DATE_WIDTH_MASK); + Assert.assertEquals(time, BaseV1.deserializeStrictTime(reader, code)); + } + PureStrictTime time = StrictTimeFunctions.newPureStrictTime(h, m, s); + byte[] serialized = serialize(this.modelRepository.newStrictTimeCoreInstance(time), BaseV1::serializeStrictTime); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(BaseV1.STRICT_TIME, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(BaseV1.SECOND_DATE_WIDTH, code & BaseV1.DATE_WIDTH_MASK); + Assert.assertEquals(time, BaseV1.deserializeStrictTime(reader, code)); + } + PureStrictTime time = StrictTimeFunctions.newPureStrictTime(h, m); + byte[] serialized = serialize(this.modelRepository.newStrictTimeCoreInstance(time), BaseV1::serializeStrictTime); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(BaseV1.STRICT_TIME, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(BaseV1.MINUTE_DATE_WIDTH, code & BaseV1.DATE_WIDTH_MASK); + Assert.assertEquals(time, BaseV1.deserializeStrictTime(reader, code)); + } + } + } + + @Test + public void testStringSerialization() + { + for (String string : new String[]{"The Quick Brown Fox Jumped Over The Lazy Dog", "", "a", "AbCdEfGhIjKlMnOpQrStUvWxYz0987654321!@#$%^&*()"}) + { + byte[] serialized = serialize(this.modelRepository.newStringCoreInstance(string), BaseV1::serializeString); + Reader reader = BinaryReaders.newBinaryReader(serialized); + int code = reader.readByte(); + Assert.assertEquals(BaseV1.STRING, code & BaseV1.NODE_TYPE_MASK); + Assert.assertEquals(string, BaseV1.deserializeString(reader)); + } + } + + private byte[] serialize(T value, BiConsumer serializer) + { + return serialize(w -> serializer.accept(w, value)); + } + + private byte[] serialize(Consumer serializer) + { + this.stream.reset(); + serializer.accept(BinaryWriters.newBinaryWriter(this.stream)); + return this.stream.toByteArray(); + } +}