diff --git a/catalog/format/iceberg-fixturegen/src/main/java/org/projectnessie/catalog/formats/iceberg/fixtures/IcebergFixtures.java b/catalog/format/iceberg-fixturegen/src/main/java/org/projectnessie/catalog/formats/iceberg/fixtures/IcebergFixtures.java index 7b88da773c9..9cf5b32b454 100644 --- a/catalog/format/iceberg-fixturegen/src/main/java/org/projectnessie/catalog/formats/iceberg/fixtures/IcebergFixtures.java +++ b/catalog/format/iceberg-fixturegen/src/main/java/org/projectnessie/catalog/formats/iceberg/fixtures/IcebergFixtures.java @@ -39,8 +39,10 @@ import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.stringType; import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.structType; import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timeType; +import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampNanosType; +import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampNanosTzType; import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampType; -import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestamptzType; +import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampTzType; import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.uuidType; import java.util.Iterator; @@ -98,7 +100,9 @@ public static Stream icebergTypes(IntSupplier idSupplier) { decimalType(10, 3), fixedType(42), timestampType(), - timestamptzType()); + timestampTzType(), + timestampNanosType(), + timestampNanosTzType()); } public static IcebergSchema icebergSchemaAllTypes() { diff --git a/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/nessie/NessieModelIceberg.java b/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/nessie/NessieModelIceberg.java index 88fe5ad6403..37fa7c99d5f 100644 --- a/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/nessie/NessieModelIceberg.java +++ b/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/nessie/NessieModelIceberg.java @@ -36,7 +36,8 @@ import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.TYPE_MAP; import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.TYPE_STRUCT; import static org.projectnessie.catalog.model.id.NessieId.transientNessieId; -import static org.projectnessie.catalog.model.schema.types.NessieType.DEFAULT_TIME_PRECISION; +import static org.projectnessie.catalog.model.schema.types.NessieType.MICROS_TIME_PRECISION; +import static org.projectnessie.catalog.model.schema.types.NessieType.NANOS_TIME_PRECISION; import static org.projectnessie.catalog.model.snapshot.NessieViewRepresentation.NessieViewSQLRepresentation.nessieViewSQLRepresentation; import static org.projectnessie.catalog.model.snapshot.TableFormat.ICEBERG; import static org.projectnessie.catalog.model.statistics.NessiePartitionStatisticsFile.partitionStatisticsFile; @@ -295,18 +296,23 @@ public static IcebergType nessieTypeToIcebergType(NessieTypeSpec type) { return IcebergType.dateType(); case TIME: NessieTimeTypeSpec time = (NessieTimeTypeSpec) type; - if (time.precision() != DEFAULT_TIME_PRECISION || time.withTimeZone()) { + if (time.precision() != MICROS_TIME_PRECISION || time.withTimeZone()) { throw new IllegalArgumentException("Data type not supported in Iceberg: " + type); } return IcebergType.timeType(); case TIMESTAMP: NessieTimestampTypeSpec timestamp = (NessieTimestampTypeSpec) type; - if (timestamp.precision() != DEFAULT_TIME_PRECISION) { - throw new IllegalArgumentException("Data type not supported in Iceberg: " + type); + switch (timestamp.precision()) { + case MICROS_TIME_PRECISION: + return timestamp.withTimeZone() + ? IcebergType.timestampTzType() + : IcebergType.timestampType(); + case NANOS_TIME_PRECISION: + return timestamp.withTimeZone() + ? IcebergType.timestampNanosTzType() + : IcebergType.timestampNanosType(); } - return timestamp.withTimeZone() - ? IcebergType.timestamptzType() - : IcebergType.timestampType(); + throw new IllegalArgumentException("Data type not supported in Iceberg: " + type); case BINARY: return IcebergType.binaryType(); case DECIMAL: @@ -357,9 +363,13 @@ static NessieTypeSpec icebergTypeToNessieType( IcebergType type, Map icebergFields) { switch (type.type()) { case IcebergType.TYPE_TIMESTAMP_TZ: - return NessieType.timestampType(true); + return NessieType.timestampType(MICROS_TIME_PRECISION, true); case IcebergType.TYPE_TIMESTAMP: - return NessieType.timestampType(false); + return NessieType.timestampType(MICROS_TIME_PRECISION, false); + case IcebergType.TYPE_TIMESTAMP_NS_TZ: + return NessieType.timestampType(NANOS_TIME_PRECISION, true); + case IcebergType.TYPE_TIMESTAMP_NS: + return NessieType.timestampType(NANOS_TIME_PRECISION, false); case IcebergType.TYPE_BOOLEAN: return NessieType.booleanType(); case IcebergType.TYPE_UUID: diff --git a/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTimestampNanosType.java b/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTimestampNanosType.java new file mode 100644 index 00000000000..a7827000407 --- /dev/null +++ b/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTimestampNanosType.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.catalog.formats.iceberg.types; + +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; + +/** Iceberg timestamp, nanosecond precision. */ +public final class IcebergTimestampNanosType extends IcebergPrimitiveType { + private static final Schema TIMESTAMP_NS_SCHEMA = + LogicalTypes.timestampNanos().addToSchema(Schema.create(Schema.Type.LONG)); + private static final Schema TIMESTAMPTZ_NS_SCHEMA = + LogicalTypes.timestampNanos().addToSchema(Schema.create(Schema.Type.LONG)); + public static final String ADJUST_TO_UTC_PROP = "adjust-to-utc"; + + static { + TIMESTAMP_NS_SCHEMA.addProp(ADJUST_TO_UTC_PROP, false); + TIMESTAMPTZ_NS_SCHEMA.addProp(ADJUST_TO_UTC_PROP, true); + } + + private final boolean adjustToUTC; + + IcebergTimestampNanosType(boolean adjustToUTC) { + this.adjustToUTC = adjustToUTC; + } + + public boolean adjustToUTC() { + return adjustToUTC; + } + + @Override + public String type() { + return adjustToUTC() ? TYPE_TIMESTAMP_NS_TZ : TYPE_TIMESTAMP_NS; + } + + @Override + public Schema avroSchema(int fieldId) { + return adjustToUTC() ? TIMESTAMPTZ_NS_SCHEMA : TIMESTAMP_NS_SCHEMA; + } + + @Override + public byte[] serializeSingleValue(Object value) { + return IcebergLongType.serializeLong((Long) value); + } + + @Override + public Object deserializeSingleValue(byte[] value) { + return IcebergLongType.deserializeLong(value); + } + + @Override + public int hashCode() { + return type().hashCode() ^ (adjustToUTC ? 1 : 0); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof IcebergTimestampNanosType)) { + return false; + } + if (obj == this) { + return true; + } + IcebergTimestampNanosType o = (IcebergTimestampNanosType) obj; + return o.adjustToUTC == adjustToUTC; + } +} diff --git a/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTimestampType.java b/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTimestampType.java index 73f59fa4380..075411d8fa5 100644 --- a/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTimestampType.java +++ b/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTimestampType.java @@ -18,6 +18,7 @@ import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; +/** Iceberg timestamp, microsecond precision. */ public final class IcebergTimestampType extends IcebergPrimitiveType { private static final Schema TIMESTAMP_SCHEMA = LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG)); diff --git a/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergType.java b/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergType.java index 00a76138cc4..ea1be54a7c5 100644 --- a/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergType.java +++ b/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergType.java @@ -43,6 +43,8 @@ public interface IcebergType { String TYPE_BINARY = "binary"; String TYPE_TIMESTAMP = "timestamp"; String TYPE_TIMESTAMP_TZ = "timestamptz"; + String TYPE_TIMESTAMP_NS = "timestamp_ns"; + String TYPE_TIMESTAMP_NS_TZ = "timestamptz_ns"; String TYPE_FIXED = "fixed"; String TYPE_DECIMAL = "decimal"; String TYPE_STRUCT = "struct"; @@ -89,7 +91,7 @@ static IcebergBinaryType binaryType() { return IcebergTypes.BINARY; } - static IcebergTimestampType timestamptzType() { + static IcebergTimestampType timestampTzType() { return IcebergTypes.TIMESTAMPTZ; } @@ -97,6 +99,14 @@ static IcebergTimestampType timestampType() { return IcebergTypes.TIMESTAMP; } + static IcebergTimestampNanosType timestampNanosTzType() { + return IcebergTypes.TIMESTAMPTZ_NS; + } + + static IcebergTimestampNanosType timestampNanosType() { + return IcebergTypes.TIMESTAMP_NS; + } + static IcebergFixedType fixedType(int length) { return ImmutableIcebergFixedType.of(length); } diff --git a/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTypes.java b/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTypes.java index 4ad996fcd25..77c1cfe1556 100644 --- a/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTypes.java +++ b/catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTypes.java @@ -45,35 +45,41 @@ final class IcebergTypes { static final IcebergBinaryType BINARY = new IcebergBinaryType(); static final IcebergTimestampType TIMESTAMPTZ = new IcebergTimestampType(true); static final IcebergTimestampType TIMESTAMP = new IcebergTimestampType(false); + static final IcebergTimestampNanosType TIMESTAMPTZ_NS = new IcebergTimestampNanosType(true); + static final IcebergTimestampNanosType TIMESTAMP_NS = new IcebergTimestampNanosType(false); private IcebergTypes() {} static IcebergPrimitiveType primitiveFromString(String primitiveType) { switch (primitiveType) { - case IcebergBooleanType.TYPE_BOOLEAN: + case IcebergType.TYPE_BOOLEAN: return IcebergType.booleanType(); - case IcebergUuidType.TYPE_UUID: + case IcebergType.TYPE_UUID: return IcebergType.uuidType(); - case IcebergIntegerType.TYPE_INT: + case IcebergType.TYPE_INT: return IcebergType.integerType(); - case IcebergLongType.TYPE_LONG: + case IcebergType.TYPE_LONG: return IcebergType.longType(); - case IcebergFloatType.TYPE_FLOAT: + case IcebergType.TYPE_FLOAT: return IcebergType.floatType(); - case IcebergDoubleType.TYPE_DOUBLE: + case IcebergType.TYPE_DOUBLE: return IcebergType.doubleType(); - case IcebergDateType.TYPE_DATE: + case IcebergType.TYPE_DATE: return IcebergType.dateType(); - case IcebergTimeType.TYPE_TIME: + case IcebergType.TYPE_TIME: return IcebergType.timeType(); - case IcebergStringType.TYPE_STRING: + case IcebergType.TYPE_STRING: return IcebergType.stringType(); - case IcebergBinaryType.TYPE_BINARY: + case IcebergType.TYPE_BINARY: return IcebergType.binaryType(); - case IcebergTimestampType.TYPE_TIMESTAMP_TZ: - return IcebergType.timestamptzType(); - case IcebergTimestampType.TYPE_TIMESTAMP: + case IcebergType.TYPE_TIMESTAMP_TZ: + return IcebergType.timestampTzType(); + case IcebergType.TYPE_TIMESTAMP: return IcebergType.timestampType(); + case IcebergType.TYPE_TIMESTAMP_NS_TZ: + return IcebergType.timestampNanosTzType(); + case IcebergType.TYPE_TIMESTAMP_NS: + return IcebergType.timestampNanosType(); default: Matcher m = DECIMAL_PATTERN.matcher(primitiveType); if (m.matches()) { diff --git a/catalog/format/iceberg/src/test/java/org/projectnessie/catalog/formats/iceberg/nessie/TestNessieModelIceberg.java b/catalog/format/iceberg/src/test/java/org/projectnessie/catalog/formats/iceberg/nessie/TestNessieModelIceberg.java index 3bf77ed02b2..5c4d3038b4c 100644 --- a/catalog/format/iceberg/src/test/java/org/projectnessie/catalog/formats/iceberg/nessie/TestNessieModelIceberg.java +++ b/catalog/format/iceberg/src/test/java/org/projectnessie/catalog/formats/iceberg/nessie/TestNessieModelIceberg.java @@ -38,7 +38,7 @@ import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.mapType; import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.stringType; import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.structType; -import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestamptzType; +import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampTzType; import static org.projectnessie.catalog.model.id.NessieIdHasher.nessieIdHasher; import com.fasterxml.jackson.databind.ObjectMapper; @@ -367,7 +367,7 @@ static Stream icebergNested() { nestedField(102, "topic", false, stringType(), null), nestedField(103, "partition", false, integerType(), null), nestedField(104, "offset", false, longType(), null), - nestedField(105, "timestamp", false, timestamptzType(), null), + nestedField(105, "timestamp", false, timestampTzType(), null), nestedField(106, "timestampType", false, integerType(), null), nestedField( 107, @@ -433,7 +433,7 @@ static Stream icebergNested() { nestedField(3, "topic", false, stringType(), null), nestedField(4, "partition", false, integerType(), null), nestedField(5, "offset", false, longType(), null), - nestedField(6, "timestamp", false, timestamptzType(), null), + nestedField(6, "timestamp", false, timestampTzType(), null), nestedField(7, "timestampType", false, integerType(), null), nestedField( 8, diff --git a/catalog/format/iceberg/src/test/java/org/projectnessie/catalog/formats/iceberg/types/TestIcebergTypes.java b/catalog/format/iceberg/src/test/java/org/projectnessie/catalog/formats/iceberg/types/TestIcebergTypes.java index 145d43659b1..0bda019eaa0 100644 --- a/catalog/format/iceberg/src/test/java/org/projectnessie/catalog/formats/iceberg/types/TestIcebergTypes.java +++ b/catalog/format/iceberg/src/test/java/org/projectnessie/catalog/formats/iceberg/types/TestIcebergTypes.java @@ -33,8 +33,10 @@ import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.stringType; import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.structType; import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timeType; +import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampNanosType; +import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampNanosTzType; import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampType; -import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestamptzType; +import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampTzType; import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.uuidType; import java.util.stream.Stream; @@ -75,7 +77,9 @@ static Stream types() { arguments(dateType(), "\"date\""), arguments(timeType(), "\"time\""), arguments(timestampType(), "\"timestamp\""), - arguments(timestamptzType(), "\"timestamptz\""), + arguments(timestampTzType(), "\"timestamptz\""), + arguments(timestampNanosType(), "\"timestamp_ns\""), + arguments(timestampNanosTzType(), "\"timestamptz_ns\""), arguments(uuidType(), "\"uuid\""), arguments(fixedType(42), "\"fixed[42]\""), arguments(decimalType(33, 11), "\"decimal(33, 11)\""), @@ -153,6 +157,8 @@ static Stream icebergTypes() { arguments(Types.DecimalType.of(10, 3), decimalType(10, 3)), arguments(Types.FixedType.ofLength(42), fixedType(42)), arguments(Types.TimestampType.withoutZone(), timestampType()), - arguments(Types.TimestampType.withZone(), timestamptzType())); + arguments(Types.TimestampType.withZone(), timestampTzType()), + arguments(Types.TimestampNanoType.withoutZone(), timestampNanosType()), + arguments(Types.TimestampNanoType.withZone(), timestampNanosTzType())); } } diff --git a/catalog/model/src/main/java/org/projectnessie/catalog/model/schema/types/NessieType.java b/catalog/model/src/main/java/org/projectnessie/catalog/model/schema/types/NessieType.java index 7e034a54e49..d806a32ff4b 100644 --- a/catalog/model/src/main/java/org/projectnessie/catalog/model/schema/types/NessieType.java +++ b/catalog/model/src/main/java/org/projectnessie/catalog/model/schema/types/NessieType.java @@ -45,7 +45,8 @@ public enum NessieType { MAP("map", NessieMapTypeSpec.class), STRUCT("struct", NessieStructTypeSpec.class); - public static final int DEFAULT_TIME_PRECISION = 6; + public static final int MICROS_TIME_PRECISION = 6; + public static final int NANOS_TIME_PRECISION = 9; private final String lowerCaseName; private final Class typeSpec; @@ -85,22 +86,32 @@ public Class typeSpec() { private static final NessieTimeTypeSpec TIME_TYPE = ImmutableNessieTimeTypeSpec.builder() .withTimeZone(false) - .precision(DEFAULT_TIME_PRECISION) + .precision(MICROS_TIME_PRECISION) .build(); private static final NessieTimeTypeSpec TIME_WITH_TZ_TYPE = ImmutableNessieTimeTypeSpec.builder() .withTimeZone(true) - .precision(DEFAULT_TIME_PRECISION) + .precision(MICROS_TIME_PRECISION) .build(); private static final NessieTimestampTypeSpec TIMESTAMP_TYPE = ImmutableNessieTimestampTypeSpec.builder() .withTimeZone(false) - .precision(DEFAULT_TIME_PRECISION) + .precision(MICROS_TIME_PRECISION) .build(); private static final NessieTimestampTypeSpec TIMESTAMP_WITH_TZ_TYPE = ImmutableNessieTimestampTypeSpec.builder() .withTimeZone(true) - .precision(DEFAULT_TIME_PRECISION) + .precision(MICROS_TIME_PRECISION) + .build(); + private static final NessieTimestampTypeSpec TIMESTAMP_NS_TYPE = + ImmutableNessieTimestampTypeSpec.builder() + .withTimeZone(false) + .precision(NANOS_TIME_PRECISION) + .build(); + private static final NessieTimestampTypeSpec TIMESTAMP_NS_WITH_TZ_TYPE = + ImmutableNessieTimestampTypeSpec.builder() + .withTimeZone(true) + .precision(NANOS_TIME_PRECISION) .build(); private static final NessieIntervalTypeSpec INTERVAL_TYPE = ImmutableNessieIntervalTypeSpec.builder().build(); @@ -123,6 +134,8 @@ public Class typeSpec() { DATE_TYPE, TIMESTAMP_TYPE, TIMESTAMP_WITH_TZ_TYPE, + TIMESTAMP_NS_TYPE, + TIMESTAMP_NS_WITH_TZ_TYPE, INTERVAL_TYPE)); public static List primitiveTypes() { @@ -190,7 +203,7 @@ public static NessieTimeTypeSpec timeType(boolean withTimeZone) { } public static NessieTimeTypeSpec timeType(int precision, boolean withTimeZone) { - return precision == DEFAULT_TIME_PRECISION + return precision == MICROS_TIME_PRECISION ? timeType(withTimeZone) : ImmutableNessieTimeTypeSpec.of(precision, withTimeZone); } @@ -199,10 +212,19 @@ public static NessieTimestampTypeSpec timestampType(boolean withTimeZone) { return withTimeZone ? TIMESTAMP_WITH_TZ_TYPE : TIMESTAMP_TYPE; } + public static NessieTimestampTypeSpec timestampNanosType(boolean withTimeZone) { + return withTimeZone ? TIMESTAMP_NS_WITH_TZ_TYPE : TIMESTAMP_NS_TYPE; + } + public static NessieTimestampTypeSpec timestampType(int precision, boolean withTimeZone) { - return precision == DEFAULT_TIME_PRECISION - ? timestampType(withTimeZone) - : ImmutableNessieTimestampTypeSpec.of(precision, withTimeZone); + switch (precision) { + case MICROS_TIME_PRECISION: + return timestampType(withTimeZone); + case NANOS_TIME_PRECISION: + return timestampNanosType(withTimeZone); + default: + return ImmutableNessieTimestampTypeSpec.of(precision, withTimeZone); + } } public static NessieIntervalTypeSpec intervalType() {