From dc37b13e46dac0211c4d70fdde3d223a58bbab31 Mon Sep 17 00:00:00 2001 From: Vincent Guilpain Date: Thu, 30 Jan 2025 10:58:51 +0900 Subject: [PATCH] Change DATE type encoding to int --- .../db/storage/cosmos/ResultInterpreter.java | 3 +- .../db/storage/dynamo/ResultInterpreter.java | 2 +- .../db/storage/dynamo/bytes/BytesUtils.java | 12 +++++-- .../dynamo/bytes/DateBytesEncoder.java | 8 ++--- .../storage/dynamo/bytes/IntBytesEncoder.java | 7 ++-- .../db/storage/jdbc/RdbEngineSqlite.java | 8 ++--- .../storage/jdbc/RdbEngineTimeTypeSqlite.java | 5 +-- .../util/TimeRelatedColumnEncodingUtils.java | 8 ++--- .../TimeRelatedColumnEncodingUtilsTest.java | 34 +++++++++++++++++-- .../scalar/db/storage/jdbc/JdbcAdminTest.java | 4 +-- 10 files changed, 63 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/com/scalar/db/storage/cosmos/ResultInterpreter.java b/core/src/main/java/com/scalar/db/storage/cosmos/ResultInterpreter.java index 95c546ffa5..2b15439631 100644 --- a/core/src/main/java/com/scalar/db/storage/cosmos/ResultInterpreter.java +++ b/core/src/main/java/com/scalar/db/storage/cosmos/ResultInterpreter.java @@ -98,8 +98,7 @@ private Column convert(@Nullable Object recordValue, String name, DataType da return recordValue == null ? DateColumn.ofNull(name) : DateColumn.of( - name, - TimeRelatedColumnEncodingUtils.decodeDate(((Number) recordValue).longValue())); + name, TimeRelatedColumnEncodingUtils.decodeDate(((Number) recordValue).intValue())); case TIME: return recordValue == null ? TimeColumn.ofNull(name) diff --git a/core/src/main/java/com/scalar/db/storage/dynamo/ResultInterpreter.java b/core/src/main/java/com/scalar/db/storage/dynamo/ResultInterpreter.java index d5b724d208..7c71a831fe 100644 --- a/core/src/main/java/com/scalar/db/storage/dynamo/ResultInterpreter.java +++ b/core/src/main/java/com/scalar/db/storage/dynamo/ResultInterpreter.java @@ -81,7 +81,7 @@ private Column convert(@Nullable AttributeValue itemValue, String name, DataT return isNull ? DateColumn.ofNull(name) : DateColumn.of( - name, TimeRelatedColumnEncodingUtils.decodeDate(Long.parseLong(itemValue.n()))); + name, TimeRelatedColumnEncodingUtils.decodeDate(Integer.parseInt(itemValue.n()))); case TIME: return isNull ? TimeColumn.ofNull(name) diff --git a/core/src/main/java/com/scalar/db/storage/dynamo/bytes/BytesUtils.java b/core/src/main/java/com/scalar/db/storage/dynamo/bytes/BytesUtils.java index 4d32f75154..4e24c800fe 100644 --- a/core/src/main/java/com/scalar/db/storage/dynamo/bytes/BytesUtils.java +++ b/core/src/main/java/com/scalar/db/storage/dynamo/bytes/BytesUtils.java @@ -112,8 +112,8 @@ public static byte[] toBytes(ByteBuffer src) { } static void encodeLong(long value, Order order, ByteBuffer dst) { - dst.put( - mask((byte) ((value >> 56) ^ 0x80), order)); // Flip a sign bit to make it binary comparable + // Flip a sign bit to make it binary comparable + dst.put(mask((byte) ((value >> 56) ^ 0x80), order)); dst.put(mask((byte) (value >> 48), order)); dst.put(mask((byte) (value >> 40), order)); dst.put(mask((byte) (value >> 32), order)); @@ -122,4 +122,12 @@ static void encodeLong(long value, Order order, ByteBuffer dst) { dst.put(mask((byte) (value >> 8), order)); dst.put(mask((byte) value, order)); } + + static void encodeInt(int value, Order order, ByteBuffer dst) { + // Flip a sign bit to make it binary comparable + dst.put(mask((byte) ((value >> 24) ^ 0x80), order)); + dst.put(mask((byte) (value >> 16), order)); + dst.put(mask((byte) (value >> 8), order)); + dst.put(mask((byte) value, order)); + } } diff --git a/core/src/main/java/com/scalar/db/storage/dynamo/bytes/DateBytesEncoder.java b/core/src/main/java/com/scalar/db/storage/dynamo/bytes/DateBytesEncoder.java index 7dd8c93f3e..eca7a401ca 100644 --- a/core/src/main/java/com/scalar/db/storage/dynamo/bytes/DateBytesEncoder.java +++ b/core/src/main/java/com/scalar/db/storage/dynamo/bytes/DateBytesEncoder.java @@ -1,6 +1,6 @@ package com.scalar.db.storage.dynamo.bytes; -import static com.scalar.db.storage.dynamo.bytes.BytesUtils.encodeLong; +import static com.scalar.db.storage.dynamo.bytes.BytesUtils.encodeInt; import com.scalar.db.api.Scan.Ordering.Order; import com.scalar.db.io.DateColumn; @@ -17,14 +17,14 @@ public class DateBytesEncoder implements BytesEncoder { public int encodedLength(DateColumn column, Order order) { assert column.getValue().isPresent(); - return 8; + return 4; } @Override public void encode(DateColumn column, Order order, ByteBuffer dst) { assert column.getValue().isPresent(); - long value = TimeRelatedColumnEncodingUtils.encode(column); - encodeLong(value, order, dst); + int value = TimeRelatedColumnEncodingUtils.encode(column); + encodeInt(value, order, dst); } } diff --git a/core/src/main/java/com/scalar/db/storage/dynamo/bytes/IntBytesEncoder.java b/core/src/main/java/com/scalar/db/storage/dynamo/bytes/IntBytesEncoder.java index 384d732d0f..22a9adc344 100644 --- a/core/src/main/java/com/scalar/db/storage/dynamo/bytes/IntBytesEncoder.java +++ b/core/src/main/java/com/scalar/db/storage/dynamo/bytes/IntBytesEncoder.java @@ -1,6 +1,6 @@ package com.scalar.db.storage.dynamo.bytes; -import static com.scalar.db.storage.dynamo.bytes.BytesUtils.mask; +import static com.scalar.db.storage.dynamo.bytes.BytesUtils.encodeInt; import com.scalar.db.api.Scan.Ordering.Order; import com.scalar.db.io.IntColumn; @@ -22,9 +22,6 @@ public void encode(IntColumn column, Order order, ByteBuffer dst) { assert !column.hasNullValue(); int v = column.getIntValue(); - dst.put(mask((byte) ((v >> 24) ^ 0x80), order)); // Flip a sign bit to make it binary comparable - dst.put(mask((byte) (v >> 16), order)); - dst.put(mask((byte) (v >> 8), order)); - dst.put(mask((byte) v, order)); + encodeInt(v, order, dst); } } diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineSqlite.java b/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineSqlite.java index fb436751c4..0e2c77262e 100644 --- a/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineSqlite.java +++ b/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineSqlite.java @@ -95,9 +95,9 @@ public String getDataTypeForEngine(DataType scalarDbDataType) { case BOOLEAN: return "BOOLEAN"; case INT: + case DATE: return "INT"; case BIGINT: - case DATE: case TIME: case TIMESTAMP: case TIMESTAMPTZ: @@ -126,8 +126,8 @@ public int getSqlTypes(DataType dataType) { case BOOLEAN: return Types.BOOLEAN; case INT: - return Types.INTEGER; case DATE: + return Types.INTEGER; case TIME: case TIMESTAMP: case TIMESTAMPTZ: @@ -307,7 +307,7 @@ public String tryAddIfNotExistsToCreateIndexSql(String createIndexSql) { @Override public DateColumn parseDateColumn(ResultSet resultSet, String columnName) throws SQLException { return DateColumn.of( - columnName, TimeRelatedColumnEncodingUtils.decodeDate(resultSet.getLong(columnName))); + columnName, TimeRelatedColumnEncodingUtils.decodeDate(resultSet.getInt(columnName))); } @Override @@ -332,7 +332,7 @@ public TimestampTZColumn parseTimestampTZColumn(ResultSet resultSet, String colu } @Override - public RdbEngineTimeTypeStrategy getTimeTypeStrategy() { + public RdbEngineTimeTypeStrategy getTimeTypeStrategy() { return timeTypeEngine; } } diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineTimeTypeSqlite.java b/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineTimeTypeSqlite.java index c8f3d26e39..0e03197edc 100644 --- a/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineTimeTypeSqlite.java +++ b/core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineTimeTypeSqlite.java @@ -6,10 +6,11 @@ import java.time.LocalTime; import java.time.OffsetDateTime; -public class RdbEngineTimeTypeSqlite implements RdbEngineTimeTypeStrategy { +public class RdbEngineTimeTypeSqlite + implements RdbEngineTimeTypeStrategy { @Override - public Long convert(LocalDate date) { + public Integer convert(LocalDate date) { return TimeRelatedColumnEncodingUtils.encode(date); } diff --git a/core/src/main/java/com/scalar/db/util/TimeRelatedColumnEncodingUtils.java b/core/src/main/java/com/scalar/db/util/TimeRelatedColumnEncodingUtils.java index cbdbd090b5..0bf09e9510 100644 --- a/core/src/main/java/com/scalar/db/util/TimeRelatedColumnEncodingUtils.java +++ b/core/src/main/java/com/scalar/db/util/TimeRelatedColumnEncodingUtils.java @@ -17,16 +17,16 @@ public final class TimeRelatedColumnEncodingUtils { private TimeRelatedColumnEncodingUtils() {} - public static long encode(DateColumn column) { + public static int encode(DateColumn column) { assert column.getDateValue() != null; return encode(column.getDateValue()); } - public static long encode(LocalDate date) { - return date.toEpochDay(); + public static int encode(LocalDate date) { + return Math.toIntExact(date.toEpochDay()); } - public static LocalDate decodeDate(long epochDay) { + public static LocalDate decodeDate(int epochDay) { return LocalDate.ofEpochDay(epochDay); } diff --git a/core/src/test/java/com/scalar/db/storage/TimeRelatedColumnEncodingUtilsTest.java b/core/src/test/java/com/scalar/db/storage/TimeRelatedColumnEncodingUtilsTest.java index b226cf1cf5..69ce5ad478 100644 --- a/core/src/test/java/com/scalar/db/storage/TimeRelatedColumnEncodingUtilsTest.java +++ b/core/src/test/java/com/scalar/db/storage/TimeRelatedColumnEncodingUtilsTest.java @@ -26,12 +26,27 @@ public void encodeDate_ShouldWorkProperly() { DateColumn column = DateColumn.of("date", LocalDate.of(2023, 10, 1)); // Act - long encoded = TimeRelatedColumnEncodingUtils.encode(column); + int encoded = TimeRelatedColumnEncodingUtils.encode(column); // Assert assertThat(encoded).isEqualTo(LocalDate.of(2023, 10, 1).toEpochDay()); } + @Test + public void encodeThenDecodeDate_ShouldPreserveDataIntegrity() { + // Arrange + DateColumn min = DateColumn.of("date", DateColumn.MIN_VALUE); + DateColumn max = DateColumn.of("date", DateColumn.MAX_VALUE); + + // Act Assert + assertThat( + TimeRelatedColumnEncodingUtils.decodeDate(TimeRelatedColumnEncodingUtils.encode(min))) + .isEqualTo(min.getDateValue()); + assertThat( + TimeRelatedColumnEncodingUtils.decodeDate(TimeRelatedColumnEncodingUtils.encode(max))) + .isEqualTo(max.getDateValue()); + } + @Test public void encodeTime_ShouldWorkProperly() { // Arrange @@ -44,6 +59,21 @@ public void encodeTime_ShouldWorkProperly() { assertThat(encoded).isEqualTo(LocalTime.of(12, 34, 56, 123_456_000).toNanoOfDay()); } + @Test + public void encodeThenDecodeTime_ShouldPreserveDataIntegrity() { + // Arrange + TimeColumn min = TimeColumn.of("time", TimeColumn.MIN_VALUE); + TimeColumn max = TimeColumn.of("time", TimeColumn.MAX_VALUE); + + // Act Assert + assertThat( + TimeRelatedColumnEncodingUtils.decodeTime(TimeRelatedColumnEncodingUtils.encode(min))) + .isEqualTo(min.getTimeValue()); + assertThat( + TimeRelatedColumnEncodingUtils.decodeTime(TimeRelatedColumnEncodingUtils.encode(max))) + .isEqualTo(max.getTimeValue()); + } + @Test public void encodeTimestamp_ShouldWorkProperly() { // Arrange @@ -124,7 +154,7 @@ public void encodeTimestampTZ_ShouldWorkProperly() { public void decodeDate_ShouldWorkProperly() { // Arrange Act LocalDate date = - TimeRelatedColumnEncodingUtils.decodeDate(LocalDate.of(2023, 10, 1).toEpochDay()); + TimeRelatedColumnEncodingUtils.decodeDate((int) LocalDate.of(2023, 10, 1).toEpochDay()); // Assert assertThat(date).isEqualTo(LocalDate.of(2023, 10, 1)); diff --git a/core/src/test/java/com/scalar/db/storage/jdbc/JdbcAdminTest.java b/core/src/test/java/com/scalar/db/storage/jdbc/JdbcAdminTest.java index 925812e26f..d3ea629a59 100644 --- a/core/src/test/java/com/scalar/db/storage/jdbc/JdbcAdminTest.java +++ b/core/src/test/java/com/scalar/db/storage/jdbc/JdbcAdminTest.java @@ -550,7 +550,7 @@ public void createTableInternal_ForOracle_ShouldCreateTableAndIndexes() throws S public void createTableInternal_ForSqlite_ShouldCreateTableAndIndexes() throws SQLException { createTableInternal_ForX_CreateTableAndIndexes( RdbEngine.SQLITE, - "CREATE TABLE \"my_ns$foo_table\"(\"c3\" BOOLEAN,\"c1\" TEXT,\"c4\" BLOB,\"c2\" BIGINT,\"c5\" INT,\"c6\" DOUBLE,\"c7\" FLOAT,\"c8\" BIGINT,\"c9\" BIGINT,\"c10\" BIGINT,\"c11\" BIGINT, PRIMARY KEY (\"c3\",\"c1\",\"c4\"))", + "CREATE TABLE \"my_ns$foo_table\"(\"c3\" BOOLEAN,\"c1\" TEXT,\"c4\" BLOB,\"c2\" BIGINT,\"c5\" INT,\"c6\" DOUBLE,\"c7\" FLOAT,\"c8\" INT,\"c9\" BIGINT,\"c10\" BIGINT,\"c11\" BIGINT, PRIMARY KEY (\"c3\",\"c1\",\"c4\"))", "CREATE INDEX \"index_my_ns_foo_table_c4\" ON \"my_ns$foo_table\" (\"c4\")", "CREATE INDEX \"index_my_ns_foo_table_c1\" ON \"my_ns$foo_table\" (\"c1\")"); } @@ -655,7 +655,7 @@ public void createTableInternal_IfNotExistsForSqlite_ShouldCreateTableAndIndexes throws SQLException { createTableInternal_IfNotExistsForX_createTableAndIndexesIfNotExists( RdbEngine.SQLITE, - "CREATE TABLE IF NOT EXISTS \"my_ns$foo_table\"(\"c3\" BOOLEAN,\"c1\" TEXT,\"c4\" BLOB,\"c2\" BIGINT,\"c5\" INT,\"c6\" DOUBLE,\"c7\" FLOAT,\"c8\" BIGINT,\"c9\" BIGINT,\"c10\" BIGINT,\"c11\" BIGINT, PRIMARY KEY (\"c3\",\"c1\",\"c4\"))", + "CREATE TABLE IF NOT EXISTS \"my_ns$foo_table\"(\"c3\" BOOLEAN,\"c1\" TEXT,\"c4\" BLOB,\"c2\" BIGINT,\"c5\" INT,\"c6\" DOUBLE,\"c7\" FLOAT,\"c8\" INT,\"c9\" BIGINT,\"c10\" BIGINT,\"c11\" BIGINT, PRIMARY KEY (\"c3\",\"c1\",\"c4\"))", "CREATE INDEX IF NOT EXISTS \"index_my_ns_foo_table_c4\" ON \"my_ns$foo_table\" (\"c4\")", "CREATE INDEX IF NOT EXISTS \"index_my_ns_foo_table_c1\" ON \"my_ns$foo_table\" (\"c1\")"); }