Bugfix: Duration nanos INT to BIGINT (complying with MMS) (#63)

* bugfix: set nanos in Duration to BIGINT MC type to comply with MMS * bump version * checkstyle
goto · Feb 3, 2025 · 727d83e · 727d83e
1 parent d09ba65
commit 727d83e
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 13 deletions.
diff --git a/build.gradle b/build.gradle
@@ -22,7 +22,7 @@ plugins {
 }
 
 group 'com.gotocompany'
-version '0.10.5'
+version '0.10.6'
 
 repositories {
     mavenLocal()

diff --git a/.../java/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverter.java b/.../java/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverter.java
@@ -20,7 +20,7 @@ public class DurationProtobufMaxComputeConverter implements ProtobufMaxComputeCo
     private static final String SECONDS = "seconds";
     private static final String NANOS = "nanos";
     private static final List<String> FIELD_NAMES = Arrays.asList(SECONDS, NANOS);
-    private static final List<TypeInfo> TYPE_INFOS = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.INT);
+    private static final List<TypeInfo> TYPE_INFOS = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.BIGINT);
 
     @Override
     public TypeInfo convertSingularTypeInfo(Descriptors.FieldDescriptor fieldDescriptor) {
@@ -35,8 +35,9 @@ public Object convertSingularPayload(ProtoPayload protoPayload) {
 
     private static List<Object> getValues(Message durationMessage) {
         List<Object> values = new ArrayList<>();
+        Integer nanos = (Integer) durationMessage.getField(durationMessage.getDescriptorForType().findFieldByName(NANOS));
         values.add(durationMessage.getField(durationMessage.getDescriptorForType().findFieldByName(SECONDS)));
-        values.add(durationMessage.getField(durationMessage.getDescriptorForType().findFieldByName(NANOS)));
+        values.add(nanos.longValue());
         return values;
     }
 

diff --git a/...a/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverterTest.java b/...a/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverterTest.java
@@ -28,7 +28,7 @@ public void shouldConvertToStruct() {
 
         TypeInfo typeInfo = durationProtobufMaxComputeConverter.convertTypeInfo(fieldDescriptor);
 
-        assertEquals("STRUCT<seconds:BIGINT,nanos:INT>", typeInfo.getTypeName());
+        assertEquals("STRUCT<seconds:BIGINT,nanos:BIGINT>", typeInfo.getTypeName());
     }
 
     @Test
@@ -41,8 +41,8 @@ public void shouldConvertDurationPayloadToStruct() {
                 .setDurationField(duration)
                 .build();
         List<String> expectedFieldNames = Arrays.asList("seconds", "nanos");
-        List<TypeInfo> expectedTypeInfos = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.INT);
-        List<Object> values = Arrays.asList(1L, 1);
+        List<TypeInfo> expectedTypeInfos = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.BIGINT);
+        List<Object> values = Arrays.asList(1L, 1L);
         Object result = durationProtobufMaxComputeConverter.convertPayload(new ProtoPayload(descriptor.getFields().get(5), message.getField(descriptor.getFields().get(5)), true));
 
         assertThat(result)
@@ -65,9 +65,9 @@ public void shouldConvertRepeatedDurationPayloadToStructList() {
                 .addAllDurationFields(Arrays.asList(duration1, duration2))
                 .build();
         List<String> expectedFieldNames = Arrays.asList("seconds", "nanos");
-        List<TypeInfo> expectedTypeInfos = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.INT);
-        List<Object> values1 = Arrays.asList(1L, 1);
-        List<Object> values2 = Arrays.asList(2L, 2);
+        List<TypeInfo> expectedTypeInfos = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.BIGINT);
+        List<Object> values1 = Arrays.asList(1L, 1L);
+        List<Object> values2 = Arrays.asList(2L, 2L);
 
         Object result = durationProtobufMaxComputeConverter.convertPayload(new ProtoPayload(repeatedDescriptor.getFields().get(5), message.getField(repeatedDescriptor.getFields().get(5)), true));
 

diff --git a/...va/com/gotocompany/depot/maxcompute/converter/MessageProtobufMaxComputeConverterTest.java b/...va/com/gotocompany/depot/maxcompute/converter/MessageProtobufMaxComputeConverterTest.java
@@ -83,7 +83,7 @@ public void shouldConvertToStruct() {
                 .newBuilder()
                 .setBuyer(message)
                 .build();
-        StructTypeInfo durationTypeInfo = TypeInfoFactory.getStructTypeInfo(Arrays.asList("seconds", "nanos"), Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.INT));
+        StructTypeInfo durationTypeInfo = TypeInfoFactory.getStructTypeInfo(Arrays.asList("seconds", "nanos"), Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.BIGINT));
         StructTypeInfo itemTypeInfo = TypeInfoFactory.getStructTypeInfo(Arrays.asList("id", "quantity"), Arrays.asList(TypeInfoFactory.STRING, TypeInfoFactory.INT));
         StructTypeInfo cartTypeInfo = TypeInfoFactory.getStructTypeInfo(
                 Arrays.asList("cart_id", "items", "created_at", "cart_age"),
@@ -100,7 +100,7 @@ public void shouldConvertToStruct() {
                                 "cart_id",
                                 Arrays.asList(new SimpleStruct(itemTypeInfo, Arrays.asList("item1", 1)), new SimpleStruct(itemTypeInfo, Arrays.asList("item2", null))),
                                 LocalDateTime.ofEpochSecond(timestamp.getSeconds(), 0, java.time.ZoneOffset.UTC),
-                                new SimpleStruct(durationTypeInfo, Arrays.asList(duration.getSeconds(), duration.getNanos())))),
+                                new SimpleStruct(durationTypeInfo, Arrays.asList(duration.getSeconds(), ((Integer) duration.getNanos()).longValue())))),
                 LocalDateTime.ofEpochSecond(timestamp.getSeconds(), 0, java.time.ZoneOffset.UTC)
         );
 

diff --git a/...st/java/com/gotocompany/depot/maxcompute/converter/ProtobufConverterOrchestratorTest.java b/...st/java/com/gotocompany/depot/maxcompute/converter/ProtobufConverterOrchestratorTest.java
@@ -53,7 +53,7 @@ public void shouldConvertPayloadToTypeInfo() {
         String expectedMessageTypeRepresentation = "STRUCT<string_field:STRING,another_inner_field:STRUCT<string_field:STRING>,another_inner_list_field:ARRAY<STRUCT<string_field:STRING>>>";
         String expectedRepeatedMessageTypeRepresentation = String.format("ARRAY<%s>", expectedMessageTypeRepresentation);
         String expectedTimestampTypeInfoRepresentation = "TIMESTAMP_NTZ";
-        String expectedDurationTypeInfoRepresentation = "STRUCT<seconds:BIGINT,nanos:INT>";
+        String expectedDurationTypeInfoRepresentation = "STRUCT<seconds:BIGINT,nanos:BIGINT>";
         String expectedStructTypeInfoRepresentation = "STRING";
 
         TypeInfo stringTypeInfo = protobufConverterOrchestrator.toMaxComputeTypeInfo(descriptor.findFieldByName("string_field"));
@@ -122,7 +122,7 @@ public void shouldConvertPayloadToRecord() {
 
         assertEquals("string_field", stringRecord);
         assertEquals(LocalDateTime.ofEpochSecond(100, 0, ZoneOffset.UTC), timestampRecord);
-        assertEquals(new SimpleStruct(TypeInfoFactory.getStructTypeInfo(Arrays.asList("seconds", "nanos"), Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.INT)), Arrays.asList(100L, 0)), durationRecord);
+        assertEquals(new SimpleStruct(TypeInfoFactory.getStructTypeInfo(Arrays.asList("seconds", "nanos"), Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.BIGINT)), Arrays.asList(100L, 0L)), durationRecord);
         assertEquals(expectedMessage, messageRecord);
         assertEquals(Collections.singletonList(expectedMessage), repeatedMessageRecord);
         assertEquals("{\"intField\":1.0,\"stringField\":\"String\"}", structRecord);