From 727d83ef34bd00b8e1c4b87b2a075cc0ba619b5d Mon Sep 17 00:00:00 2001 From: ekawinataa Date: Mon, 3 Feb 2025 18:41:59 +0700 Subject: [PATCH] Bugfix: Duration nanos INT to BIGINT (complying with MMS) (#63) * bugfix: set nanos in Duration to BIGINT MC type to comply with MMS * bump version * checkstyle --- build.gradle | 2 +- .../DurationProtobufMaxComputeConverter.java | 5 +++-- .../DurationProtobufMaxComputeConverterTest.java | 12 ++++++------ .../MessageProtobufMaxComputeConverterTest.java | 4 ++-- .../converter/ProtobufConverterOrchestratorTest.java | 4 ++-- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/build.gradle b/build.gradle index 06db1a5c..f5058bda 100644 --- a/build.gradle +++ b/build.gradle @@ -22,7 +22,7 @@ plugins { } group 'com.gotocompany' -version '0.10.5' +version '0.10.6' repositories { mavenLocal() diff --git a/src/main/java/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverter.java b/src/main/java/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverter.java index f9059db2..1525e257 100644 --- a/src/main/java/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverter.java +++ b/src/main/java/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverter.java @@ -20,7 +20,7 @@ public class DurationProtobufMaxComputeConverter implements ProtobufMaxComputeCo private static final String SECONDS = "seconds"; private static final String NANOS = "nanos"; private static final List FIELD_NAMES = Arrays.asList(SECONDS, NANOS); - private static final List TYPE_INFOS = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.INT); + private static final List TYPE_INFOS = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.BIGINT); @Override public TypeInfo convertSingularTypeInfo(Descriptors.FieldDescriptor fieldDescriptor) { @@ -35,8 +35,9 @@ public Object convertSingularPayload(ProtoPayload protoPayload) { private static List getValues(Message durationMessage) { List values = new ArrayList<>(); + Integer nanos = (Integer) durationMessage.getField(durationMessage.getDescriptorForType().findFieldByName(NANOS)); values.add(durationMessage.getField(durationMessage.getDescriptorForType().findFieldByName(SECONDS))); - values.add(durationMessage.getField(durationMessage.getDescriptorForType().findFieldByName(NANOS))); + values.add(nanos.longValue()); return values; } diff --git a/src/test/java/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverterTest.java b/src/test/java/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverterTest.java index 9bf1b00a..956ae264 100644 --- a/src/test/java/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverterTest.java +++ b/src/test/java/com/gotocompany/depot/maxcompute/converter/DurationProtobufMaxComputeConverterTest.java @@ -28,7 +28,7 @@ public void shouldConvertToStruct() { TypeInfo typeInfo = durationProtobufMaxComputeConverter.convertTypeInfo(fieldDescriptor); - assertEquals("STRUCT", typeInfo.getTypeName()); + assertEquals("STRUCT", typeInfo.getTypeName()); } @Test @@ -41,8 +41,8 @@ public void shouldConvertDurationPayloadToStruct() { .setDurationField(duration) .build(); List expectedFieldNames = Arrays.asList("seconds", "nanos"); - List expectedTypeInfos = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.INT); - List values = Arrays.asList(1L, 1); + List expectedTypeInfos = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.BIGINT); + List values = Arrays.asList(1L, 1L); Object result = durationProtobufMaxComputeConverter.convertPayload(new ProtoPayload(descriptor.getFields().get(5), message.getField(descriptor.getFields().get(5)), true)); assertThat(result) @@ -65,9 +65,9 @@ public void shouldConvertRepeatedDurationPayloadToStructList() { .addAllDurationFields(Arrays.asList(duration1, duration2)) .build(); List expectedFieldNames = Arrays.asList("seconds", "nanos"); - List expectedTypeInfos = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.INT); - List values1 = Arrays.asList(1L, 1); - List values2 = Arrays.asList(2L, 2); + List expectedTypeInfos = Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.BIGINT); + List values1 = Arrays.asList(1L, 1L); + List values2 = Arrays.asList(2L, 2L); Object result = durationProtobufMaxComputeConverter.convertPayload(new ProtoPayload(repeatedDescriptor.getFields().get(5), message.getField(repeatedDescriptor.getFields().get(5)), true)); diff --git a/src/test/java/com/gotocompany/depot/maxcompute/converter/MessageProtobufMaxComputeConverterTest.java b/src/test/java/com/gotocompany/depot/maxcompute/converter/MessageProtobufMaxComputeConverterTest.java index 680b8f86..831a74a6 100644 --- a/src/test/java/com/gotocompany/depot/maxcompute/converter/MessageProtobufMaxComputeConverterTest.java +++ b/src/test/java/com/gotocompany/depot/maxcompute/converter/MessageProtobufMaxComputeConverterTest.java @@ -83,7 +83,7 @@ public void shouldConvertToStruct() { .newBuilder() .setBuyer(message) .build(); - StructTypeInfo durationTypeInfo = TypeInfoFactory.getStructTypeInfo(Arrays.asList("seconds", "nanos"), Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.INT)); + StructTypeInfo durationTypeInfo = TypeInfoFactory.getStructTypeInfo(Arrays.asList("seconds", "nanos"), Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.BIGINT)); StructTypeInfo itemTypeInfo = TypeInfoFactory.getStructTypeInfo(Arrays.asList("id", "quantity"), Arrays.asList(TypeInfoFactory.STRING, TypeInfoFactory.INT)); StructTypeInfo cartTypeInfo = TypeInfoFactory.getStructTypeInfo( Arrays.asList("cart_id", "items", "created_at", "cart_age"), @@ -100,7 +100,7 @@ public void shouldConvertToStruct() { "cart_id", Arrays.asList(new SimpleStruct(itemTypeInfo, Arrays.asList("item1", 1)), new SimpleStruct(itemTypeInfo, Arrays.asList("item2", null))), LocalDateTime.ofEpochSecond(timestamp.getSeconds(), 0, java.time.ZoneOffset.UTC), - new SimpleStruct(durationTypeInfo, Arrays.asList(duration.getSeconds(), duration.getNanos())))), + new SimpleStruct(durationTypeInfo, Arrays.asList(duration.getSeconds(), ((Integer) duration.getNanos()).longValue())))), LocalDateTime.ofEpochSecond(timestamp.getSeconds(), 0, java.time.ZoneOffset.UTC) ); diff --git a/src/test/java/com/gotocompany/depot/maxcompute/converter/ProtobufConverterOrchestratorTest.java b/src/test/java/com/gotocompany/depot/maxcompute/converter/ProtobufConverterOrchestratorTest.java index 05323eda..1f798e6b 100644 --- a/src/test/java/com/gotocompany/depot/maxcompute/converter/ProtobufConverterOrchestratorTest.java +++ b/src/test/java/com/gotocompany/depot/maxcompute/converter/ProtobufConverterOrchestratorTest.java @@ -53,7 +53,7 @@ public void shouldConvertPayloadToTypeInfo() { String expectedMessageTypeRepresentation = "STRUCT,another_inner_list_field:ARRAY>>"; String expectedRepeatedMessageTypeRepresentation = String.format("ARRAY<%s>", expectedMessageTypeRepresentation); String expectedTimestampTypeInfoRepresentation = "TIMESTAMP_NTZ"; - String expectedDurationTypeInfoRepresentation = "STRUCT"; + String expectedDurationTypeInfoRepresentation = "STRUCT"; String expectedStructTypeInfoRepresentation = "STRING"; TypeInfo stringTypeInfo = protobufConverterOrchestrator.toMaxComputeTypeInfo(descriptor.findFieldByName("string_field")); @@ -122,7 +122,7 @@ public void shouldConvertPayloadToRecord() { assertEquals("string_field", stringRecord); assertEquals(LocalDateTime.ofEpochSecond(100, 0, ZoneOffset.UTC), timestampRecord); - assertEquals(new SimpleStruct(TypeInfoFactory.getStructTypeInfo(Arrays.asList("seconds", "nanos"), Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.INT)), Arrays.asList(100L, 0)), durationRecord); + assertEquals(new SimpleStruct(TypeInfoFactory.getStructTypeInfo(Arrays.asList("seconds", "nanos"), Arrays.asList(TypeInfoFactory.BIGINT, TypeInfoFactory.BIGINT)), Arrays.asList(100L, 0L)), durationRecord); assertEquals(expectedMessage, messageRecord); assertEquals(Collections.singletonList(expectedMessage), repeatedMessageRecord); assertEquals("{\"intField\":1.0,\"stringField\":\"String\"}", structRecord);