From 1f130c307007eff4d78614d05f9059a28416dc7e Mon Sep 17 00:00:00 2001 From: Kellen Dye Date: Thu, 21 Nov 2024 02:16:42 -0500 Subject: [PATCH 1/5] wip --- .github/dependabot.yml | 21 ++++ pom.xml | 245 ++++++++++++++++++++++------------------- 2 files changed, 155 insertions(+), 111 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f38eae0d..448b0dad 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,15 +12,36 @@ updates: - dependency-name: com.fasterxml.jackson.core:jackson-annotations - dependency-name: com.fasterxml.jackson.core:jackson-core - dependency-name: com.fasterxml.jackson.core:jackson-databind + - dependency-name: com.fasterxml.jackson:jackson-bom + - dependency-name: com.github.luben:zstd-jni - dependency-name: com.google.api-client:google-api-client - dependency-name: com.google.api.grpc:proto-google-iam-v1 - dependency-name: com.google.apis:google-api-services-storage + - dependency-name: com.google.auto.value:auto-value + - dependency-name: com.google.auto.value:auto-value-annotations - dependency-name: com.google.cloud.sql:mysql-socket-factory - dependency-name: com.google.cloud.sql:postgres-socket-factory - dependency-name: com.google.cloud:libraries-bom + - dependency-name: com.google.errorprone:error_prone_annotations + - dependency-name: com.google.guava:guava-bom - dependency-name: com.google.http-client:google-http-client - dependency-name: com.google.http-client:google-http-client-test - dependency-name: com.google.oauth-client:google-oauth-client - dependency-name: com.google.protobuf:protobuf-java - dependency-name: com.google.protobuf:protobuf-java-util + - dependency-name: commons-codec:commons-codec + - dependency-name: org.apache.beam:beam-sdks-java-bom # manually bump this + - dependency-name: org.apache.commons:commons-compress + - dependency-name: org.apache.httpcomponents:httpcore + - dependency-name: org.apache.httpcomponents:httpclient + - dependency-name: org.checkerframework:checker-qual + - dependency-name: org.slf4j:slf4j-api + - dependency-name: org.slf4j:slf4j-jdk14 + - dependency-name: org.threeten:threetenbp - dependency-name: io.grpc:grpc-bom + - dependency-name: io.netty:netty-bom + - dependency-name: joda-time:joda-time + # sync with libraries-bom + - dependency-name: io.opencensus:opencensus-contrib-grpc-metrics + + diff --git a/pom.xml b/pom.xml index 6e887a48..5e0a7fe8 100644 --- a/pom.xml +++ b/pom.xml @@ -100,138 +100,145 @@ 8 - 1.11.0 - 1.11.3 - 1.78.1 - - - 2.59.0 + UTF-8 + UTF-8 + + + + 2.60.0 + + 1.9 + 3.42.0 + 1.17.1 + 1.26.2 + 2.10.0 + 33.1.0-jre + 2.1 + 4.5.13 + 4.4.14 + 2.15.4 + 2.10.14 + 4.1.100.Final + 1.7.30 + 1.6.8 + 1.5.6-3 + + 26.45.0 - 33.3.0-jre - 3.0 + + 0.31.1 + + + 1.11.3 + 1.78.1 4.13.2 - 2.17.2 8.4.0 - 4.1.113.Final 42.7.4 - UTF-8 - UTF-8 - 2.0.16 1.18.0 + + - io.opencensus - opencensus-contrib-grpc-metrics - 0.31.1 - - - joda-time - joda-time - 2.12.7 - - - org.threeten - threetenbp - 1.7.0 + com.fasterxml.jackson + jackson-bom + ${jackson.version} + import + pom - org.checkerframework - checker-qual - 3.47.0 + com.github.luben + zstd-jni + ${zstd-jni.version} - org.apache.httpcomponents - httpcore - 4.4.16 + com.google.auto.value + auto-value + ${auto-value.version} + provided - org.apache.httpcomponents - httpclient - 4.5.14 + com.google.auto.value + auto-value-annotations + ${auto-value.version} com.google.errorprone error_prone_annotations - 2.32.0 + ${errorprone.version} - com.google.apis - google-api-services-storage - v1-rev20240809-2.0.0 + com.google.guava + guava-bom + ${guava.version} + import + pom commons-codec commons-codec - 1.17.1 + ${commons-codec.version} - org.apache.commons - commons-compress - 1.26.2 + joda-time + joda-time + ${joda-time.version} org.apache.commons - commons-lang3 - 3.17.0 + commons-compress + ${commons-compress.version} - net.bytebuddy - byte-buddy - 1.15.1 + org.apache.httpcomponents + httpcore + ${httpcore.version} - - - org.xerial.snappy - snappy-java - 1.1.10.7 + org.apache.httpcomponents + httpclient + ${httpclient.version} - org.bouncycastle - bcpkix-jdk18on - ${bouncycastle.version} + org.checkerframework + checker-qual + ${checker-qual.version} - org.bouncycastle - bcprov-jdk18on - ${bouncycastle.version} + org.slf4j + slf4j-api + ${slf4j.version} - com.google.guava - guava-bom - ${guava.version} - import - pom + org.slf4j + slf4j-jdk14 + ${slf4j.version} + runtime - - com.google.apis - google-api-services-cloudkms - v1-rev20240314-2.0.0 + org.threeten + threetenbp + ${threetenbp.version} + - com.github.luben - zstd-jni - 1.5.6-6 + io.opencensus + opencensus-contrib-grpc-metrics + ${opencensus.version} + + + - io.netty - netty-bom - ${netty.version} - import + org.apache.beam + beam-sdks-java-bom + ${beam.version} pom - - - com.fasterxml.jackson - jackson-bom - ${jackson.version} import - pom com.google.cloud @@ -241,22 +248,21 @@ import - org.slf4j - slf4j-api - ${slf4j.version} + io.netty + netty-bom + ${netty.version} + import + pom - org.slf4j - slf4j-jdk14 - ${slf4j.version} - runtime + com.google.apis + google-api-services-cloudkms + v1-rev20240314-2.0.0 - org.apache.beam - beam-sdks-java-bom - ${beam.version} - pom - import + org.apache.avro + avro + ${avro.version} @@ -285,6 +291,18 @@ junit junit + + org.checkerframework + checker-qual + + + org.apache.arrow + arrow-vector + + + com.google.guava + guava + com.google.api.grpc grpc-google-cloud-pubsub-v1 @@ -347,22 +365,6 @@ - - org.apache.avro - avro - ${avro.version} - - - com.google.auto.value - auto-value-annotations - ${auto-value.version} - - - com.google.auto.value - auto-value - ${auto-value.version} - provided - @@ -396,7 +398,6 @@ - junit junit @@ -554,6 +555,28 @@ [3.3.9,) + + + + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.datatype:jackson-datatype-jsr310 + com.google.auto.value:auto-value + com.google.auto.value:auto-value-annotations + com.google.errorprone:error_prone_annotations + com.google.guava:guava + commons-codec:commons-codec + joda-time:joda-time + org.apache.commons:commons-compress + org.apache.httpcomponents:httpcore + org.apache.httpcomponents:httpclient + org.checkerframework:checker-qual + org.slf4j:slf4j-api + org.slf4j:slf4j-jdk14 + org.threeten:threetenbp + + io.opencensus:opencensus-contrib-grpc-metrics + + @@ -717,7 +740,7 @@ - + com.spotify.dbeam.jobs.JdbcAvroJob From ac8f258dfb32964b18d5efc60f9feb3a14a59363 Mon Sep 17 00:00:00 2001 From: Kellen Dye Date: Thu, 21 Nov 2024 02:21:59 -0500 Subject: [PATCH 2/5] wip --- .github/dependabot.yml | 2 -- pom.xml | 14 ++------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 448b0dad..202c6aca 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -43,5 +43,3 @@ updates: - dependency-name: joda-time:joda-time # sync with libraries-bom - dependency-name: io.opencensus:opencensus-contrib-grpc-metrics - - diff --git a/pom.xml b/pom.xml index 5e0a7fe8..8d49ceac 100644 --- a/pom.xml +++ b/pom.xml @@ -291,18 +291,6 @@ junit junit - - org.checkerframework - checker-qual - - - org.apache.arrow - arrow-vector - - - com.google.guava - guava - com.google.api.grpc grpc-google-cloud-pubsub-v1 @@ -558,7 +546,9 @@ + com.fasterxml.jackson.core:jackson-annotations com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.core:jackson-databind com.fasterxml.jackson.datatype:jackson-datatype-jsr310 com.google.auto.value:auto-value com.google.auto.value:auto-value-annotations From dcc9fcae21763f43323af26c80cb8745c3d78310 Mon Sep 17 00:00:00 2001 From: Kellen Dye Date: Thu, 21 Nov 2024 03:01:09 -0500 Subject: [PATCH 3/5] wip --- NOTICE | 2 +- docs/type-conversion.md | 56 ++++++++++++++++++++--------------------- pom.xml | 3 ++- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/NOTICE b/NOTICE index ac7ac638..652c189a 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,2 @@ DBeam -Copyright 2017 Spotify AB +Copyright 2024 Spotify AB diff --git a/docs/type-conversion.md b/docs/type-conversion.md index fcbc9e43..4ee6819d 100644 --- a/docs/type-conversion.md +++ b/docs/type-conversion.md @@ -5,31 +5,31 @@ When applicable and `--useAvroLogicalTypes` parameter is set to `true`, Avro log To represent nullable columns, unions with the Avro NULL type are used. - **Java SQL type** | **Avro type** | **Avro schema annotations** | **Comments** ---- | --- | --- | --- -BIGINT | long / string | | Depends on a SQL column precision -INTEGER | int | | -SMALLINT | int | | -TINYINT | int | | -TIMESTAMP | long |logicalType: time-millis | -DATE | long | logicalType: time-millis | -TIME | long | logicalType: time-millis | -TIME_WITH_TIMEZONE | long | logicalType: time-millis | -BOOLEAN | boolean | | -BIT | boolean / bytes | | Depends on a SQL column precision -BINARY | bytes | | -BINARY | bytes | | -VARBINARY | bytes | | -LONGVARBINARY | bytes | | -ARRAY | bytes | | -BLOB | bytes | | -DOUBLE | double | | -FLOAT | float | | -REAL | float | | -VARCHAR | string | | -CHAR | string | | -CLOB | string | | -LONGNVARCHAR | string | | -LONGVARCHAR | string | | -NCHAR | string | | -all other Java SQL types | string | | +| **Java SQL type** | **Avro type** | **Avro schema annotations** | **Comments** | +|--------------------------|-----------------|-----------------------------|-----------------------------------| +| BIGINT | long / string | | Depends on a SQL column precision | +| INTEGER | int | | | +| SMALLINT | int | | | +| TINYINT | int | | | +| TIMESTAMP | long | logicalType: time-millis | | +| DATE | long | logicalType: time-millis | | +| TIME | long | logicalType: time-millis | | +| TIME_WITH_TIMEZONE | long | logicalType: time-millis | | +| BOOLEAN | boolean | | | +| BIT | boolean / bytes | | Depends on a SQL column precision | +| BINARY | bytes | | | +| BINARY | bytes | | | +| VARBINARY | bytes | | | +| LONGVARBINARY | bytes | | | +| ARRAY | bytes | | | +| BLOB | bytes | | | +| DOUBLE | double | | | +| FLOAT | float | | | +| REAL | float | | | +| VARCHAR | string | | | +| CHAR | string | | | +| CLOB | string | | | +| LONGNVARCHAR | string | | | +| LONGVARCHAR | string | | | +| NCHAR | string | | | +| all other Java SQL types | string | | | diff --git a/pom.xml b/pom.xml index 8d49ceac..710675ab 100644 --- a/pom.xml +++ b/pom.xml @@ -131,7 +131,7 @@ 0.31.1 - 1.11.3 + 1.11.4 1.78.1 4.13.2 8.4.0 @@ -550,6 +550,7 @@ com.fasterxml.jackson.core:jackson-core com.fasterxml.jackson.core:jackson-databind com.fasterxml.jackson.datatype:jackson-datatype-jsr310 + com.github.luben:zstd-jni com.google.auto.value:auto-value com.google.auto.value:auto-value-annotations com.google.errorprone:error_prone_annotations From 8dd394c6119a47a586aacce25c2c9dcd695df07f Mon Sep 17 00:00:00 2001 From: Kellen Dye Date: Fri, 22 Nov 2024 09:56:16 -0500 Subject: [PATCH 4/5] avro --- pom.xml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 710675ab..9d3cec6f 100644 --- a/pom.xml +++ b/pom.xml @@ -108,6 +108,7 @@ 2.60.0 1.9 + 1.11.3 3.42.0 1.17.1 1.26.2 @@ -131,7 +132,6 @@ 0.31.1 - 1.11.4 1.78.1 4.13.2 8.4.0 @@ -143,6 +143,11 @@ + + org.apache.avro + avro + ${avro.version} + com.fasterxml.jackson jackson-bom @@ -259,11 +264,6 @@ google-api-services-cloudkms v1-rev20240314-2.0.0 - - org.apache.avro - avro - ${avro.version} - @@ -557,6 +557,7 @@ com.google.guava:guava commons-codec:commons-codec joda-time:joda-time + org.apache.avro:avro org.apache.commons:commons-compress org.apache.httpcomponents:httpcore org.apache.httpcomponents:httpclient From 1d174c235d4bf1a8197832e906a17f1b0d0fbf69 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Wed, 11 Dec 2024 08:54:58 +0100 Subject: [PATCH 5/5] Update to beam 2.61 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 9d3cec6f..46b75971 100644 --- a/pom.xml +++ b/pom.xml @@ -104,13 +104,13 @@ UTF-8 - - 2.60.0 + + 2.61.0 1.9 1.11.3 3.42.0 - 1.17.1 + 1.17.1 1.26.2 2.10.0 33.1.0-jre