From 93a3bd0c8ba9dd0c7ab092d5026a6b4eb138e0fb Mon Sep 17 00:00:00 2001 From: Sebastian Alfers Date: Thu, 23 Nov 2023 09:21:55 +0100 Subject: [PATCH] bump: bump avro to 1.13.1 (was 1.8.2) (#3039) Explicitly bumps Avro to 1.13.1 which brings smaller API changes. Fixes CVE-2023-39410. --- .../docs/javadsl/AvroParquetSinkTest.java | 3 ++- .../src/test/java/docs/javadsl/Examples.java | 3 ++- .../docs/scaladsl/AbstractAvroParquet.scala | 22 ++++++++++++++----- project/Dependencies.scala | 6 +++-- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/avroparquet-tests/src/test/java/docs/javadsl/AvroParquetSinkTest.java b/avroparquet-tests/src/test/java/docs/javadsl/AvroParquetSinkTest.java index bd2eb8eab5..8874d996a2 100644 --- a/avroparquet-tests/src/test/java/docs/javadsl/AvroParquetSinkTest.java +++ b/avroparquet-tests/src/test/java/docs/javadsl/AvroParquetSinkTest.java @@ -19,6 +19,7 @@ import org.apache.parquet.avro.AvroReadSupport; import org.apache.parquet.hadoop.ParquetFileWriter; import org.apache.parquet.hadoop.ParquetReader; +import org.apache.parquet.hadoop.util.HadoopOutputFile; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -75,7 +76,7 @@ public void createNewParquetFile() Configuration conf = new Configuration(); conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true); ParquetWriter writer = - AvroParquetWriter.builder(new Path(file)) + AvroParquetWriter.builder(HadoopOutputFile.fromPath(new Path(file), conf)) .withConf(conf) .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) .withSchema(schema) diff --git a/avroparquet-tests/src/test/java/docs/javadsl/Examples.java b/avroparquet-tests/src/test/java/docs/javadsl/Examples.java index e4093523e1..3b7cf4775a 100644 --- a/avroparquet-tests/src/test/java/docs/javadsl/Examples.java +++ b/avroparquet-tests/src/test/java/docs/javadsl/Examples.java @@ -22,6 +22,7 @@ import org.apache.avro.Schema; import akka.stream.javadsl.Source; import org.apache.parquet.avro.AvroParquetReader; +import org.apache.parquet.hadoop.util.HadoopOutputFile; // #init-reader public class Examples { @@ -53,7 +54,7 @@ public Examples() throws IOException { // #init-flow ParquetWriter writer = - AvroParquetWriter.builder(new Path("./test.parquet")) + AvroParquetWriter.builder(HadoopOutputFile.fromPath(new Path("./test.parquet"), conf)) .withConf(conf) .withSchema(schema) .build(); diff --git a/avroparquet-tests/src/test/scala/docs/scaladsl/AbstractAvroParquet.scala b/avroparquet-tests/src/test/scala/docs/scaladsl/AbstractAvroParquet.scala index b55cf80e54..6e65d050bf 100644 --- a/avroparquet-tests/src/test/scala/docs/scaladsl/AbstractAvroParquet.scala +++ b/avroparquet-tests/src/test/scala/docs/scaladsl/AbstractAvroParquet.scala @@ -5,7 +5,6 @@ package docs.scaladsl import java.io.File - import akka.testkit.TestKit import com.sksamuel.avro4s.RecordFormat import org.apache.avro.Schema @@ -14,7 +13,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.parquet.avro.{AvroParquetReader, AvroParquetWriter, AvroReadSupport} import org.apache.parquet.hadoop.{ParquetReader, ParquetWriter} -import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.parquet.hadoop.util.{HadoopInputFile, HadoopOutputFile} import org.scalacheck.Gen import org.scalatest.{BeforeAndAfterAll, Suite} @@ -48,7 +47,11 @@ trait AbstractAvroParquet extends BeforeAndAfterAll { conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true) def parquetWriter[T <: GenericRecord](file: String, conf: Configuration, schema: Schema): ParquetWriter[T] = - AvroParquetWriter.builder[T](new Path(file)).withConf(conf).withSchema(schema).build() + AvroParquetWriter + .builder[T](HadoopOutputFile.fromPath(new Path(file), conf)) + .withConf(conf) + .withSchema(schema) + .build() def parquetReader[T <: GenericRecord](file: String, conf: Configuration): ParquetReader[T] = AvroParquetReader.builder[T](HadoopInputFile.fromPath(new Path(file), conf)).withConf(conf).build() @@ -90,8 +93,11 @@ trait AbstractAvroParquet extends BeforeAndAfterAll { val file: String = "./sample/path/test.parquet" val conf: Configuration = new Configuration() conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true) - val writer: ParquetWriter[Record] = - AvroParquetWriter.builder[Record](new Path(file)).withConf(conf).withSchema(schema).build() + val writer: ParquetWriter[Record] = AvroParquetWriter + .builder[Record](HadoopOutputFile.fromPath(new Path(file), conf)) + .withConf(conf) + .withSchema(schema) + .build() // #prepare-sink if (writer != null) { // forces val usage } @@ -107,7 +113,11 @@ trait AbstractAvroParquet extends BeforeAndAfterAll { val file: String = "./sample/path/test.parquet" val writer: ParquetWriter[GenericRecord] = - AvroParquetWriter.builder[GenericRecord](new Path(file)).withConf(conf).withSchema(schema).build() + AvroParquetWriter + .builder[GenericRecord](HadoopOutputFile.fromPath(new Path(file), conf)) + .withConf(conf) + .withSchema(schema) + .build() // #init-writer // #init-reader val reader: ParquetReader[GenericRecord] = diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 296f70d3d6..770adcc178 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -163,7 +163,9 @@ object Dependencies { val AvroParquet = Seq( libraryDependencies ++= Seq( - "org.apache.parquet" % "parquet-avro" % "1.10.1" //Apache2 + "org.apache.parquet" % "parquet-avro" % "1.13.1", //Apache2 + // override the version brought in by parquet-avro to fix CVE-2023-39410 + "org.apache.avro" % "avro" % "1.11.3" //Apache2 ) ) @@ -230,7 +232,7 @@ object Dependencies { libraryDependencies ++= Seq( // https://github.com/googleapis/java-bigquerystorage/tree/master/proto-google-cloud-bigquerystorage-v1 "com.google.api.grpc" % "proto-google-cloud-bigquerystorage-v1" % "1.22.0" % "protobuf-src", // ApacheV2 - "org.apache.avro" % "avro" % "1.9.2" % "provided", + "org.apache.avro" % "avro" % "1.11.3" % "provided", "org.apache.arrow" % "arrow-vector" % "4.0.0" % "provided", "io.grpc" % "grpc-auth" % akka.grpc.gen.BuildInfo.grpcVersion, // ApacheV2 "com.typesafe.akka" %% "akka-http-spray-json" % AkkaHttpVersion,