bump: bump avro to 1.13.1 (was 1.8.2) (#3039)

Explicitly bumps Avro to 1.13.1 which brings smaller API changes. Fixes CVE-2023-39410.
akka · Nov 23, 2023 · 93a3bd0 · 93a3bd0
1 parent d19f621
commit 93a3bd0
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 10 deletions.
diff --git a/avroparquet-tests/src/test/java/docs/javadsl/AvroParquetSinkTest.java b/avroparquet-tests/src/test/java/docs/javadsl/AvroParquetSinkTest.java
@@ -19,6 +19,7 @@
 import org.apache.parquet.avro.AvroReadSupport;
 import org.apache.parquet.hadoop.ParquetFileWriter;
 import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.hadoop.util.HadoopOutputFile;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -75,7 +76,7 @@ public void createNewParquetFile()
     Configuration conf = new Configuration();
     conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true);
     ParquetWriter<GenericRecord> writer =
-        AvroParquetWriter.<GenericRecord>builder(new Path(file))
+        AvroParquetWriter.<GenericRecord>builder(HadoopOutputFile.fromPath(new Path(file), conf))
             .withConf(conf)
             .withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
             .withSchema(schema)

diff --git a/avroparquet-tests/src/test/java/docs/javadsl/Examples.java b/avroparquet-tests/src/test/java/docs/javadsl/Examples.java
@@ -22,6 +22,7 @@
 import org.apache.avro.Schema;
 import akka.stream.javadsl.Source;
 import org.apache.parquet.avro.AvroParquetReader;
+import org.apache.parquet.hadoop.util.HadoopOutputFile;
 // #init-reader
 
 public class Examples {
@@ -53,7 +54,7 @@ public Examples() throws IOException {
 
     // #init-flow
     ParquetWriter<GenericRecord> writer =
-        AvroParquetWriter.<GenericRecord>builder(new Path("./test.parquet"))
+        AvroParquetWriter.<GenericRecord>builder(HadoopOutputFile.fromPath(new Path("./test.parquet"), conf))
             .withConf(conf)
             .withSchema(schema)
             .build();

diff --git a/avroparquet-tests/src/test/scala/docs/scaladsl/AbstractAvroParquet.scala b/avroparquet-tests/src/test/scala/docs/scaladsl/AbstractAvroParquet.scala
@@ -5,7 +5,6 @@
 package docs.scaladsl
 
 import java.io.File
-
 import akka.testkit.TestKit
 import com.sksamuel.avro4s.RecordFormat
 import org.apache.avro.Schema
@@ -14,7 +13,7 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 import org.apache.parquet.avro.{AvroParquetReader, AvroParquetWriter, AvroReadSupport}
 import org.apache.parquet.hadoop.{ParquetReader, ParquetWriter}
-import org.apache.parquet.hadoop.util.HadoopInputFile
+import org.apache.parquet.hadoop.util.{HadoopInputFile, HadoopOutputFile}
 import org.scalacheck.Gen
 import org.scalatest.{BeforeAndAfterAll, Suite}
 
@@ -48,7 +47,11 @@ trait AbstractAvroParquet extends BeforeAndAfterAll {
   conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true)
 
   def parquetWriter[T <: GenericRecord](file: String, conf: Configuration, schema: Schema): ParquetWriter[T] =
-    AvroParquetWriter.builder[T](new Path(file)).withConf(conf).withSchema(schema).build()
+    AvroParquetWriter
+      .builder[T](HadoopOutputFile.fromPath(new Path(file), conf))
+      .withConf(conf)
+      .withSchema(schema)
+      .build()
 
   def parquetReader[T <: GenericRecord](file: String, conf: Configuration): ParquetReader[T] =
     AvroParquetReader.builder[T](HadoopInputFile.fromPath(new Path(file), conf)).withConf(conf).build()
@@ -90,8 +93,11 @@ trait AbstractAvroParquet extends BeforeAndAfterAll {
     val file: String = "./sample/path/test.parquet"
     val conf: Configuration = new Configuration()
     conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true)
-    val writer: ParquetWriter[Record] =
-      AvroParquetWriter.builder[Record](new Path(file)).withConf(conf).withSchema(schema).build()
+    val writer: ParquetWriter[Record] = AvroParquetWriter
+      .builder[Record](HadoopOutputFile.fromPath(new Path(file), conf))
+      .withConf(conf)
+      .withSchema(schema)
+      .build()
     // #prepare-sink
     if (writer != null) { // forces val usage
     }
@@ -107,7 +113,11 @@ trait AbstractAvroParquet extends BeforeAndAfterAll {
 
     val file: String = "./sample/path/test.parquet"
     val writer: ParquetWriter[GenericRecord] =
-      AvroParquetWriter.builder[GenericRecord](new Path(file)).withConf(conf).withSchema(schema).build()
+      AvroParquetWriter
+        .builder[GenericRecord](HadoopOutputFile.fromPath(new Path(file), conf))
+        .withConf(conf)
+        .withSchema(schema)
+        .build()
     // #init-writer
     // #init-reader
     val reader: ParquetReader[GenericRecord] =

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -163,7 +163,9 @@ object Dependencies {
 
   val AvroParquet = Seq(
     libraryDependencies ++= Seq(
-        "org.apache.parquet" % "parquet-avro" % "1.10.1" //Apache2
+        "org.apache.parquet" % "parquet-avro" % "1.13.1", //Apache2
+        // override the version brought in by parquet-avro to fix CVE-2023-39410
+        "org.apache.avro" % "avro" % "1.11.3" //Apache2
       )
   )
 
@@ -230,7 +232,7 @@ object Dependencies {
     libraryDependencies ++= Seq(
         // https://github.com/googleapis/java-bigquerystorage/tree/master/proto-google-cloud-bigquerystorage-v1
         "com.google.api.grpc" % "proto-google-cloud-bigquerystorage-v1" % "1.22.0" % "protobuf-src", // ApacheV2
-        "org.apache.avro" % "avro" % "1.9.2" % "provided",
+        "org.apache.avro" % "avro" % "1.11.3" % "provided",
         "org.apache.arrow" % "arrow-vector" % "4.0.0" % "provided",
         "io.grpc" % "grpc-auth" % akka.grpc.gen.BuildInfo.grpcVersion, // ApacheV2
         "com.typesafe.akka" %% "akka-http-spray-json" % AkkaHttpVersion,