This is an automated email from the ASF dual-hosted git repository.
fanningpj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-pekko-connectors.git
The following commit(s) were added to refs/heads/main by this push:
new 837dbe316 upgrade to avro 1.11.3 due to CVE (#259)
837dbe316 is described below
commit 837dbe316d028cbef0fb075689156452f1af3c0e
Author: PJ Fanning <[email protected]>
AuthorDate: Sun Oct 8 19:32:08 2023 +0100
upgrade to avro 1.11.3 due to CVE (#259)
* upgrade to avro 1.11.3 due to CVE
* suppress deprecation warnings
* try suppress deprecation warnings
* try to suppress deprecation warnings
* Update AbstractAvroParquetBase.scala
---
.../src/test/java/docs/javadsl/AvroParquetSinkTest.java | 1 +
avroparquet/src/test/java/docs/javadsl/Examples.java | 1 +
.../scala/docs/scaladsl/AbstractAvroParquetBase.scala | 17 ++++++++++-------
project/Dependencies.scala | 8 +++++---
4 files changed, 17 insertions(+), 10 deletions(-)
diff --git a/avroparquet/src/test/java/docs/javadsl/AvroParquetSinkTest.java
b/avroparquet/src/test/java/docs/javadsl/AvroParquetSinkTest.java
index d72820e8b..a6b7b4da9 100644
--- a/avroparquet/src/test/java/docs/javadsl/AvroParquetSinkTest.java
+++ b/avroparquet/src/test/java/docs/javadsl/AvroParquetSinkTest.java
@@ -76,6 +76,7 @@ public class AvroParquetSinkTest {
records.add(new GenericRecordBuilder(schema).set("id", "3").set("body",
"body13").build());
}
+ @SuppressWarnings("deprecation")
@Test
public void createNewParquetFile()
throws InterruptedException, IOException, TimeoutException,
ExecutionException {
diff --git a/avroparquet/src/test/java/docs/javadsl/Examples.java
b/avroparquet/src/test/java/docs/javadsl/Examples.java
index 406631a25..a1a884f5e 100644
--- a/avroparquet/src/test/java/docs/javadsl/Examples.java
+++ b/avroparquet/src/test/java/docs/javadsl/Examples.java
@@ -58,6 +58,7 @@ public class Examples {
Source<GenericRecord, NotUsed> source = AvroParquetSource.create(reader);
// #init-source
+ @SuppressWarnings("deprecation")
public Examples() throws IOException {
// #init-flow
diff --git
a/avroparquet/src/test/scala/docs/scaladsl/AbstractAvroParquetBase.scala
b/avroparquet/src/test/scala/docs/scaladsl/AbstractAvroParquetBase.scala
index 97a4052f8..580b271a4 100644
--- a/avroparquet/src/test/scala/docs/scaladsl/AbstractAvroParquetBase.scala
+++ b/avroparquet/src/test/scala/docs/scaladsl/AbstractAvroParquetBase.scala
@@ -22,6 +22,7 @@ import org.apache.parquet.hadoop.util.HadoopInputFile
import org.apache.parquet.hadoop.{ ParquetReader, ParquetWriter }
import org.scalacheck.Gen
+import scala.annotation.nowarn
import scala.util.Random
trait AbstractAvroParquetBase {
@@ -47,6 +48,7 @@ trait AbstractAvroParquetBase {
val conf: Configuration = new Configuration()
conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true)
+ @nowarn("msg=deprecated")
def parquetWriter[T <: GenericRecord](file: String, conf: Configuration,
schema: Schema): ParquetWriter[T] =
AvroParquetWriter.builder[T](new
Path(file)).withConf(conf).withSchema(schema).build()
@@ -80,6 +82,7 @@ trait AbstractAvroParquetBase {
// #prepare-source
}
+ @nowarn("msg=deprecated")
def sinkDocumentation(): Unit = {
// #prepare-sink
import com.sksamuel.avro4s.Record
@@ -87,30 +90,30 @@ trait AbstractAvroParquetBase {
import org.apache.hadoop.fs.Path
import org.apache.parquet.avro.AvroReadSupport
- val file: String = "./sample/path/test.parquet"
- val conf: Configuration = new Configuration()
+ val file = "./sample/path/test.parquet"
+ val conf = new Configuration()
conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true)
- val writer: ParquetWriter[Record] =
+ val writer =
AvroParquetWriter.builder[Record](new
Path(file)).withConf(conf).withSchema(schema).build()
// #prepare-sink
if (writer != null) { // forces val usage
}
}
+ @nowarn("msg=deprecated")
def initWriterDocumentation(): Unit = {
// #init-writer
import org.apache.avro.generic.GenericRecord
import org.apache.hadoop.fs.Path
import org.apache.parquet.avro.AvroParquetReader
- import org.apache.parquet.hadoop.ParquetReader
import org.apache.parquet.hadoop.util.HadoopInputFile
- val file: String = "./sample/path/test.parquet"
- val writer: ParquetWriter[GenericRecord] =
+ val file = "./sample/path/test.parquet"
+ val writer =
AvroParquetWriter.builder[GenericRecord](new
Path(file)).withConf(conf).withSchema(schema).build()
// #init-writer
// #init-reader
- val reader: ParquetReader[GenericRecord] =
+ val reader =
AvroParquetReader.builder[GenericRecord](HadoopInputFile.fromPath(new
Path(file), conf)).withConf(conf).build()
// #init-reader
if (writer != null && reader != null) { // forces val usage
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 0dbd617a9..4abb9666c 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -25,6 +25,7 @@ object Dependencies {
val InfluxDBJavaVersion = "2.15"
+ val AvroVersion = "1.11.3"
val AwsSdk2Version = "2.17.113"
val AwsSpiPekkoHttpVersion = "0.1.0"
// Sync with plugins.sbt
@@ -157,12 +158,13 @@ object Dependencies {
"com.google.jimfs" % "jimfs" % "1.2" % Test))
val avro4sVersion: Def.Initialize[String] = Def.setting {
- if (Common.isScala3.value) "5.0.4" else "4.1.1"
+ if (Common.isScala3.value) "5.0.5" else "4.1.1"
}
val AvroParquet = Seq(
libraryDependencies ++= Seq(
- "org.apache.parquet" % "parquet-avro" % "1.10.1", // Apache2
+ "org.apache.parquet" % "parquet-avro" % "1.13.1", // Apache2
+ "org.apache.avro" % "avro" % AvroVersion,
("org.apache.hadoop" % "hadoop-client" % "3.2.1" %
Test).exclude("log4j", "log4j"), // Apache2
("org.apache.hadoop" % "hadoop-common" % "3.2.1" %
Test).exclude("log4j", "log4j"), // Apache2
"com.sksamuel.avro4s" %% "avro4s-core" % avro4sVersion.value % Test,
@@ -214,7 +216,7 @@ object Dependencies {
libraryDependencies ++= Seq(
//
https://github.com/googleapis/java-bigquerystorage/tree/master/proto-google-cloud-bigquerystorage-v1
"com.google.api.grpc" % "proto-google-cloud-bigquerystorage-v1" %
"1.22.0" % "protobuf-src",
- "org.apache.avro" % "avro" % "1.9.2" % "provided",
+ "org.apache.avro" % "avro" % AvroVersion % "provided",
"org.apache.arrow" % "arrow-vector" % "4.0.0" % "provided",
"io.grpc" % "grpc-auth" %
org.apache.pekko.grpc.gen.BuildInfo.grpcVersion,
"com.google.protobuf" % "protobuf-java" % protobufJavaVersion,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
