This is an automated email from the ASF dual-hosted git repository.
xqhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 91fb8ab8e6b upgrade org.apache.parquet:parquet-avro to to 1.15.2
(#35037)
91fb8ab8e6b is described below
commit 91fb8ab8e6bdc2966b9bcf1cb8fec3eb16065162
Author: liferoad <[email protected]>
AuthorDate: Thu Jun 12 18:31:12 2025 -0400
upgrade org.apache.parquet:parquet-avro to to 1.15.2 (#35037)
* upgrade org.apache.parquet:parquet-avro to to 1.15.2
* keep iceberg to 1.6.1
* use 1.9.0 for iceberg
* run all iceberg tests
* require Java 11
* fixed java11 for iceberg
* switched back to iceberg_version = "1.6.1"
* merged master
---------
Co-authored-by: Vitaly Terentyev <[email protected]>
---
.github/trigger_files/IO_Iceberg_Integration_Tests.json | 2 +-
.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json | 2 +-
.../IO_Iceberg_Managed_Integration_Tests_Dataflow.json | 2 +-
it/google-cloud-platform/build.gradle | 6 +++---
sdks/java/io/expansion-service/build.gradle | 2 ++
sdks/java/io/file-schema-transform/build.gradle | 2 +-
sdks/java/io/iceberg/build.gradle | 5 ++++-
.../src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java | 6 +++++-
sdks/java/io/parquet/build.gradle | 2 +-
9 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests.json
b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
index b73af5e61a4..89e73b29da0 100644
--- a/.github/trigger_files/IO_Iceberg_Integration_Tests.json
+++ b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run.",
- "modification": 1
+ "modification": 6
}
diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json
b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json
index 316d6e41388..69d5b8bdcd7 100644
--- a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json
+++ b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json
@@ -1,5 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run.",
- "modification": 2,
+ "modification": 3,
"https://github.com/apache/beam/pull/35159": "moving WindowedValue and
making an interface"
}
diff --git
a/.github/trigger_files/IO_Iceberg_Managed_Integration_Tests_Dataflow.json
b/.github/trigger_files/IO_Iceberg_Managed_Integration_Tests_Dataflow.json
index 5abe02fc09c..ab4daeae234 100644
--- a/.github/trigger_files/IO_Iceberg_Managed_Integration_Tests_Dataflow.json
+++ b/.github/trigger_files/IO_Iceberg_Managed_Integration_Tests_Dataflow.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run.",
- "modification": 1
+ "modification": 3
}
diff --git a/it/google-cloud-platform/build.gradle
b/it/google-cloud-platform/build.gradle
index f5fbcd8ba5d..56a1d730734 100644
--- a/it/google-cloud-platform/build.gradle
+++ b/it/google-cloud-platform/build.gradle
@@ -47,9 +47,9 @@ dependencies {
implementation library.java.jackson_databind
implementation 'org.apache.hadoop:hadoop-common:3.3.5'
implementation 'org.apache.avro:avro:1.11.1'
- implementation 'org.apache.parquet:parquet-avro:1.12.0'
- implementation 'org.apache.parquet:parquet-common:1.12.0'
- implementation 'org.apache.parquet:parquet-hadoop:1.12.0'
+ implementation 'org.apache.parquet:parquet-avro:1.15.2'
+ implementation 'org.apache.parquet:parquet-common:1.15.2'
+ implementation 'org.apache.parquet:parquet-hadoop:1.15.2'
implementation 'org.apache.commons:commons-lang3:3.9'
implementation library.java.gax
implementation library.java.google_api_common
diff --git a/sdks/java/io/expansion-service/build.gradle
b/sdks/java/io/expansion-service/build.gradle
index 2e6274571a8..c139315d925 100644
--- a/sdks/java/io/expansion-service/build.gradle
+++ b/sdks/java/io/expansion-service/build.gradle
@@ -36,6 +36,8 @@ configurations.runtimeClasspath {
// Pin avro to 1.11.4 due to https://github.com/apache/beam/issues/34968
// cannot upgrade this to the latest version due to
https://github.com/apache/beam/issues/34993
resolutionStrategy.force 'org.apache.avro:avro:1.11.4'
+ // force parquet-avro:1.15.2 to fix CVE-2025-46762
+ resolutionStrategy.force 'org.apache.parquet:parquet-avro:1.15.2'
// Pin Jetty version due to hadoop 3.4.1 using 9.4.53.v20231009, which had
// two direct vulnerabilities. There is one dependency vulnerability left
diff --git a/sdks/java/io/file-schema-transform/build.gradle
b/sdks/java/io/file-schema-transform/build.gradle
index 23cfe3f1be9..e3ee0316e22 100644
--- a/sdks/java/io/file-schema-transform/build.gradle
+++ b/sdks/java/io/file-schema-transform/build.gradle
@@ -32,7 +32,7 @@ configurations.implementation {
}
}
-def parquet_version = "1.15.1"
+def parquet_version = "1.15.2"
dependencies {
implementation library.java.avro
diff --git a/sdks/java/io/iceberg/build.gradle
b/sdks/java/io/iceberg/build.gradle
index 2ac04eb067a..41649ac8fdb 100644
--- a/sdks/java/io/iceberg/build.gradle
+++ b/sdks/java/io/iceberg/build.gradle
@@ -37,8 +37,11 @@ def hadoopVersions = [
hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
+// we cannot upgrade this since the newer iceberg requires Java 11
+// many other modules like examples/expansion use Java 8 and have the iceberg
dependency
+// def iceberg_version = "1.9.0"
def iceberg_version = "1.6.1"
-def parquet_version = "1.15.1"
+def parquet_version = "1.15.2"
def orc_version = "1.9.2"
def hive_version = "3.1.3"
diff --git
a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
index 90e6c376637..0c2bc71c6f8 100644
---
a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
+++
b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
@@ -111,8 +111,9 @@ public class IcebergUtils {
return Schema.FieldType.map(
icebergTypeToBeamFieldType(type.asMapType().keyType()),
icebergTypeToBeamFieldType(type.asMapType().valueType()));
+ default:
+ throw new RuntimeException("Unrecognized Iceberg Type: " +
type.typeId());
}
- throw new RuntimeException("Unrecognized IcebergIO Type");
}
private static Schema.Field icebergFieldToBeamField(final Types.NestedField
field) {
@@ -379,6 +380,9 @@ public class IcebergUtils {
}
Optional.ofNullable(icebergMap).ifPresent(v -> rec.setField(name, v));
break;
+ default:
+ // Do nothing for unsupported types
+ break;
}
}
diff --git a/sdks/java/io/parquet/build.gradle
b/sdks/java/io/parquet/build.gradle
index 9ed46a36773..73ba843e6b9 100644
--- a/sdks/java/io/parquet/build.gradle
+++ b/sdks/java/io/parquet/build.gradle
@@ -35,7 +35,7 @@ def hadoopVersions = [
hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
-def parquet_version = "1.15.1"
+def parquet_version = "1.15.2"
dependencies {
implementation library.java.vendored_guava_32_1_2_jre
