This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 85bff0d2c1a Reapply "bump hadoop version (#33011)" (#33312)
85bff0d2c1a is described below
commit 85bff0d2c1adb743c4738f51428c88fb74532021
Author: Yi Hu <[email protected]>
AuthorDate: Mon Dec 9 11:59:38 2024 -0500
Reapply "bump hadoop version (#33011)" (#33312)
* Reapply "bump hadoop version (#33011)" (#33257)
This reverts commit 7e25649b88b1ac08e48db28f8af09978b649da17.
* Fix hbase and hcatalog test dependencies
* Add missing pinned hadoop dependency version for compat test target
---
.../trigger_files/beam_PostCommit_Java_Hadoop_Versions.json | 2 +-
.github/trigger_files/beam_PreCommit_Java_HBase_IO_Direct.json | 1 +
CHANGES.md | 1 +
.../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
sdks/java/io/hadoop-common/build.gradle | 2 +-
sdks/java/io/hadoop-file-system/build.gradle | 2 +-
sdks/java/io/hadoop-format/build.gradle | 2 +-
sdks/java/io/hbase/build.gradle | 9 ++-------
sdks/java/io/hcatalog/build.gradle | 10 +++++++++-
sdks/java/io/iceberg/build.gradle | 8 ++++++++
10 files changed, 26 insertions(+), 13 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json
b/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json
index 920c8d132e4..8784d0786c0 100644
--- a/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json
+++ b/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "modification": 1
+ "modification": 2
}
\ No newline at end of file
diff --git a/.github/trigger_files/beam_PreCommit_Java_HBase_IO_Direct.json
b/.github/trigger_files/beam_PreCommit_Java_HBase_IO_Direct.json
new file mode 100644
index 00000000000..0967ef424bc
--- /dev/null
+++ b/.github/trigger_files/beam_PreCommit_Java_HBase_IO_Direct.json
@@ -0,0 +1 @@
+{}
diff --git a/CHANGES.md b/CHANGES.md
index 1b943a99f8a..dbadd588ae3 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -64,6 +64,7 @@
* gcs-connector config options can be set via GcsOptions (Java)
([#32769](https://github.com/apache/beam/pull/32769)).
* Support for X source added (Java/Python)
([#X](https://github.com/apache/beam/issues/X)).
+* Upgraded the default version of Hadoop dependencies to 3.4.1. Hadoop 2.10.2
is still supported (Java)
([#33011](https://github.com/apache/beam/issues/33011)).
## New Features / Improvements
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 84c7c3ecfd4..2abd43a5d4c 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -614,7 +614,7 @@ class BeamModulePlugin implements Plugin<Project> {
// [bomupgrader] determined by: io.grpc:grpc-netty, consistent with:
google_cloud_platform_libraries_bom
def grpc_version = "1.67.1"
def guava_version = "33.1.0-jre"
- def hadoop_version = "2.10.2"
+ def hadoop_version = "3.4.1"
def hamcrest_version = "2.1"
def influxdb_version = "2.19"
def httpclient_version = "4.5.13"
diff --git a/sdks/java/io/hadoop-common/build.gradle
b/sdks/java/io/hadoop-common/build.gradle
index b0303d29ff9..4375001ffa8 100644
--- a/sdks/java/io/hadoop-common/build.gradle
+++ b/sdks/java/io/hadoop-common/build.gradle
@@ -28,7 +28,7 @@ def hadoopVersions = [
"2102": "2.10.2",
"324": "3.2.4",
"336": "3.3.6",
- "341": "3.4.1",
+ // "341": "3.4.1", // tests already exercised on the default version
]
hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
diff --git a/sdks/java/io/hadoop-file-system/build.gradle
b/sdks/java/io/hadoop-file-system/build.gradle
index fafa8b5c7e3..b4ebbfa08c5 100644
--- a/sdks/java/io/hadoop-file-system/build.gradle
+++ b/sdks/java/io/hadoop-file-system/build.gradle
@@ -29,7 +29,7 @@ def hadoopVersions = [
"2102": "2.10.2",
"324": "3.2.4",
"336": "3.3.6",
- "341": "3.4.1",
+ // "341": "3.4.1", // tests already exercised on the default version
]
hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
diff --git a/sdks/java/io/hadoop-format/build.gradle
b/sdks/java/io/hadoop-format/build.gradle
index 4664005a1fc..73fc44a0f31 100644
--- a/sdks/java/io/hadoop-format/build.gradle
+++ b/sdks/java/io/hadoop-format/build.gradle
@@ -33,7 +33,7 @@ def hadoopVersions = [
"2102": "2.10.2",
"324": "3.2.4",
"336": "3.3.6",
- "341": "3.4.1",
+ // "341": "3.4.1", // tests already exercised on the default version
]
hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
diff --git a/sdks/java/io/hbase/build.gradle b/sdks/java/io/hbase/build.gradle
index d85c0fc610b..07014f2d5e3 100644
--- a/sdks/java/io/hbase/build.gradle
+++ b/sdks/java/io/hbase/build.gradle
@@ -34,7 +34,7 @@ test {
jvmArgs "-Dtest.build.data.basedirectory=build/test-data"
}
-def hbase_version = "2.5.5"
+def hbase_version = "2.6.1-hadoop3"
dependencies {
implementation library.java.vendored_guava_32_1_2_jre
@@ -46,12 +46,7 @@ dependencies {
testImplementation project(path: ":sdks:java:core", configuration:
"shadowTest")
testImplementation library.java.junit
testImplementation library.java.hamcrest
- testImplementation library.java.hadoop_minicluster
- testImplementation library.java.hadoop_hdfs
- testImplementation library.java.hadoop_common
+ // shaded-testing-utils has shaded all Hadoop/HBase dependencies
testImplementation("org.apache.hbase:hbase-shaded-testing-util:$hbase_version")
- testImplementation
"org.apache.hbase:hbase-hadoop-compat:$hbase_version:tests"
- testImplementation
"org.apache.hbase:hbase-hadoop2-compat:$hbase_version:tests"
testRuntimeOnly project(path: ":runners:direct-java", configuration:
"shadow")
}
-
diff --git a/sdks/java/io/hcatalog/build.gradle
b/sdks/java/io/hcatalog/build.gradle
index 364c10fa738..d07904f3465 100644
--- a/sdks/java/io/hcatalog/build.gradle
+++ b/sdks/java/io/hcatalog/build.gradle
@@ -33,7 +33,7 @@ def hadoopVersions = [
"2102": "2.10.2",
"324": "3.2.4",
"336": "3.3.6",
- "341": "3.4.1",
+ // "341": "3.4.1", // tests already exercised on the default version
]
hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
@@ -71,13 +71,21 @@ dependencies {
testRuntimeOnly project(path: ":runners:direct-java", configuration:
"shadow")
hadoopVersions.each {kv ->
"hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"
+ "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-hdfs:$kv.value"
+ "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
+ "hadoopVersion$kv.key"
"org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
}
}
hadoopVersions.each {kv ->
configurations."hadoopVersion$kv.key" {
resolutionStrategy {
+ force "org.apache.hadoop:hadoop-client:$kv.value"
force "org.apache.hadoop:hadoop-common:$kv.value"
+ force "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
+ force "org.apache.hadoop:hadoop-minicluster:$kv.value"
+ force "org.apache.hadoop:hadoop-hdfs:$kv.value"
+ force "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
}
}
}
diff --git a/sdks/java/io/iceberg/build.gradle
b/sdks/java/io/iceberg/build.gradle
index a2d192b6720..fa1e2426ce6 100644
--- a/sdks/java/io/iceberg/build.gradle
+++ b/sdks/java/io/iceberg/build.gradle
@@ -71,6 +71,9 @@ dependencies {
testRuntimeOnly project(path: ":runners:google-cloud-dataflow-java")
hadoopVersions.each {kv ->
"hadoopVersion$kv.key" "org.apache.hadoop:hadoop-client:$kv.value"
+ "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-minicluster:$kv.value"
+ "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
+ "hadoopVersion$kv.key"
"org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
}
}
@@ -78,6 +81,11 @@ hadoopVersions.each {kv ->
configurations."hadoopVersion$kv.key" {
resolutionStrategy {
force "org.apache.hadoop:hadoop-client:$kv.value"
+ force "org.apache.hadoop:hadoop-common:$kv.value"
+ force "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
+ force "org.apache.hadoop:hadoop-minicluster:$kv.value"
+ force "org.apache.hadoop:hadoop-hdfs:$kv.value"
+ force "org.apache.hadoop:hadoop-hdfs-client:$kv.value"
}
}
}
