This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8dc1064c99a [SPARK-39863][BUILD] Upgrade Hadoop to 3.3.4
8dc1064c99a is described below
commit 8dc1064c99a25b2eb84bf04b2a2b6b4281c55d3a
Author: Chao Sun <[email protected]>
AuthorDate: Mon Aug 8 23:10:35 2022 -0700
[SPARK-39863][BUILD] Upgrade Hadoop to 3.3.4
### What changes were proposed in this pull request?
This PR aims to upgrade to Hadoop 3.3.4, which was just announced today.
### Why are the changes needed?
Hadoop 3.3.4 comes with many bug fixes as well as CVE fixes. Please check
[release
notes](http://hadoop.apache.org/docs/r3.3.4/hadoop-project-dist/hadoop-common/release/3.3.4/RELEASENOTES.3.3.4.html)
and [change
log](http://hadoop.apache.org/docs/r3.3.4/hadoop-project-dist/hadoop-common/release/3.3.4/CHANGELOG.3.3.4.html).
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs.
Closes #37281 from sunchao/SPARK-39863/hadoop-3.3.4.
Authored-by: Chao Sun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
dev/deps/spark-deps-hadoop-3-hive-2.3 | 24 ++++++++++------------
pom.xml | 2 +-
.../sql/hive/client/IsolatedClientLoader.scala | 2 +-
3 files changed, 13 insertions(+), 15 deletions(-)
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3
b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 1b3027cdae6..b3fdb12c0d9 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -25,7 +25,7 @@ automaton/1.11-8//automaton-1.11-8.jar
avro-ipc/1.11.0//avro-ipc-1.11.0.jar
avro-mapred/1.11.0//avro-mapred-1.11.0.jar
avro/1.11.0//avro-1.11.0.jar
-aws-java-sdk-bundle/1.11.1026//aws-java-sdk-bundle-1.11.1026.jar
+aws-java-sdk-bundle/1.12.132//aws-java-sdk-bundle-1.12.132.jar
azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar
azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar
azure-storage/7.0.1//azure-storage-7.0.1.jar
@@ -52,7 +52,6 @@ commons-math3/3.6.1//commons-math3-3.6.1.jar
commons-pool/1.5.4//commons-pool-1.5.4.jar
commons-text/1.9//commons-text-1.9.jar
compress-lzf/1.1//compress-lzf-1.1.jar
-cos_api-bundle/5.6.19//cos_api-bundle-5.6.19.jar
curator-client/2.13.0//curator-client-2.13.0.jar
curator-framework/2.13.0//curator-framework-2.13.0.jar
curator-recipes/2.13.0//curator-recipes-2.13.0.jar
@@ -66,18 +65,17 @@ generex/1.0.2//generex-1.0.2.jar
gmetric4j/1.0.10//gmetric4j-1.0.10.jar
gson/2.2.4//gson-2.2.4.jar
guava/14.0.1//guava-14.0.1.jar
-hadoop-aliyun/3.3.3//hadoop-aliyun-3.3.3.jar
-hadoop-annotations/3.3.3//hadoop-annotations-3.3.3.jar
-hadoop-aws/3.3.3//hadoop-aws-3.3.3.jar
-hadoop-azure-datalake/3.3.3//hadoop-azure-datalake-3.3.3.jar
-hadoop-azure/3.3.3//hadoop-azure-3.3.3.jar
-hadoop-client-api/3.3.3//hadoop-client-api-3.3.3.jar
-hadoop-client-runtime/3.3.3//hadoop-client-runtime-3.3.3.jar
-hadoop-cloud-storage/3.3.3//hadoop-cloud-storage-3.3.3.jar
-hadoop-cos/3.3.3//hadoop-cos-3.3.3.jar
-hadoop-openstack/3.3.3//hadoop-openstack-3.3.3.jar
+hadoop-aliyun/3.3.4//hadoop-aliyun-3.3.4.jar
+hadoop-annotations/3.3.4//hadoop-annotations-3.3.4.jar
+hadoop-aws/3.3.4//hadoop-aws-3.3.4.jar
+hadoop-azure-datalake/3.3.4//hadoop-azure-datalake-3.3.4.jar
+hadoop-azure/3.3.4//hadoop-azure-3.3.4.jar
+hadoop-client-api/3.3.4//hadoop-client-api-3.3.4.jar
+hadoop-client-runtime/3.3.4//hadoop-client-runtime-3.3.4.jar
+hadoop-cloud-storage/3.3.4//hadoop-cloud-storage-3.3.4.jar
+hadoop-openstack/3.3.4//hadoop-openstack-3.3.4.jar
hadoop-shaded-guava/1.1.1//hadoop-shaded-guava-1.1.1.jar
-hadoop-yarn-server-web-proxy/3.3.3//hadoop-yarn-server-web-proxy-3.3.3.jar
+hadoop-yarn-server-web-proxy/3.3.4//hadoop-yarn-server-web-proxy-3.3.4.jar
hive-beeline/2.3.9//hive-beeline-2.3.9.jar
hive-cli/2.3.9//hive-cli-2.3.9.jar
hive-common/2.3.9//hive-common-2.3.9.jar
diff --git a/pom.xml b/pom.xml
index da1e3a8a1e5..d6404e80c98 100644
--- a/pom.xml
+++ b/pom.xml
@@ -115,7 +115,7 @@
<slf4j.version>1.7.36</slf4j.version>
<log4j.version>2.18.0</log4j.version>
<!-- make sure to update IsolatedClientLoader whenever this version is
changed -->
- <hadoop.version>3.3.3</hadoop.version>
+ <hadoop.version>3.3.4</hadoop.version>
<protobuf.version>2.5.0</protobuf.version>
<yarn.version>${hadoop.version}</yarn.version>
<zookeeper.version>3.6.2</zookeeper.version>
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index 12bb1b3631c..e65e6d42937 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -69,7 +69,7 @@ private[hive] object IsolatedClientLoader extends Logging {
// If the error message contains hadoop, it is probably because
the hadoop
// version cannot be resolved.
val fallbackVersion = if (VersionUtils.isHadoop3) {
- "3.3.3"
+ "3.3.4"
} else {
"2.7.4"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
