This is an automated email from the ASF dual-hosted git repository.
benjobs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/dev by this push:
new 98ce2b3a6 [Improve] Bump Hadoop Version to 3.3.4 (#2650)
98ce2b3a6 is described below
commit 98ce2b3a605b6b7d71b278b7cdc4dfa0b17ea89e
Author: zhoulii <[email protected]>
AuthorDate: Thu Apr 20 12:56:53 2023 +0800
[Improve] Bump Hadoop Version to 3.3.4 (#2650)
Co-authored-by: zhoulii <[email protected]>
---
pom.xml | 200 ++-------------------
streampark-common/pom.xml | 10 +-
.../streampark-console-service/pom.xml | 29 +--
.../streampark-flink-client-api/pom.xml | 4 +-
.../streampark-flink-client-core/pom.xml | 8 +-
.../streampark-flink-connector-hbase/pom.xml | 13 +-
.../streampark-flink-kubernetes/pom.xml | 9 +-
.../streampark-flink-shims-base/pom.xml | 8 +-
.../streampark-flink-shims_flink-1.16/pom.xml | 8 +-
.../streampark-flink-shims_flink-1.17/pom.xml | 8 +-
streampark-spark/pom.xml | 4 +-
11 files changed, 68 insertions(+), 233 deletions(-)
diff --git a/pom.xml b/pom.xml
index 704a2bafc..51aae97ae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,7 +100,7 @@
<spark.version>3.2.0</spark.version>
<scalikejdbc.version>4.0.0</scalikejdbc.version>
<hive.version>2.3.4</hive.version>
- <hadoop.version>2.10.2</hadoop.version>
+ <hadoop.version>3.3.4</hadoop.version>
<hbase.version>2.1.10</hbase.version>
<redis.version>3.3.0</redis.version>
<es.version>6.2.3</es.version>
@@ -117,8 +117,8 @@
<snakeyaml.version>2.0</snakeyaml.version>
<typesafe-conf.version>1.4.2</typesafe-conf.version>
<json4s-jackson.version>4.0.6</json4s-jackson.version>
- <hbase-client.version>1.3.5</hbase-client.version>
<commons-cli.version>1.3.1</commons-cli.version>
+ <commons-net.version>3.9.0</commons-net.version>
<commons-lang3.version>3.8.1</commons-lang3.version>
<enumeratum.version>1.6.1</enumeratum.version>
@@ -298,216 +298,50 @@
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${httpclient.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ <version>${commons-net.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
- <groupId>*</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </exclusion>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>*</artifactId>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
</exclusion>
<exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>*</artifactId>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<!--hbase-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>reload4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
- <version>${hbase-client.version}</version>
+ <version>${hbase.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
diff --git a/streampark-common/pom.xml b/streampark-common/pom.xml
index 9f20548b1..b9a5e646e 100644
--- a/streampark-common/pom.xml
+++ b/streampark-common/pom.xml
@@ -78,12 +78,6 @@
</dependency>
<!--hbase-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <optional>true</optional>
- </dependency>
-
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
@@ -92,13 +86,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<optional>true</optional>
</dependency>
diff --git a/streampark-console/streampark-console-service/pom.xml
b/streampark-console/streampark-console-service/pom.xml
index 37443e1aa..288028831 100644
--- a/streampark-console/streampark-console-service/pom.xml
+++ b/streampark-console/streampark-console-service/pom.xml
@@ -114,6 +114,11 @@
<version>${commons-compress.version}</version>
</dependency>
+ <dependency>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ </dependency>
+
<dependency>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
@@ -283,32 +288,12 @@
<!-- hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
</dependency>
<dependency>
diff --git
a/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml
b/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml
index 763c35190..27b15c455 100644
---
a/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml
+++
b/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml
@@ -62,13 +62,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
diff --git
a/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml
b/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml
index d843df2e5..7fb1fc282 100644
---
a/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml
+++
b/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml
@@ -60,7 +60,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
<scope>provided</scope>
</dependency>
diff --git
a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/pom.xml
b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/pom.xml
index 450e0f3be..3f15f4e20 100644
---
a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/pom.xml
+++
b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/pom.xml
@@ -80,13 +80,12 @@
<!--hbase-->
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
+ <artifactId>hadoop-client-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
</dependency>
<dependency>
diff --git a/streampark-flink/streampark-flink-kubernetes/pom.xml
b/streampark-flink/streampark-flink-kubernetes/pom.xml
index 17c1fd70c..a3495ceb2 100644
--- a/streampark-flink/streampark-flink-kubernetes/pom.xml
+++ b/streampark-flink/streampark-flink-kubernetes/pom.xml
@@ -110,20 +110,19 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>provided</scope>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
</dependency>
<dependency>
diff --git
a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/pom.xml
b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/pom.xml
index 5cb843558..776aad238 100644
---
a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/pom.xml
+++
b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/pom.xml
@@ -96,7 +96,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
<optional>true</optional>
</dependency>
</dependencies>
diff --git
a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/pom.xml
b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/pom.xml
index f75b89955..7ae37cb45 100644
---
a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/pom.xml
+++
b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/pom.xml
@@ -98,7 +98,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
<optional>true</optional>
</dependency>
diff --git
a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/pom.xml
b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/pom.xml
index 1334a304f..357477db2 100644
---
a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/pom.xml
+++
b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/pom.xml
@@ -98,7 +98,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
<optional>true</optional>
</dependency>
diff --git a/streampark-spark/pom.xml b/streampark-spark/pom.xml
index bef4c67d3..df95c37b6 100644
--- a/streampark-spark/pom.xml
+++ b/streampark-spark/pom.xml
@@ -68,13 +68,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<scope>provided</scope>
</dependency>
