This is an automated email from the ASF dual-hosted git repository.
wanghailin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new a7837f1f19 [Hotfix][Zeta] Fix the dependency conflict between the
guava in hadoop-aws and hive-exec (#7986)
a7837f1f19 is described below
commit a7837f1f199db266861de34f47ff46b514752f5a
Author: dailai <[email protected]>
AuthorDate: Thu Nov 7 17:34:32 2024 +0800
[Hotfix][Zeta] Fix the dependency conflict between the guava in hadoop-aws
and hive-exec (#7986)
---
pom.xml | 2 +
.../connector-file/connector-file-s3/pom.xml | 27 +-----
seatunnel-dist/pom.xml | 16 +++-
.../src/main/assembly/assembly-bin-ci.xml | 4 +-
seatunnel-dist/src/main/assembly/assembly-bin.xml | 13 +++
seatunnel-shade/pom.xml | 1 +
seatunnel-shade/seatunnel-hadoop-aws/pom.xml | 106 +++++++++++++++++++++
7 files changed, 140 insertions(+), 29 deletions(-)
diff --git a/pom.xml b/pom.xml
index f9d59c79ab..872e94112b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -158,6 +158,8 @@
<prometheus.simpleclient.version>0.16.0</prometheus.simpleclient.version>
<enableSourceJarCreation>true</enableSourceJarCreation>
+ <hadoop-aws.version>3.1.4</hadoop-aws.version>
+
</properties>
<dependencyManagement>
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml
b/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml
index fbf0016fce..82ddae46e6 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml
+++ b/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml
@@ -29,21 +29,6 @@
<artifactId>connector-file-s3</artifactId>
<name>SeaTunnel : Connectors V2 : File : S3</name>
- <properties>
- <hadoop-aws.version>3.1.4</hadoop-aws.version>
- <guava.version>27.0-jre</guava.version>
- </properties>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
<dependencies>
<dependency>
@@ -67,14 +52,10 @@
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aws</artifactId>
- <version>${hadoop-aws.version}</version>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-hadoop-aws</artifactId>
+ <version>${project.version}</version>
+ <classifier>optional</classifier>
<scope>provided</scope>
<exclusions>
<exclusion>
diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml
index fdde84ba8c..e1e77616f0 100644
--- a/seatunnel-dist/pom.xml
+++ b/seatunnel-dist/pom.xml
@@ -104,7 +104,6 @@
<!-- Imap storage dependency package -->
<hadoop-aliyun.version>3.1.4</hadoop-aliyun.version>
<json-smart.version>2.4.7</json-smart.version>
- <hadoop-aws.version>3.1.4</hadoop-aws.version>
<aws-java-sdk.version>1.11.271</aws-java-sdk.version>
<netty-buffer.version>4.1.89.Final</netty-buffer.version>
<hive.exec.version>3.1.3</hive.exec.version>
@@ -735,9 +734,10 @@
<!-- hadoop jar -->
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aws</artifactId>
- <version>${hadoop-aws.version}</version>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-hadoop-aws</artifactId>
+ <version>${project.version}</version>
+ <classifier>optional</classifier>
<scope>provided</scope>
</dependency>
<dependency>
@@ -960,6 +960,14 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-hadoop-aws</artifactId>
+ <version>${project.version}</version>
+ <classifier>optional</classifier>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
<finalName>apache-seatunnel-${project.version}</finalName>
diff --git a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
index 4510579d81..2ed7f180b7 100644
--- a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
+++ b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
@@ -161,7 +161,7 @@
<scope>provided</scope>
</dependencySet>
- <!-- =================== JDBC Connector Drivers And SeaTunnel Hadoop3
Uber Jar =================== -->
+ <!-- =================== JDBC Connector Drivers, SeaTunnel Hadoop3
Uber Jar And SeaTunnel Hadoop AWS Uber Jar =================== -->
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<useTransitiveDependencies>true</useTransitiveDependencies>
@@ -181,9 +181,9 @@
<include>com.amazon.redshift:redshift-jdbc42:jar</include>
<include>net.snowflake.snowflake-jdbc:jar</include>
<include>com.xugudb:xugu-jdbc:jar</include>
- <include>org.apache.hadoop:hadoop-aws:jar</include>
<include>com.amazonaws:aws-java-sdk-bundle:jar</include>
<include>org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:jar:*:optional</include>
+
<include>org.apache.seatunnel:seatunnel-hadoop-aws:jar:*:optional</include>
<!--Add hadoop aliyun jar -->
<include>org.apache.hadoop:hadoop-aliyun:jar</include>
<include>com.aliyun.oss:aliyun-sdk-oss:jar</include>
diff --git a/seatunnel-dist/src/main/assembly/assembly-bin.xml
b/seatunnel-dist/src/main/assembly/assembly-bin.xml
index f16841f7a9..cc91a3c47b 100644
--- a/seatunnel-dist/src/main/assembly/assembly-bin.xml
+++ b/seatunnel-dist/src/main/assembly/assembly-bin.xml
@@ -174,6 +174,19 @@
<scope>provided</scope>
</dependencySet>
+ <!-- ============ SeaTunnel Hadoop-AWS Uber Jar============ -->
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <unpack>false</unpack>
+ <includes>
+
<include>org.apache.seatunnel:seatunnel-hadoop-aws:jar:*:optional</include>
+ </includes>
+
<outputFileNameMapping>${artifact.file.name}</outputFileNameMapping>
+ <outputDirectory>/lib</outputDirectory>
+ <scope>provided</scope>
+ </dependencySet>
+
<!-- ============ Connectors Jars And Transforms V2 Jar ============
-->
<!-- SeaTunnel connectors for Demo -->
<dependencySet>
diff --git a/seatunnel-shade/pom.xml b/seatunnel-shade/pom.xml
index dafc7eaea4..2b378739de 100644
--- a/seatunnel-shade/pom.xml
+++ b/seatunnel-shade/pom.xml
@@ -35,6 +35,7 @@
<module>seatunnel-hazelcast</module>
<module>seatunnel-janino</module>
<module>seatunnel-jetty9-9.4.56</module>
+ <module>seatunnel-hadoop-aws</module>
</modules>
diff --git a/seatunnel-shade/seatunnel-hadoop-aws/pom.xml
b/seatunnel-shade/seatunnel-hadoop-aws/pom.xml
new file mode 100644
index 0000000000..b37c06e4a0
--- /dev/null
+++ b/seatunnel-shade/seatunnel-hadoop-aws/pom.xml
@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-shade</artifactId>
+ <version>${revision}</version>
+ </parent>
+
+ <artifactId>seatunnel-hadoop-aws</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop-aws.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <finalName>seatunnel-hadoop-aws</finalName>
+
<createSourcesJar>${enableSourceJarCreation}</createSourcesJar>
+ <shadeSourcesContent>true</shadeSourcesContent>
+
<shadedArtifactAttached>false</shadedArtifactAttached>
+
<createDependencyReducedPom>false</createDependencyReducedPom>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+
<shadedPattern>${seatunnel.shade.package}.com.google.common</shadedPattern>
+ <includes>
+
<include>com.google.common.base.*</include>
+
<include>com.google.common.cache.*</include>
+
<include>com.google.common.collect.*</include>
+ </includes>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <artifacts>
+ <artifact>
+
<file>${basedir}/target/seatunnel-hadoop-aws.jar</file>
+ <type>jar</type>
+ <classifier>optional</classifier>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
