This is an automated email from the ASF dual-hosted git repository.
xianjingfeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 2808d25dd [#1277] chore: add flatten maven plugin (#1278)
2808d25dd is described below
commit 2808d25dd0fcbef891d24e06b242b9bd5f04ff47
Author: xianjingfeng <[email protected]>
AuthorDate: Mon Oct 30 16:19:09 2023 +0800
[#1277] chore: add flatten maven plugin (#1278)
### What changes were proposed in this pull request?
add flatten maven plugin
### Why are the changes needed?
If we add shaded module for clients, we must add flatten-maven-plugin, or
the dependences in not-shaded module will not be included in the shaded jar.
Fix: #1277
### Does this PR introduce any user-facing change?
No.
### How was this patch tested?
CI
---
.gitignore | 1 +
client-mr/core/pom.xml | 5 +++--
client-spark/spark2/pom.xml | 9 +++++----
client-spark/spark3/pom.xml | 5 +++--
client-tez/pom.xml | 8 ++++++++
integration-test/mr/pom.xml | 2 ++
integration-test/spark-common/pom.xml | 4 ++++
integration-test/spark2/pom.xml | 21 +++------------------
integration-test/spark3/pom.xml | 5 +++--
integration-test/tez/pom.xml | 5 +++++
pom.xml | 28 ++++++++++++++++++++++++++++
11 files changed, 65 insertions(+), 28 deletions(-)
diff --git a/.gitignore b/.gitignore
index 359b8e83c..c1bd43acc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -50,3 +50,4 @@ rust/experimental/server/target
rust/experimental/server/.idea
rust/experimental/server/._target
rust/experimental/server/src/proto/uniffle.rs
+.flattened-pom.xml
diff --git a/client-mr/core/pom.xml b/client-mr/core/pom.xml
index 5167be076..f8ac47bce 100644
--- a/client-mr/core/pom.xml
+++ b/client-mr/core/pom.xml
@@ -114,6 +114,7 @@
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>hadoop${hadoop.short.version}-shim</artifactId>
+ <version>${project.version}</version>
</dependency>
</dependencies>
@@ -215,7 +216,7 @@
<configuration>
<target>
<echo message="Shade netty native libraries to
${rss.shade.packageName}" />
- <unzip src="${artifactId}-${version}.jar"
dest="${project.build.directory}/unpacked/" />
+ <unzip
src="${project.artifactId}-${project.version}.jar"
dest="${project.build.directory}/unpacked/" />
<echo message="renaming native epoll
library"></echo>
<move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
@@ -236,7 +237,7 @@
type="glob"></mapper>
</move>
<echo message="repackaging netty jar"></echo>
- <jar destfile="${artifactId}-${version}.jar"
basedir="${project.build.directory}/unpacked" />
+ <jar
destfile="${project.artifactId}-${project.version}.jar"
basedir="${project.build.directory}/unpacked" />
</target>
</configuration>
</execution>
diff --git a/client-spark/spark2/pom.xml b/client-spark/spark2/pom.xml
index ddb250821..810851d24 100644
--- a/client-spark/spark2/pom.xml
+++ b/client-spark/spark2/pom.xml
@@ -27,9 +27,7 @@
<relativePath>../../pom.xml</relativePath>
</parent>
- <groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark2</artifactId>
- <version>0.9.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Uniffle Client (Spark 2)</name>
@@ -41,10 +39,12 @@
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -74,6 +74,7 @@
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -182,7 +183,7 @@
<configuration>
<target>
<echo message="Shade netty native libraries to
${rss.shade.packageName}" />
- <unzip src="${artifactId}-${version}.jar"
dest="${project.build.directory}/unpacked/" />
+ <unzip src="${project.artifactId}-${project.version}.jar"
dest="${project.build.directory}/unpacked/" />
<echo message="renaming native epoll library"></echo>
<move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
@@ -203,7 +204,7 @@
type="glob"></mapper>
</move>
<echo message="repackaging netty jar"></echo>
- <jar destfile="${artifactId}-${version}.jar"
basedir="${project.build.directory}/unpacked" />
+ <jar destfile="${project.artifactId}-${project.version}.jar"
basedir="${project.build.directory}/unpacked" />
</target>
</configuration>
</execution>
diff --git a/client-spark/spark3/pom.xml b/client-spark/spark3/pom.xml
index 153408a90..f39d20773 100644
--- a/client-spark/spark3/pom.xml
+++ b/client-spark/spark3/pom.xml
@@ -27,9 +27,7 @@
<relativePath>../../pom.xml</relativePath>
</parent>
- <groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark3</artifactId>
- <version>0.9.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Uniffle Client (Spark 3)</name>
@@ -48,10 +46,12 @@
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
@@ -65,6 +65,7 @@
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
diff --git a/client-tez/pom.xml b/client-tez/pom.xml
index 4a5f09ced..edab39797 100644
--- a/client-tez/pom.xml
+++ b/client-tez/pom.xml
@@ -37,10 +37,12 @@
<dependency>
<groupId>org.apache.tez</groupId>
<artifactId>tez-common</artifactId>
+ <version>${tez.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
<artifactId>tez-runtime-library</artifactId>
+ <version>${tez.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -51,14 +53,17 @@
<dependency>
<groupId>org.apache.tez</groupId>
<artifactId>tez-runtime-internals</artifactId>
+ <version>${tez.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
<artifactId>tez-dag</artifactId>
+ <version>${tez.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
<artifactId>tez-api</artifactId>
+ <version>${tez.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -83,6 +88,7 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
+ <version>${protobuf.version}</version>
</dependency>
<dependency>
@@ -98,6 +104,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
@@ -108,6 +115,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
+ <version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
diff --git a/integration-test/mr/pom.xml b/integration-test/mr/pom.xml
index 26e5667cc..fb788886d 100644
--- a/integration-test/mr/pom.xml
+++ b/integration-test/mr/pom.xml
@@ -43,6 +43,7 @@
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-integration-common-test</artifactId>
+ <version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -106,6 +107,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-examples</artifactId>
+ <version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
diff --git a/integration-test/spark-common/pom.xml
b/integration-test/spark-common/pom.xml
index 08edecee2..b2193f854 100644
--- a/integration-test/spark-common/pom.xml
+++ b/integration-test/spark-common/pom.xml
@@ -75,23 +75,27 @@
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-integration-common-test</artifactId>
+ <version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark${client.type}</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/integration-test/spark2/pom.xml b/integration-test/spark2/pom.xml
index e43a3559c..b0733952e 100644
--- a/integration-test/spark2/pom.xml
+++ b/integration-test/spark2/pom.xml
@@ -27,9 +27,7 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.uniffle</groupId>
<artifactId>rss-integration-spark2-test</artifactId>
- <version>0.9.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Uniffle Integration Test (Spark 2)</name>
@@ -38,17 +36,20 @@
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-integration-spark-common-test</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark2</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
@@ -91,22 +92,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>shuffle-storage</artifactId>
diff --git a/integration-test/spark3/pom.xml b/integration-test/spark3/pom.xml
index d405367be..c089e040c 100644
--- a/integration-test/spark3/pom.xml
+++ b/integration-test/spark3/pom.xml
@@ -27,9 +27,7 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.uniffle</groupId>
<artifactId>rss-integration-spark3-test</artifactId>
- <version>0.9.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Uniffle Integration Test (Spark 3)</name>
@@ -38,17 +36,20 @@
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-integration-spark-common-test</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark3</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
diff --git a/integration-test/tez/pom.xml b/integration-test/tez/pom.xml
index f471c603f..ba65a8541 100644
--- a/integration-test/tez/pom.xml
+++ b/integration-test/tez/pom.xml
@@ -42,6 +42,7 @@
<dependency>
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-integration-common-test</artifactId>
+ <version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -64,6 +65,7 @@
<dependency>
<groupId>org.apache.tez</groupId>
<artifactId>tez-dag</artifactId>
+ <version>${tez.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
@@ -83,6 +85,7 @@
<dependency>
<groupId>org.apache.tez</groupId>
<artifactId>tez-tests</artifactId>
+ <version>${tez.version}</version>
<type>test-jar</type>
<scope>test</scope>
<exclusions>
@@ -119,11 +122,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
+ <version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
+ <version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<!-- tez-0.9.1 depends on guava 11.0.2, is not compatibility with uniffle
and hadoop-3.2.1 (guava-27.0-jre) until
diff --git a/pom.xml b/pom.xml
index b4431109a..808e5b2ac 100644
--- a/pom.xml
+++ b/pom.xml
@@ -779,6 +779,30 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>flatten-maven-plugin</artifactId>
+ <version>1.3.0</version>
+ <configuration>
+ <flattenMode>oss</flattenMode>
+ </configuration>
+ <executions>
+ <execution>
+ <id>flatten</id>
+ <goals>
+ <goal>flatten</goal>
+ </goals>
+ <phase>process-resources</phase>
+ </execution>
+ <execution>
+ <id>flatten.clean</id>
+ <goals>
+ <goal>clean</goal>
+ </goals>
+ <phase>clean</phase>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
@@ -1061,6 +1085,10 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>flatten-maven-plugin</artifactId>
+ </plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
