This is an automated email from the ASF dual-hosted git repository. xianjingfeng pushed a commit to branch branch-0.8 in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
commit 5b57bf34a6892a060c5df18dd0ce4fcf741d4e32 Author: xianjingfeng <[email protected]> AuthorDate: Mon Oct 30 16:19:09 2023 +0800 [#1277] chore: add flatten maven plugin (#1278) ### What changes were proposed in this pull request? add flatten maven plugin ### Why are the changes needed? If we add shaded module for clients, we must add flatten-maven-plugin, or the dependences in not-shaded module will not be included in the shaded jar. Fix: #1277 ### Does this PR introduce any user-facing change? No. ### How was this patch tested? CI (cherry picked from commit 2808d25dd0fcbef891d24e06b242b9bd5f04ff47) --- .gitignore | 1 + client-mr/core/pom.xml | 5 +++-- client-spark/spark2/pom.xml | 9 +++++---- client-spark/spark3/pom.xml | 5 +++-- client-tez/pom.xml | 8 ++++++++ integration-test/mr/pom.xml | 2 ++ integration-test/spark-common/pom.xml | 4 ++++ integration-test/spark2/pom.xml | 21 +++------------------ integration-test/spark3/pom.xml | 5 +++-- integration-test/tez/pom.xml | 5 +++++ pom.xml | 28 ++++++++++++++++++++++++++++ 11 files changed, 65 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index b6901d7ce..2a1754c2e 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,4 @@ build/ deploy/kubernetes/integration-test/e2e/rss.yaml deploy/kubernetes/integration-test/e2e/rss-controller.yaml deploy/kubernetes/integration-test/e2e/rss-webhook.yaml +.flattened-pom.xml diff --git a/client-mr/core/pom.xml b/client-mr/core/pom.xml index c5a9906d6..223bea2d7 100644 --- a/client-mr/core/pom.xml +++ b/client-mr/core/pom.xml @@ -114,6 +114,7 @@ <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>hadoop${hadoop.short.version}-shim</artifactId> + <version>${project.version}</version> </dependency> </dependencies> @@ -215,7 +216,7 @@ <configuration> <target> <echo message="Shade netty native libraries to ${rss.shade.packageName}" /> - <unzip src="${artifactId}-${version}.jar" dest="${project.build.directory}/unpacked/" /> + <unzip src="${project.artifactId}-${project.version}.jar" dest="${project.build.directory}/unpacked/" /> <echo message="renaming native epoll library"></echo> <move includeemptydirs="false" todir="${project.build.directory}/unpacked/META-INF/native"> <fileset dir="${project.build.directory}/unpacked/META-INF/native"></fileset> @@ -236,7 +237,7 @@ type="glob"></mapper> </move> <echo message="repackaging netty jar"></echo> - <jar destfile="${artifactId}-${version}.jar" basedir="${project.build.directory}/unpacked" /> + <jar destfile="${project.artifactId}-${project.version}.jar" basedir="${project.build.directory}/unpacked" /> </target> </configuration> </execution> diff --git a/client-spark/spark2/pom.xml b/client-spark/spark2/pom.xml index b24438272..e5ba9d220 100644 --- a/client-spark/spark2/pom.xml +++ b/client-spark/spark2/pom.xml @@ -27,9 +27,7 @@ <relativePath>../../pom.xml</relativePath> </parent> - <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark2</artifactId> - <version>0.8.0</version> <packaging>jar</packaging> <name>Apache Uniffle Client (Spark 2)</name> @@ -41,10 +39,12 @@ <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark-common</artifactId> + <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_${scala.binary.version}</artifactId> + <version>${spark.version}</version> <scope>provided</scope> </dependency> <dependency> @@ -74,6 +74,7 @@ <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark-common</artifactId> + <version>${project.version}</version> <type>test-jar</type> <scope>test</scope> </dependency> @@ -182,7 +183,7 @@ <configuration> <target> <echo message="Shade netty native libraries to ${rss.shade.packageName}" /> - <unzip src="${artifactId}-${version}.jar" dest="${project.build.directory}/unpacked/" /> + <unzip src="${project.artifactId}-${project.version}.jar" dest="${project.build.directory}/unpacked/" /> <echo message="renaming native epoll library"></echo> <move includeemptydirs="false" todir="${project.build.directory}/unpacked/META-INF/native"> <fileset dir="${project.build.directory}/unpacked/META-INF/native"></fileset> @@ -203,7 +204,7 @@ type="glob"></mapper> </move> <echo message="repackaging netty jar"></echo> - <jar destfile="${artifactId}-${version}.jar" basedir="${project.build.directory}/unpacked" /> + <jar destfile="${project.artifactId}-${project.version}.jar" basedir="${project.build.directory}/unpacked" /> </target> </configuration> </execution> diff --git a/client-spark/spark3/pom.xml b/client-spark/spark3/pom.xml index fba4786f3..53e68b165 100644 --- a/client-spark/spark3/pom.xml +++ b/client-spark/spark3/pom.xml @@ -27,9 +27,7 @@ <relativePath>../../pom.xml</relativePath> </parent> - <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark3</artifactId> - <version>0.8.0</version> <packaging>jar</packaging> <name>Apache Uniffle Client (Spark 3)</name> @@ -48,10 +46,12 @@ <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client</artifactId> + <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark-common</artifactId> + <version>${project.version}</version> </dependency> <dependency> <groupId>io.netty</groupId> @@ -65,6 +65,7 @@ <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark-common</artifactId> + <version>${project.version}</version> <type>test-jar</type> <scope>test</scope> </dependency> diff --git a/client-tez/pom.xml b/client-tez/pom.xml index d31a8283b..09364005a 100644 --- a/client-tez/pom.xml +++ b/client-tez/pom.xml @@ -37,10 +37,12 @@ <dependency> <groupId>org.apache.tez</groupId> <artifactId>tez-common</artifactId> + <version>${tez.version}</version> </dependency> <dependency> <groupId>org.apache.tez</groupId> <artifactId>tez-runtime-library</artifactId> + <version>${tez.version}</version> <exclusions> <exclusion> <groupId>org.apache.hadoop</groupId> @@ -51,14 +53,17 @@ <dependency> <groupId>org.apache.tez</groupId> <artifactId>tez-runtime-internals</artifactId> + <version>${tez.version}</version> </dependency> <dependency> <groupId>org.apache.tez</groupId> <artifactId>tez-dag</artifactId> + <version>${tez.version}</version> </dependency> <dependency> <groupId>org.apache.tez</groupId> <artifactId>tez-api</artifactId> + <version>${tez.version}</version> <exclusions> <exclusion> <groupId>org.apache.hadoop</groupId> @@ -83,6 +88,7 @@ <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> + <version>${protobuf.version}</version> </dependency> <dependency> @@ -98,6 +104,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> <exclusions> <exclusion> <groupId>com.google.protobuf</groupId> @@ -108,6 +115,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> + <version>${hadoop.version}</version> <scope>test</scope> <exclusions> <exclusion> diff --git a/integration-test/mr/pom.xml b/integration-test/mr/pom.xml index 431fc82ee..457c67ce1 100644 --- a/integration-test/mr/pom.xml +++ b/integration-test/mr/pom.xml @@ -43,6 +43,7 @@ <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-integration-common-test</artifactId> + <version>${project.version}</version> <type>test-jar</type> <scope>test</scope> </dependency> @@ -106,6 +107,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-examples</artifactId> + <version>${hadoop.version}</version> <scope>test</scope> </dependency> diff --git a/integration-test/spark-common/pom.xml b/integration-test/spark-common/pom.xml index 53d34e963..385dbee3b 100644 --- a/integration-test/spark-common/pom.xml +++ b/integration-test/spark-common/pom.xml @@ -75,23 +75,27 @@ <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-integration-common-test</artifactId> + <version>${project.version}</version> <type>test-jar</type> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark-common</artifactId> + <version>${project.version}</version> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark-common</artifactId> + <version>${project.version}</version> <scope>test</scope> <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark${client.type}</artifactId> + <version>${project.version}</version> <scope>test</scope> </dependency> <dependency> diff --git a/integration-test/spark2/pom.xml b/integration-test/spark2/pom.xml index 5db551e32..83d0d5ef4 100644 --- a/integration-test/spark2/pom.xml +++ b/integration-test/spark2/pom.xml @@ -27,9 +27,7 @@ </parent> <modelVersion>4.0.0</modelVersion> - <groupId>org.apache.uniffle</groupId> <artifactId>rss-integration-spark2-test</artifactId> - <version>0.8.0</version> <packaging>jar</packaging> <name>Apache Uniffle Integration Test (Spark 2)</name> @@ -38,17 +36,20 @@ <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-integration-spark-common-test</artifactId> + <version>${project.version}</version> <scope>test</scope> <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark2</artifactId> + <version>${project.version}</version> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark-common</artifactId> + <version>${project.version}</version> <scope>test</scope> <exclusions> <exclusion> @@ -91,22 +92,6 @@ <type>test-jar</type> <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-core_${scala.binary.version}</artifactId> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>jackson-module-scala_${scala.binary.version}</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-sql_${scala.binary.version}</artifactId> - <scope>test</scope> - </dependency> <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>shuffle-storage</artifactId> diff --git a/integration-test/spark3/pom.xml b/integration-test/spark3/pom.xml index b3fcd2991..1a6bf1ef4 100644 --- a/integration-test/spark3/pom.xml +++ b/integration-test/spark3/pom.xml @@ -27,9 +27,7 @@ </parent> <modelVersion>4.0.0</modelVersion> - <groupId>org.apache.uniffle</groupId> <artifactId>rss-integration-spark3-test</artifactId> - <version>0.8.0</version> <packaging>jar</packaging> <name>Apache Uniffle Integration Test (Spark 3)</name> @@ -38,17 +36,20 @@ <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-integration-spark-common-test</artifactId> + <version>${project.version}</version> <scope>test</scope> <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark3</artifactId> + <version>${project.version}</version> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-client-spark-common</artifactId> + <version>${project.version}</version> <scope>test</scope> <exclusions> <exclusion> diff --git a/integration-test/tez/pom.xml b/integration-test/tez/pom.xml index 730d022dc..0d6e4fbd8 100644 --- a/integration-test/tez/pom.xml +++ b/integration-test/tez/pom.xml @@ -42,6 +42,7 @@ <dependency> <groupId>org.apache.uniffle</groupId> <artifactId>rss-integration-common-test</artifactId> + <version>${project.version}</version> <type>test-jar</type> <scope>test</scope> </dependency> @@ -64,6 +65,7 @@ <dependency> <groupId>org.apache.tez</groupId> <artifactId>tez-dag</artifactId> + <version>${tez.version}</version> <scope>test</scope> <exclusions> <exclusion> @@ -83,6 +85,7 @@ <dependency> <groupId>org.apache.tez</groupId> <artifactId>tez-tests</artifactId> + <version>${tez.version}</version> <type>test-jar</type> <scope>test</scope> <exclusions> @@ -119,11 +122,13 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-api</artifactId> + <version>${hadoop.version}</version> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-common</artifactId> + <version>${hadoop.version}</version> <scope>test</scope> </dependency> <!-- tez-0.9.1 depends on guava 11.0.2, is not compatibility with uniffle and hadoop-3.2.1 (guava-27.0-jre) until diff --git a/pom.xml b/pom.xml index c650a2cef..6d2458105 100644 --- a/pom.xml +++ b/pom.xml @@ -779,6 +779,30 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>flatten-maven-plugin</artifactId> + <version>1.3.0</version> + <configuration> + <flattenMode>oss</flattenMode> + </configuration> + <executions> + <execution> + <id>flatten</id> + <goals> + <goal>flatten</goal> + </goals> + <phase>process-resources</phase> + </execution> + <execution> + <id>flatten.clean</id> + <goals> + <goal>clean</goal> + </goals> + <phase>clean</phase> + </execution> + </executions> + </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> @@ -1061,6 +1085,10 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>flatten-maven-plugin</artifactId> + </plugin> <plugin> <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId>
