This is an automated email from the ASF dual-hosted git repository.
zhengchenyu pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new 41e173634 [#1735] fix(client): Shade spark client grpc/jetty and
others dependencies (#1878)
41e173634 is described below
commit 41e173634836f448dacb9a8cf6285ae9db542aa0
Author: maobaolong <[email protected]>
AuthorDate: Wed Jul 10 11:41:21 2024 +0800
[#1735] fix(client): Shade spark client grpc/jetty and others dependencies
(#1878)
### What changes were proposed in this pull request?
Relocate all package name of client dependencies. Excepts for the following
cases:
- Package belongs to uniffle
- Package provided by spark and hadoop env, something like
- spark
- hadoop
- dependencies belongs to the above, like lz4 belongs to spark.
- log4j related.
- part of javax package belongs to jdk
### Why are the changes needed?
Fix: #1735
Before this change
<img width="342" alt="image"
src="https://github.com/apache/incubator-uniffle/assets/17329931/bdd9487e-80f0-428f-967f-30eb21b05664";>
After this change
<img width="423" alt="image"
src="https://github.com/apache/incubator-uniffle/assets/17329931/71e24225-2cbd-49b6-ab53-d501923f99c3";>
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- Tested by a simple spark-shell which involved shuffle data writing and
reading.
- Tested by a spark sql which product huge pressure of our uniffle cluster.
---
client-spark/spark2-shaded/pom.xml | 136 +++++++++++++++++++++++++++++++------
client-spark/spark3-shaded/pom.xml | 136 +++++++++++++++++++++++++++++++------
2 files changed, 228 insertions(+), 44 deletions(-)
diff --git a/client-spark/spark2-shaded/pom.xml
b/client-spark/spark2-shaded/pom.xml
index 7bec7f99b..5b61ed2f8 100644
--- a/client-spark/spark2-shaded/pom.xml
+++ b/client-spark/spark2-shaded/pom.xml
@@ -35,7 +35,45 @@
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark2</artifactId>
<version>${project.version}</version>
+ <!-- use the lz4 from spark env -->
+ <exclusions>
+ <exclusion>
+ <artifactId>lz4</artifactId>
+ <groupId>net.jpountz.lz4</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-api</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
+ <!-- log4j2 -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <!-- Use log4j-slf4j-impl by default -->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <!-- end -->
</dependencies>
<build>
@@ -53,22 +91,7 @@
<configuration>
<artifactSet>
<includes>
- <include>org.apache.uniffle:*</include>
- <include>com.google.protobuf:*</include>
- <include>com.google.code.gson:gson</include>
- <include>io.grpc:*</include>
- <include>com.google.android:annotations</include>
- <include>io.perfmark:perfmark-api</include>
- <include>io.netty:*</include>
-
<include>com.google.api.grpc:proto-google-common-protos</include>
-
<include>org.codehaus.mojo:animal-sniffer-annotations</include>
- <include>com.google.guava:*</include>
-
<include>com.fasterxml.jackson.core:jackson-databind</include>
- <include>com.fasterxml.jackson.core:jackson-core</include>
-
<include>com.fasterxml.jackson.core:jackson-annotations</include>
- <include>org.roaringbitmap:RoaringBitmap</include>
- <include>org.roaringbitmap:shims</include>
- <include>org.apache.commons:commons-collections4</include>
+ <include>*:*</include>
</includes>
</artifactSet>
<finalName>${project.artifactId}-${project.version}</finalName>
@@ -88,17 +111,14 @@
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
+ <exclude>**/*.proto</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
- <pattern>com.google.protobuf</pattern>
-
<shadedPattern>${rss.shade.packageName}.com.google.protobuf</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.common</pattern>
-
<shadedPattern>${rss.shade.packageName}.com.google.common</shadedPattern>
+ <pattern>com.google</pattern>
+
<shadedPattern>${rss.shade.packageName}.com.google</shadedPattern>
</relocation>
<relocation>
<pattern>google.protobuf</pattern>
@@ -120,10 +140,82 @@
<pattern>io.netty</pattern>
<shadedPattern>${rss.shade.packageName}.io.netty</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>io.grpc</pattern>
+
<shadedPattern>${rss.shade.packageName}.io.grpc</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.perfmark</pattern>
+
<shadedPattern>${rss.shade.packageName}.io.perfmark</shadedPattern>
+ </relocation>
<relocation>
<pattern>org.roaringbitmap</pattern>
<shadedPattern>${rss.shade.packageName}.org.roaringbitmap</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>org.apache</pattern>
+
<shadedPattern>${rss.shade.packageName}.org.apache</shadedPattern>
+ <excludes>
+ <!-- Exclude the packages belonging to uniffle -->
+ <exclude>org/apache/uniffle/**/*</exclude>
+ <exclude>org/apache/hadoop/*</exclude>
+ <exclude>org/apache/hadoop/**/*</exclude>
+ <!-- Exclude the logging packages-->
+ <exclude>**/pom.xml</exclude>
+ <exclude>org/apache/commons/logging/*</exclude>
+ <exclude>org/apache/commons/logging/**/*</exclude>
+ <exclude>org/apache/log4j/*</exclude>
+ <exclude>org/apache/log4j/**/*</exclude>
+ <!-- Exclude spark packages -->
+ <exclude>org/apache/spark/**/*</exclude>
+ <!-- Exclude commons-io packages -->
+ <exclude>org/apache/commons/io/**/*</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>org.codehaus</pattern>
+
<shadedPattern>${rss.shade.packageName}.codehaus</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>android.annotation</pattern>
+
<shadedPattern>${rss.shade.packageName}.android.annotation</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javax</pattern>
+ <shadedPattern>${rss.shade.packageName}.javax</shadedPattern>
+ <includes>
+ <include>javax/annotation/**/*</include>
+ <include>javax/activation/**/*</include>
+ <include>javax/xml/**/*</include>
+ <include>javax/inject/**/*</include>
+ <include>javax/validation/**/*</include>
+ <include>javax/servlet/**/*</include>
+ </includes>
+ </relocation>
+ <relocation>
+ <pattern>javassist</pattern>
+
<shadedPattern>${rss.shade.packageName}.javassist</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.prometheus.client</pattern>
+
<shadedPattern>${rss.shade.packageName}.io.prometheus.client</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.sun.activation</pattern>
+
<shadedPattern>${rss.shade.packageName}.com.sun.activation</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.checkerframework</pattern>
+
<shadedPattern>${rss.shade.packageName}.org.checkerframework</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.eclipse</pattern>
+
<shadedPattern>${rss.shade.packageName}.org.eclipse</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>picocli</pattern>
+
<shadedPattern>${rss.shade.packageName}.picocli</shadedPattern>
+ </relocation>
</relocations>
</configuration>
</execution>
diff --git a/client-spark/spark3-shaded/pom.xml
b/client-spark/spark3-shaded/pom.xml
index d972b9d8d..6f8a2a34c 100644
--- a/client-spark/spark3-shaded/pom.xml
+++ b/client-spark/spark3-shaded/pom.xml
@@ -35,7 +35,45 @@
<groupId>org.apache.uniffle</groupId>
<artifactId>rss-client-spark3</artifactId>
<version>${project.version}</version>
+ <!-- use the lz4 from spark env -->
+ <exclusions>
+ <exclusion>
+ <artifactId>lz4</artifactId>
+ <groupId>net.jpountz.lz4</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-api</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
+ <!-- log4j2 -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <!-- Use log4j-slf4j-impl by default -->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <!-- end -->
</dependencies>
<build>
@@ -53,22 +91,7 @@
<configuration>
<artifactSet>
<includes>
- <include>org.apache.uniffle:*</include>
- <include>com.google.protobuf:*</include>
- <include>com.google.code.gson:gson</include>
- <include>io.grpc:*</include>
- <include>com.google.android:annotations</include>
- <include>io.perfmark:perfmark-api</include>
- <include>io.netty:*</include>
-
<include>com.google.api.grpc:proto-google-common-protos</include>
-
<include>org.codehaus.mojo:animal-sniffer-annotations</include>
- <include>com.google.guava:*</include>
-
<include>com.fasterxml.jackson.core:jackson-databind</include>
- <include>com.fasterxml.jackson.core:jackson-core</include>
-
<include>com.fasterxml.jackson.core:jackson-annotations</include>
- <include>org.roaringbitmap:RoaringBitmap</include>
- <include>org.roaringbitmap:shims</include>
- <include>org.apache.commons:commons-collections4</include>
+ <include>*:*</include>
</includes>
</artifactSet>
<finalName>${project.artifactId}-${project.version}</finalName>
@@ -88,17 +111,14 @@
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
+ <exclude>**/*.proto</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
- <pattern>com.google.protobuf</pattern>
-
<shadedPattern>${rss.shade.packageName}.com.google.protobuf</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.common</pattern>
-
<shadedPattern>${rss.shade.packageName}.com.google.common</shadedPattern>
+ <pattern>com.google</pattern>
+
<shadedPattern>${rss.shade.packageName}.com.google</shadedPattern>
</relocation>
<relocation>
<pattern>google.protobuf</pattern>
@@ -120,10 +140,82 @@
<pattern>io.netty</pattern>
<shadedPattern>${rss.shade.packageName}.io.netty</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>io.grpc</pattern>
+
<shadedPattern>${rss.shade.packageName}.io.grpc</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.perfmark</pattern>
+
<shadedPattern>${rss.shade.packageName}.io.perfmark</shadedPattern>
+ </relocation>
<relocation>
<pattern>org.roaringbitmap</pattern>
<shadedPattern>${rss.shade.packageName}.org.roaringbitmap</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>org.apache</pattern>
+
<shadedPattern>${rss.shade.packageName}.org.apache</shadedPattern>
+ <excludes>
+ <!-- Exclude the packages belonging to uniffle -->
+ <exclude>org/apache/uniffle/**/*</exclude>
+ <exclude>org/apache/hadoop/*</exclude>
+ <exclude>org/apache/hadoop/**/*</exclude>
+ <!-- Exclude the logging packages-->
+ <exclude>**/pom.xml</exclude>
+ <exclude>org/apache/commons/logging/*</exclude>
+ <exclude>org/apache/commons/logging/**/*</exclude>
+ <exclude>org/apache/log4j/*</exclude>
+ <exclude>org/apache/log4j/**/*</exclude>
+ <!-- Exclude spark packages -->
+ <exclude>org/apache/spark/**/*</exclude>
+ <!-- Exclude commons-io packages -->
+ <exclude>org/apache/commons/io/**/*</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>org.codehaus</pattern>
+
<shadedPattern>${rss.shade.packageName}.codehaus</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>android.annotation</pattern>
+
<shadedPattern>${rss.shade.packageName}.android.annotation</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javax</pattern>
+ <shadedPattern>${rss.shade.packageName}.javax</shadedPattern>
+ <includes>
+ <include>javax/annotation/**/*</include>
+ <include>javax/activation/**/*</include>
+ <include>javax/xml/**/*</include>
+ <include>javax/inject/**/*</include>
+ <include>javax/validation/**/*</include>
+ <include>javax/servlet/**/*</include>
+ </includes>
+ </relocation>
+ <relocation>
+ <pattern>javassist</pattern>
+
<shadedPattern>${rss.shade.packageName}.javassist</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.prometheus.client</pattern>
+
<shadedPattern>${rss.shade.packageName}.io.prometheus.client</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.sun.activation</pattern>
+
<shadedPattern>${rss.shade.packageName}.com.sun.activation</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.checkerframework</pattern>
+
<shadedPattern>${rss.shade.packageName}.org.checkerframework</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.eclipse</pattern>
+
<shadedPattern>${rss.shade.packageName}.org.eclipse</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>picocli</pattern>
+
<shadedPattern>${rss.shade.packageName}.picocli</shadedPattern>
+ </relocation>
</relocations>
</configuration>
</execution>
