This is an automated email from the ASF dual-hosted git repository.
yangjie01 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 2da520e88266 [SPARK-45593][BUILD][3.5] Correct relocation connect
guava dependency
2da520e88266 is described below
commit 2da520e88266530b2283ef3c9ac90bdc806b7556
Author: yikaifei <yikai...@apache.org>
AuthorDate: Mon Apr 1 15:35:23 2024 +0800
[SPARK-45593][BUILD][3.5] Correct relocation connect guava dependency
### What changes were proposed in this pull request?
This PR amins to correct relocation connect guava dependency and remove
duplicate connect-common from SBT build jars.
This PR cherry-pick from https://github.com/apache/spark/pull/43436 and
https://github.com/apache/spark/pull/44801 as a backport to 3.5 branch.
### Why are the changes needed?
Bugfix
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Follow the steps described at
https://github.com/apache/spark/pull/43195#issue-1921234067 to test manually.
In addition, will continue to observe the GA situation in recent days.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #45775 from Yikf/branch-3.5.
Authored-by: yikaifei <yikai...@apache.org>
Signed-off-by: yangjie01 <yangji...@baidu.com>
---
assembly/pom.xml | 6 ++++++
connector/connect/client/jvm/pom.xml | 22 ++++++++++++++++++----
connector/connect/common/pom.xml | 33 +++++++++++++++++++++------------
connector/connect/server/pom.xml | 1 +
project/SparkBuild.scala | 6 +++++-
5 files changed, 51 insertions(+), 17 deletions(-)
diff --git a/assembly/pom.xml b/assembly/pom.xml
index d1ef9b24afda..21330058f77d 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -159,6 +159,12 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-connect_${scala.binary.version}</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+
<artifactId>spark-connect-common_${scala.binary.version}</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
diff --git a/connector/connect/client/jvm/pom.xml
b/connector/connect/client/jvm/pom.xml
index 53ff0b0147e0..6febc5ee6bd6 100644
--- a/connector/connect/client/jvm/pom.xml
+++ b/connector/connect/client/jvm/pom.xml
@@ -51,9 +51,14 @@
<version>${project.version}</version>
</dependency>
<!--
- We need to define guava and protobuf here because we need to change the
scope of both from
+ We need to define protobuf here because we need to change the scope of
both from
provided to compile. If we don't do this we can't shade these libraries.
-->
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <scope>compile</scope>
+ </dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@@ -61,8 +66,9 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
+ <groupId>com.google.guava</groupId>
+ <artifactId>failureaccess</artifactId>
+ <version>${guava.failureaccess.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
@@ -108,6 +114,7 @@
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<artifactSet>
<includes>
+ <include>com.google.guava:*</include>
<include>com.google.android:*</include>
<include>com.google.api.grpc:*</include>
<include>com.google.code.findbugs:*</include>
@@ -127,6 +134,13 @@
</includes>
</artifactSet>
<relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+
<shadedPattern>${spark.shade.packageName}.connect.guava</shadedPattern>
+ <includes>
+ <include>com.google.common.**</include>
+ </includes>
+ </relocation>
<relocation>
<pattern>io.grpc</pattern>
<shadedPattern>${spark.shade.packageName}.io.grpc</shadedPattern>
@@ -138,7 +152,7 @@
<pattern>com.google</pattern>
<shadedPattern>${spark.shade.packageName}.com.google</shadedPattern>
<excludes>
- <!-- Guava is relocated to ${spark.shade.packageName}.guava
(see the parent pom.xml) -->
+ <!-- Guava is relocated to
${spark.shade.packageName}.connect.guava -->
<exclude>com.google.common.**</exclude>
</excludes>
</relocation>
diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml
index 7ce0aa6615d3..3c07b63c50a5 100644
--- a/connector/connect/common/pom.xml
+++ b/connector/connect/common/pom.xml
@@ -47,18 +47,6 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${connect.guava.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>failureaccess</artifactId>
- <version>${guava.failureaccess.version}</version>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
@@ -152,6 +140,27 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <artifactSet>
+ <includes>
+ <include>org.spark-project.spark:unused</include>
+
<include>org.apache.tomcat:annotations-api</include>
+ </includes>
+ </artifactSet>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
<profiles>
diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml
index 724bad616f82..804ff2ff15a1 100644
--- a/connector/connect/server/pom.xml
+++ b/connector/connect/server/pom.xml
@@ -168,6 +168,7 @@
<groupId>com.google.guava</groupId>
<artifactId>failureaccess</artifactId>
<version>${guava.failureaccess.version}</version>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index dfadfea172d8..40c90a3461b0 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1535,7 +1535,11 @@ object CopyDependencies {
if (destJar.isFile()) {
destJar.delete()
}
- if (jar.getName.contains("spark-connect") &&
+
+ if (jar.getName.contains("spark-connect-common") &&
+ !SbtPomKeys.profiles.value.contains("noshade-connect")) {
+ // Don't copy the spark connect common JAR as it is shaded in the
spark connect.
+ } else if (jar.getName.contains("spark-connect") &&
!SbtPomKeys.profiles.value.contains("noshade-connect")) {
Files.copy(fid.toPath, destJar.toPath)
} else if (jar.getName.contains("connect-client-jvm") &&
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
