This is an automated email from the ASF dual-hosted git repository. yangjie01 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new ea0436752fe5 [SPARK-45593][BUILD][FOLLOWUP] Correct relocation connect guava dependency ea0436752fe5 is described below commit ea0436752fe5b2a1ca58fad3877f48905b3c2d8a Author: yikaifei <yikai...@apache.org> AuthorDate: Wed Jan 24 19:03:00 2024 +0800 [SPARK-45593][BUILD][FOLLOWUP] Correct relocation connect guava dependency ### What changes were proposed in this pull request? This PR amins to correct relocation connect guava dependency and remove duplicate connect-common from SBT build jars. **Item 1:** In https://github.com/apache/spark/pull/43436, We fixed the connect module dependency on guava, but the dependency on guava was relocation incorrectly. - connect server and connect client jvm don't relocation guava dependency, this runs the risk of causing conflict problems; - connect common relocation does not take effect because it defines conflicting relocation rules with the parent pom(Now, we remove guava dependency from connect-common as it never use this library); **Item2:** Remove duplicate connect-common from SBT build jars as it is shaded in the spark connect. Also, in fact, before this PR, in the output jars built using SBT, connect-common and common-server were the same thing, because they both hit the `jar.getName.contains("spark-connect")` condition. ### Why are the changes needed? Bugfix ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? GA ### Was this patch authored or co-authored using generative AI tooling? No. Closes #44801 from Yikf/SPARK-45593-SBT. Authored-by: yikaifei <yikai...@apache.org> Signed-off-by: yangjie01 <yangji...@baidu.com> --- connector/connect/client/jvm/pom.xml | 22 +++++++++++++++++++++- connector/connect/common/pom.xml | 25 ------------------------- connector/connect/server/pom.xml | 26 ++++++++++++++++++++++++++ project/SparkBuild.scala | 6 +++++- 4 files changed, 52 insertions(+), 27 deletions(-) diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index 9bedebf523a7..81ffb140226e 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -59,6 +59,18 @@ <artifactId>protobuf-java</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>${connect.guava.version}</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>failureaccess</artifactId> + <version>${guava.failureaccess.version}</version> + <scope>compile</scope> + </dependency> <dependency> <groupId>com.lihaoyi</groupId> <artifactId>ammonite_${scala.version}</artifactId> @@ -105,6 +117,7 @@ <promoteTransitiveDependencies>true</promoteTransitiveDependencies> <artifactSet> <includes> + <include>com.google.guava:*</include> <include>com.google.android:*</include> <include>com.google.api.grpc:*</include> <include>com.google.code.findbugs:*</include> @@ -124,6 +137,13 @@ </includes> </artifactSet> <relocations> + <relocation> + <pattern>com.google.common</pattern> + <shadedPattern>${spark.shade.packageName}.connect.guava</shadedPattern> + <includes> + <include>com.google.common.**</include> + </includes> + </relocation> <relocation> <pattern>io.grpc</pattern> <shadedPattern>${spark.shade.packageName}.io.grpc</shadedPattern> @@ -135,7 +155,7 @@ <pattern>com.google</pattern> <shadedPattern>${spark.shade.packageName}.com.google</shadedPattern> <excludes> - <!-- Guava is relocated to ${spark.shade.packageName}.guava (see the parent pom.xml) --> + <!-- Guava is relocated to ${spark.shade.packageName}.connect.guava --> <exclude>com.google.common.**</exclude> </excludes> </relocation> diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml index 336d83e04c15..b0f015246f4c 100644 --- a/connector/connect/common/pom.xml +++ b/connector/connect/common/pom.xml @@ -47,23 +47,6 @@ <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> </dependency> - <!-- - SPARK-45593: spark connect relies on a specific version of Guava, We perform shading - of the Guava library within the connect-common module to ensure both connect-server and - connect-client modules maintain consistent and accurate Guava dependencies. - --> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>${connect.guava.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>failureaccess</artifactId> - <version>${guava.failureaccess.version}</version> - <scope>compile</scope> - </dependency> <dependency> <groupId>io.grpc</groupId> <artifactId>grpc-netty</artifactId> @@ -158,17 +141,9 @@ <artifactSet> <includes> <include>org.spark-project.spark:unused</include> - <include>com.google.guava:guava</include> - <include>com.google.guava:failureaccess</include> <include>org.apache.tomcat:annotations-api</include> </includes> </artifactSet> - <relocations> - <relocation> - <pattern>com.google.common</pattern> - <shadedPattern>${spark.shade.packageName}.connect.guava</shadedPattern> - </relocation> - </relocations> </configuration> <executions> <execution> diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml index 82127f736ccb..bdea8a627000 100644 --- a/connector/connect/server/pom.xml +++ b/connector/connect/server/pom.xml @@ -51,6 +51,12 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-connect-common_${scala.binary.version}</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.spark</groupId> @@ -150,6 +156,18 @@ <groupId>org.scala-lang.modules</groupId> <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId> </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>${connect.guava.version}</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>failureaccess</artifactId> + <version>${guava.failureaccess.version}</version> + <scope>compile</scope> + </dependency> <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> @@ -270,6 +288,7 @@ <shadedArtifactAttached>false</shadedArtifactAttached> <artifactSet> <includes> + <include>com.google.guava:*</include> <include>io.grpc:*:</include> <include>com.google.protobuf:*</include> @@ -289,6 +308,13 @@ </includes> </artifactSet> <relocations> + <relocation> + <pattern>com.google.common</pattern> + <shadedPattern>${spark.shade.packageName}.connect.guava</shadedPattern> + <includes> + <include>com.google.common.**</include> + </includes> + </relocation> <relocation> <pattern>com.google.thirdparty</pattern> <shadedPattern>${spark.shade.packageName}.connect.guava</shadedPattern> diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 3123f931cd88..45b51cb0ff5b 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1487,7 +1487,11 @@ object CopyDependencies { if (destJar.isFile()) { destJar.delete() } - if (jar.getName.contains("spark-connect") && + + if (jar.getName.contains("spark-connect-common") && + !SbtPomKeys.profiles.value.contains("noshade-connect")) { + // Don't copy the spark connect common JAR as it is shaded in the spark connect. + } else if (jar.getName.contains("spark-connect") && !SbtPomKeys.profiles.value.contains("noshade-connect")) { Files.copy(fid.toPath, destJar.toPath) } else if (jar.getName.contains("connect-client-jvm") && --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org