This is an automated email from the ASF dual-hosted git repository.
hvanhovell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 50d9a56f824 [SPARK-44832][CONNECT] Make transitive dependencies work
properly for Scala Client
50d9a56f824 is described below
commit 50d9a56f824ae51d10543f4573753ff60dc9053b
Author: Herman van Hovell <[email protected]>
AuthorDate: Mon Aug 28 19:53:40 2023 +0200
[SPARK-44832][CONNECT] Make transitive dependencies work properly for Scala
Client
### What changes were proposed in this pull request?
This PR cleans up the Maven build for the Spark Connect Client and Spark
Connect Common. The most important change is that we move `sql-api` from a
`provided` to `compile` dependency. The net effect of this is that when a user
takes a dependency on the client, all of its required (transitive) dependencies
are automatically added.
Please note that this does not address concerns around creating an überjar
and shading. That is for a different day :)
### Why are the changes needed?
When you take a dependency on the connect scala client you need to manually
add the `sql-api` module as a dependency. This is rather poor UX.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manually running maven, checking dependency tree, ...
Closes #42518 from hvanhovell/SPARK-44832.
Authored-by: Herman van Hovell <[email protected]>
Signed-off-by: Herman van Hovell <[email protected]>
---
connector/connect/client/jvm/pom.xml | 48 ----------------------
.../CheckConnectJvmClientCompatibility.scala | 33 ++++++++++++---
connector/connect/common/pom.xml | 6 ---
dev/connect-jvm-client-mima-check | 2 +-
4 files changed, 28 insertions(+), 61 deletions(-)
diff --git a/connector/connect/client/jvm/pom.xml
b/connector/connect/client/jvm/pom.xml
index d4e9b147e02..8cb6758ec9f 100644
--- a/connector/connect/client/jvm/pom.xml
+++ b/connector/connect/client/jvm/pom.xml
@@ -39,55 +39,21 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-connect-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-api_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sketch_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${connect.guava.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>failureaccess</artifactId>
- <version>${guava.failureaccess.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec-http2</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-handler-proxy</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport-native-unix-common</artifactId>
- <version>${netty.version}</version>
</dependency>
<dependency>
<groupId>com.lihaoyi</groupId>
@@ -95,19 +61,6 @@
<version>${ammonite.version}</version>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-connect-common_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.scalacheck</groupId>
<artifactId>scalacheck_${scala.binary.version}</artifactId>
@@ -148,7 +101,6 @@
<include>org.codehaus.mojo:*</include>
<include>org.checkerframework:*</include>
<include>org.apache.spark:spark-connect-common_${scala.binary.version}</include>
-
<include>org.apache.spark:spark-common-utils_${scala.binary.version}</include>
</includes>
</artifactSet>
<relocations>
diff --git
a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala
b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala
index 1100babde79..1e536cd37fe 100644
---
a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala
+++
b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala
@@ -24,6 +24,7 @@ import java.util.regex.Pattern
import com.typesafe.tools.mima.core._
import com.typesafe.tools.mima.lib.MiMaLib
+import org.apache.spark.SparkBuildInfo.spark_version
import org.apache.spark.sql.test.IntegrationTestUtils._
/**
@@ -46,18 +47,38 @@ object CheckConnectJvmClientCompatibility {
sys.env("SPARK_HOME")
}
+ private val sqlJar = {
+ val path = Paths.get(
+ sparkHome,
+ "sql",
+ "core",
+ "target",
+ "scala-" + scalaVersion,
+ "spark-sql_" + scalaVersion + "-" + spark_version + ".jar")
+ assert(Files.exists(path), s"$path does not exist")
+ path.toFile
+ }
+
+ private val clientJar = {
+ val path = Paths.get(
+ sparkHome,
+ "connector",
+ "connect",
+ "client",
+ "jvm",
+ "target",
+ "scala-" + scalaVersion,
+ "spark-connect-client-jvm_" + scalaVersion + "-" + spark_version +
".jar")
+ assert(Files.exists(path), s"$path does not exist")
+ path.toFile
+ }
+
def main(args: Array[String]): Unit = {
var resultWriter: Writer = null
try {
resultWriter = Files.newBufferedWriter(
Paths.get(s"$sparkHome/.connect-mima-check-result"),
StandardCharsets.UTF_8)
- val clientJar: File =
- findJar(
- "connector/connect/client/jvm",
- "spark-connect-client-jvm-assembly",
- "spark-connect-client-jvm")
- val sqlJar: File = findJar("sql/core", "spark-sql", "spark-sql")
val problemsWithSqlModule =
checkMiMaCompatibilityWithSqlModule(clientJar, sqlJar)
appendMimaCheckErrorMessageIfNeeded(
resultWriter,
diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml
index 8382c1f5db1..5fd6c885025 100644
--- a/connector/connect/common/pom.xml
+++ b/connector/connect/common/pom.xml
@@ -38,7 +38,6 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-api_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
@@ -47,7 +46,6 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -85,25 +83,21 @@
<groupId>io.netty</groupId>
<artifactId>netty-codec-http2</artifactId>
<version>${netty.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler-proxy</artifactId>
<version>${netty.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-unix-common</artifactId>
<version>${netty.version}</version>
- <scope>provided</scope>
</dependency>
<dependency> <!-- necessary for Java 9+ -->
<groupId>org.apache.tomcat</groupId>
<artifactId>annotations-api</artifactId>
<version>${tomcat.annotations.api.version}</version>
- <scope>provided</scope>
</dependency>
<!--
This spark-tags test-dep is needed even though it isn't used in this
module,
diff --git a/dev/connect-jvm-client-mima-check
b/dev/connect-jvm-client-mima-check
index 6a29cbf08ce..7564a614be9 100755
--- a/dev/connect-jvm-client-mima-check
+++ b/dev/connect-jvm-client-mima-check
@@ -35,7 +35,7 @@ fi
rm -f .connect-mima-check-result
echo "Build required modules..."
-build/sbt
"sql/package;connect-client-jvm/assembly;connect-client-jvm/Test/package;avro/package;protobuf/assembly"
+build/sbt
"sql/package;connect-client-jvm/package;connect-client-jvm/Test/package;avro/package;protobuf/assembly"
CONNECT_TEST_CLASSPATH="$(build/sbt -DcopyDependencies=false "export
connect-client-jvm/Test/fullClasspath" | grep jar | tail -n1)"
SQL_CLASSPATH="$(build/sbt -DcopyDependencies=false "export sql/fullClasspath"
| grep jar | tail -n1)"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
