This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 69b09bc52c29 [SPARK-49368][CONNECT] Avoid accessing protobuf lite
classes directly
69b09bc52c29 is described below
commit 69b09bc52c299e5eb610fbb8c4a0c1154abf4606
Author: Cheng Pan <[email protected]>
AuthorDate: Mon Aug 26 10:04:50 2024 +0900
[SPARK-49368][CONNECT] Avoid accessing protobuf lite classes directly
### What changes were proposed in this pull request?
This PR changes the code in the connect server module which refers gRPC
lite classes and restores `grpc-protobuf-lite` scope from "compile" back to
"runtime".
### Why are the changes needed?
gRPC/protobuf provides both standard and lite runtimes, which should not be
mixed up, connect server currently uses the standard runtime, should avoid
accessing the lite class directly.
As [grpc-java v1.61.0 release
notes](https://github.com/grpc/grpc-java/releases/tag/v1.61.0) said
> Change many compile deps to runtime deps
(https://github.com/grpc/grpc-java/commit/d6830d7f99ac7e5afb8b75100f0432151d4705e8).
This reduces the transitive classes "leaked" into the compile classpath. In
particular, grpc-core (`io.grpc.internal`) will be less frequently included
transitively at compile time.
In SPARK-46919 we upgraded the gRPC from 1.59.3 to 1.62.0, and had to
manually expose "io.grpc:grpc-protobuf-lite" to the compile scope to make the
build happy, which breaks the original intention of gRPC deps management.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass GHA.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #47857 from pan3793/SPARK-49368.
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
sql/connect/server/pom.xml | 11 ----------
.../sql/connect/service/SparkConnectService.scala | 25 +++++++++++-----------
2 files changed, 13 insertions(+), 23 deletions(-)
diff --git a/sql/connect/server/pom.xml b/sql/connect/server/pom.xml
index e77be695ab37..3afd0f2cbf07 100644
--- a/sql/connect/server/pom.xml
+++ b/sql/connect/server/pom.xml
@@ -196,17 +196,6 @@
<artifactId>grpc-protobuf</artifactId>
<version>${io.grpc.version}</version>
</dependency>
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-protobuf-lite</artifactId>
- <version>${io.grpc.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-javalite</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-services</artifactId>
diff --git
a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectService.scala
b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectService.scala
index e9c92f8d007e..d0c06e96047f 100644
---
a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectService.scala
+++
b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectService.scala
@@ -22,11 +22,11 @@ import java.util.concurrent.TimeUnit
import scala.jdk.CollectionConverters._
-import com.google.protobuf.MessageLite
+import com.google.protobuf.Message
import io.grpc.{BindableService, MethodDescriptor, Server,
ServerMethodDefinition, ServerServiceDefinition}
import io.grpc.MethodDescriptor.PrototypeMarshaller
import io.grpc.netty.NettyServerBuilder
-import io.grpc.protobuf.lite.ProtoLiteUtils
+import io.grpc.protobuf.ProtoUtils
import io.grpc.protobuf.services.ProtoReflectionService
import io.grpc.stub.StreamObserver
import org.apache.commons.lang3.StringUtils
@@ -231,20 +231,20 @@ class SparkConnectService(debug: Boolean) extends
AsyncService with BindableServ
}
}
- private def methodWithCustomMarshallers(methodDesc:
MethodDescriptor[MessageLite, MessageLite])
- : MethodDescriptor[MessageLite, MessageLite] = {
+ private def methodWithCustomMarshallers(
+ methodDesc: MethodDescriptor[Message, Message]):
MethodDescriptor[Message, Message] = {
val recursionLimit =
SparkEnv.get.conf.get(CONNECT_GRPC_MARSHALLER_RECURSION_LIMIT)
val requestMarshaller =
- ProtoLiteUtils.marshallerWithRecursionLimit(
+ ProtoUtils.marshallerWithRecursionLimit(
methodDesc.getRequestMarshaller
- .asInstanceOf[PrototypeMarshaller[MessageLite]]
+ .asInstanceOf[PrototypeMarshaller[Message]]
.getMessagePrototype,
recursionLimit)
val responseMarshaller =
- ProtoLiteUtils.marshallerWithRecursionLimit(
+ ProtoUtils.marshallerWithRecursionLimit(
methodDesc.getResponseMarshaller
- .asInstanceOf[PrototypeMarshaller[MessageLite]]
+ .asInstanceOf[PrototypeMarshaller[Message]]
.getMessagePrototype,
recursionLimit)
methodDesc.toBuilder
@@ -259,17 +259,18 @@ class SparkConnectService(debug: Boolean) extends
AsyncService with BindableServ
// Create a new ServerServiceDefinition builder
// using the name of the original service definition.
- val builder =
io.grpc.ServerServiceDefinition.builder(serviceDef.getServiceDescriptor.getName)
+ val builder =
ServerServiceDefinition.builder(serviceDef.getServiceDescriptor.getName)
// Iterate through all the methods of the original service definition.
// For each method, add a customized method descriptor (with updated
marshallers)
// and the original server call handler to the builder.
serviceDef.getMethods.asScala
- .asInstanceOf[Iterable[ServerMethodDefinition[MessageLite, MessageLite]]]
- .foreach(method =>
+ .asInstanceOf[Iterable[ServerMethodDefinition[Message, Message]]]
+ .foreach { method =>
builder.addMethod(
methodWithCustomMarshallers(method.getMethodDescriptor),
- method.getServerCallHandler))
+ method.getServerCallHandler)
+ }
// Build the final ServerServiceDefinition and return it.
builder.build()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
