This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 916b6f51af2 [SPARK-45050][SQL][CONNECT] Improve error message for
UNKNOWN io.grpc.StatusRuntimeException
916b6f51af2 is described below
commit 916b6f51af213da1fa608231ffdad97004fcbd74
Author: Yihong He <[email protected]>
AuthorDate: Thu Sep 7 09:02:38 2023 +0900
[SPARK-45050][SQL][CONNECT] Improve error message for UNKNOWN
io.grpc.StatusRuntimeException
### What changes were proposed in this pull request?
- Improve error message for UNKNOWN io.grpc.StatusRuntimeException
Before:
```
[info] - handle unknown exception *** FAILED *** (15 milliseconds)
[info] org.apache.spark.SparkException:
[info] at
org.apache.spark.sql.connect.client.GrpcExceptionConverter$.toThrowable(GrpcExceptionConverter.scala:110)
[info] at
org.apache.spark.sql.connect.client.GrpcExceptionConverter$.convert(GrpcExceptionConverter.scala:41)
[info] at
org.apache.spark.sql.connect.client.GrpcExceptionConverter$$anon$1.hasNext(GrpcExceptionConverter.scala:49)
[info] at
org.apache.spark.sql.connect.client.SparkResult.org$apache$spark$sql$connect$client$SparkResult$$processResponses(SparkResult.scala:83)
[info] at
org.apache.spark.sql.connect.client.SparkResult.length(SparkResult.scala:153)
[info] at
org.apache.spark.sql.connect.client.SparkResult.toArray(SparkResult.scala:183)
[info] at
org.apache.spark.sql.Dataset.$anonfun$collect$1(Dataset.scala:2910)
[info] at org.apache.spark.sql.Dataset.withResult(Dataset.scala:3350)
[info] at org.apache.spark.sql.Dataset.collect(Dataset.scala:2909)
[info] at
org.apache.spark.sql.ClientE2ETestSuite.$anonfun$new$19(ClientE2ETestSuite.scala:118)
```
After:
```
[info] - handle unknown exception *** FAILED *** (21 milliseconds)
[info] org.apache.spark.SparkException: io.grpc.StatusRuntimeException:
UNKNOWN
[info] at
org.apache.spark.sql.connect.client.GrpcExceptionConverter$.toThrowable(GrpcExceptionConverter.scala:110)
[info] at
org.apache.spark.sql.connect.client.GrpcExceptionConverter$.convert(GrpcExceptionConverter.scala:41)
[info] at
org.apache.spark.sql.connect.client.GrpcExceptionConverter$$anon$1.hasNext(GrpcExceptionConverter.scala:49)
[info] at
org.apache.spark.sql.connect.client.SparkResult.org$apache$spark$sql$connect$client$SparkResult$$processResponses(SparkResult.scala:83)
[info] at
org.apache.spark.sql.connect.client.SparkResult.length(SparkResult.scala:153)
[info] at
org.apache.spark.sql.connect.client.SparkResult.toArray(SparkResult.scala:183)
[info] at
org.apache.spark.sql.Dataset.$anonfun$collect$1(Dataset.scala:2910)
[info] at org.apache.spark.sql.Dataset.withResult(Dataset.scala:3350)
[info] at org.apache.spark.sql.Dataset.collect(Dataset.scala:2909)
[info] at
org.apache.spark.sql.ClientE2ETestSuite.$anonfun$new$19(ClientE2ETestSuite.scala:118)
```
### Why are the changes needed?
- Better readability of the exception message
### Does this PR introduce _any_ user-facing change?
- No
### How was this patch tested?
- build/sbt "connect-client-jvm/testOnly *ClientE2ETestSuite"
### Was this patch authored or co-authored using generative AI tooling?
Closes #42771 from heyihong/SPARK-45050.
Authored-by: Yihong He <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit e82805da876672f7e9447ec54e66175f84ea3d36)
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala | 12 ++++++++++++
.../spark/sql/connect/client/GrpcExceptionConverter.scala | 2 +-
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git
a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala
b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala
index fd443b73925..df36b53791a 100644
---
a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala
+++
b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala
@@ -109,6 +109,18 @@ class ClientE2ETestSuite extends RemoteSparkSession with
SQLHelper with PrivateM
assert(df.collect().length == 501)
}
+ test("handle unknown exception") {
+ var df = spark.range(1)
+ val limit =
spark.conf.get("spark.connect.grpc.marshallerRecursionLimit").toInt + 1
+ for (a <- 1 to limit) {
+ df = df.union(spark.range(a, a + 1))
+ }
+ val ex = intercept[SparkException] {
+ df.collect()
+ }
+ assert(ex.getMessage.contains("io.grpc.StatusRuntimeException: UNKNOWN"))
+ }
+
test("many tables") {
withSQLConf("spark.sql.execution.arrow.maxRecordsPerBatch" -> "10") {
val numTables = 20
diff --git
a/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/client/GrpcExceptionConverter.scala
b/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/client/GrpcExceptionConverter.scala
index 672d31be954..c430485bd41 100644
---
a/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/client/GrpcExceptionConverter.scala
+++
b/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/client/GrpcExceptionConverter.scala
@@ -107,7 +107,7 @@ private[client] object GrpcExceptionConverter extends
JsonUtils {
private def toThrowable(ex: StatusRuntimeException): Throwable = {
val status = StatusProto.fromThrowable(ex)
- val fallbackEx = new SparkException(status.getMessage, ex.getCause)
+ val fallbackEx = new SparkException(ex.toString, ex.getCause)
val errorInfoOpt = status.getDetailsList.asScala
.find(_.is(classOf[ErrorInfo]))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
