This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new e82805da876 [SPARK-45050][SQL][CONNECT] Improve error message for UNKNOWN io.grpc.StatusRuntimeException e82805da876 is described below commit e82805da876672f7e9447ec54e66175f84ea3d36 Author: Yihong He <yihong...@databricks.com> AuthorDate: Thu Sep 7 09:02:38 2023 +0900 [SPARK-45050][SQL][CONNECT] Improve error message for UNKNOWN io.grpc.StatusRuntimeException ### What changes were proposed in this pull request? - Improve error message for UNKNOWN io.grpc.StatusRuntimeException Before: ``` [info] - handle unknown exception *** FAILED *** (15 milliseconds) [info] org.apache.spark.SparkException: [info] at org.apache.spark.sql.connect.client.GrpcExceptionConverter$.toThrowable(GrpcExceptionConverter.scala:110) [info] at org.apache.spark.sql.connect.client.GrpcExceptionConverter$.convert(GrpcExceptionConverter.scala:41) [info] at org.apache.spark.sql.connect.client.GrpcExceptionConverter$$anon$1.hasNext(GrpcExceptionConverter.scala:49) [info] at org.apache.spark.sql.connect.client.SparkResult.org$apache$spark$sql$connect$client$SparkResult$$processResponses(SparkResult.scala:83) [info] at org.apache.spark.sql.connect.client.SparkResult.length(SparkResult.scala:153) [info] at org.apache.spark.sql.connect.client.SparkResult.toArray(SparkResult.scala:183) [info] at org.apache.spark.sql.Dataset.$anonfun$collect$1(Dataset.scala:2910) [info] at org.apache.spark.sql.Dataset.withResult(Dataset.scala:3350) [info] at org.apache.spark.sql.Dataset.collect(Dataset.scala:2909) [info] at org.apache.spark.sql.ClientE2ETestSuite.$anonfun$new$19(ClientE2ETestSuite.scala:118) ``` After: ``` [info] - handle unknown exception *** FAILED *** (21 milliseconds) [info] org.apache.spark.SparkException: io.grpc.StatusRuntimeException: UNKNOWN [info] at org.apache.spark.sql.connect.client.GrpcExceptionConverter$.toThrowable(GrpcExceptionConverter.scala:110) [info] at org.apache.spark.sql.connect.client.GrpcExceptionConverter$.convert(GrpcExceptionConverter.scala:41) [info] at org.apache.spark.sql.connect.client.GrpcExceptionConverter$$anon$1.hasNext(GrpcExceptionConverter.scala:49) [info] at org.apache.spark.sql.connect.client.SparkResult.org$apache$spark$sql$connect$client$SparkResult$$processResponses(SparkResult.scala:83) [info] at org.apache.spark.sql.connect.client.SparkResult.length(SparkResult.scala:153) [info] at org.apache.spark.sql.connect.client.SparkResult.toArray(SparkResult.scala:183) [info] at org.apache.spark.sql.Dataset.$anonfun$collect$1(Dataset.scala:2910) [info] at org.apache.spark.sql.Dataset.withResult(Dataset.scala:3350) [info] at org.apache.spark.sql.Dataset.collect(Dataset.scala:2909) [info] at org.apache.spark.sql.ClientE2ETestSuite.$anonfun$new$19(ClientE2ETestSuite.scala:118) ``` ### Why are the changes needed? - Better readability of the exception message ### Does this PR introduce _any_ user-facing change? - No ### How was this patch tested? - build/sbt "connect-client-jvm/testOnly *ClientE2ETestSuite" ### Was this patch authored or co-authored using generative AI tooling? Closes #42771 from heyihong/SPARK-45050. Authored-by: Yihong He <yihong...@databricks.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala | 12 ++++++++++++ .../spark/sql/connect/client/GrpcExceptionConverter.scala | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala index f10f5c78ead..06338f33e1d 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala @@ -109,6 +109,18 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM assert(df.collect().length == 501) } + test("handle unknown exception") { + var df = spark.range(1) + val limit = spark.conf.get("spark.connect.grpc.marshallerRecursionLimit").toInt + 1 + for (a <- 1 to limit) { + df = df.union(spark.range(a, a + 1)) + } + val ex = intercept[SparkException] { + df.collect() + } + assert(ex.getMessage.contains("io.grpc.StatusRuntimeException: UNKNOWN")) + } + test("many tables") { withSQLConf("spark.sql.execution.arrow.maxRecordsPerBatch" -> "10") { val numTables = 20 diff --git a/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/client/GrpcExceptionConverter.scala b/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/client/GrpcExceptionConverter.scala index 672d31be954..c430485bd41 100644 --- a/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/client/GrpcExceptionConverter.scala +++ b/connector/connect/common/src/main/scala/org/apache/spark/sql/connect/client/GrpcExceptionConverter.scala @@ -107,7 +107,7 @@ private[client] object GrpcExceptionConverter extends JsonUtils { private def toThrowable(ex: StatusRuntimeException): Throwable = { val status = StatusProto.fromThrowable(ex) - val fallbackEx = new SparkException(status.getMessage, ex.getCause) + val fallbackEx = new SparkException(ex.toString, ex.getCause) val errorInfoOpt = status.getDetailsList.asScala .find(_.is(classOf[ErrorInfo])) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org