ulysses-you commented on code in PR #38302:
URL: https://github.com/apache/spark/pull/38302#discussion_r1004281839
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/SQLJsonProtocolSuite.scala:
##########
@@ -74,27 +74,60 @@ class SQLJsonProtocolSuite extends SparkFunSuite with
LocalSparkSession {
test("SparkListenerSQLExecutionEnd backward compatibility") {
spark = new TestSparkSession()
val qe = spark.sql("select 1").queryExecution
- val event = SparkListenerSQLExecutionEnd(1, 10)
+ val errorMessage = SparkThrowableHelper.getMessage(new Exception("test"))
+ val event = SparkListenerSQLExecutionEnd(1, 10, Some(errorMessage))
event.duration = 1000
event.executionName = Some("test")
event.qe = qe
- event.executionFailure = Some(new RuntimeException("test"))
+ event.executionFailure = Some(new Exception("test"))
val json = JsonProtocol.sparkEventToJsonString(event)
+ // scalastyle:off line.size.limit
assert(parse(json) == parse(
"""
|{
| "Event" :
"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd",
| "executionId" : 1,
- | "time" : 10
+ | "time" : 10,
+ | "errorMessage" :
"{\"errorClass\":\"java.lang.Exception\",\"messageParameters\":{\"message\":\"test\"}}"
|}
""".stripMargin))
+ // scalastyle:on
val readBack = JsonProtocol.sparkEventFromJson(json)
event.duration = 0
event.executionName = None
event.qe = null
event.executionFailure = None
assert(readBack == event)
}
+
+ test("SPARK-40834: Use SparkListenerSQLExecutionEnd to track final SQL
status in UI") {
+ // parse old event log using new SparkListenerSQLExecutionEnd
+ val executionEnd =
+ """
+ |{
+ | "Event" :
"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd",
+ | "executionId" : 1,
+ | "time" : 10
+ |}
+ """.stripMargin
+ val readBack = JsonProtocol.sparkEventFromJson(executionEnd)
+ assert(readBack == SparkListenerSQLExecutionEnd(1, 10))
+
+ // parse new event using old SparkListenerSQLExecutionEnd
+ // scalastyle:off line.size.limit
+ val newExecutionEnd =
+ """
+ |{
+ | "Event" :
"org.apache.spark.sql.execution.OldVersionSQLExecutionEnd",
+ | "executionId" : 1,
+ | "time" : 10,
+ | "errorMessage" :
"{\"errorClass\":\"java.lang.Exception\",\"messageParameters\":{\"message\":\"test\"}}"
Review Comment:
I just follow the style of error json format, so we can keep consistent with
`SparkThrowable`. We'd expect all java throwables are wrapped by SparkThrowable
so we will never go into here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
