ulysses-you commented on a change in pull request #31830:
URL: https://github.com/apache/spark/pull/31830#discussion_r600124482
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/SQLJsonProtocolSuite.scala
##########
@@ -28,28 +29,46 @@ import org.apache.spark.util.JsonProtocol
class SQLJsonProtocolSuite extends SparkFunSuite with LocalSparkSession {
test("SparkPlanGraph backward compatibility: metadata") {
- val SQLExecutionStartJsonString =
- """
- |{
- |
"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart",
- | "executionId":0,
- | "description":"test desc",
- | "details":"test detail",
- | "physicalPlanDescription":"test plan",
- | "sparkPlanInfo": {
- | "nodeName":"TestNode",
- | "simpleString":"test string",
- | "children":[],
- | "metadata":{},
- | "metrics":[]
- | },
- | "time":0
- |}
+ Seq(true, false).foreach { newExecutionStartEvent =>
+ val event = if (newExecutionStartEvent) {
+ "org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart"
+ } else {
+ "org.apache.spark.sql.execution.OldVersionSQLExecutionStart"
+ }
+ val SQLExecutionStartJsonString =
+ s"""
+ |{
+ | "Event":"$event",
+ | "executionId":0,
+ | "description":"test desc",
+ | "details":"test detail",
+ | "physicalPlanDescription":"test plan",
+ | "sparkPlanInfo": {
+ | "nodeName":"TestNode",
+ | "simpleString":"test string",
+ | "children":[],
+ | "metadata":{},
+ | "metrics":[]
+ | },
+ | "time":0,
+ | "modifiedConfigs": {
+ | "k1":"v1"
+ | }
+ |}
""".stripMargin
- val reconstructedEvent =
JsonProtocol.sparkEventFromJson(parse(SQLExecutionStartJsonString))
- val expectedEvent = SparkListenerSQLExecutionStart(0, "test desc", "test
detail", "test plan",
- new SparkPlanInfo("TestNode", "test string", Nil, Map(), Nil), 0)
- assert(reconstructedEvent == expectedEvent)
+
+ val reconstructedEvent =
JsonProtocol.sparkEventFromJson(parse(SQLExecutionStartJsonString))
+ if (newExecutionStartEvent) {
+ val expectedEvent = SparkListenerSQLExecutionStart(0, "test desc",
"test detail",
+ "test plan", new SparkPlanInfo("TestNode", "test string", Nil,
Map(), Nil), 0,
+ Map("k1" -> "v1"))
+ assert(reconstructedEvent == expectedEvent)
+ } else {
+ val expectedOldEvent = OldVersionSQLExecutionStart(0, "test desc",
"test detail",
+ "test plan", new SparkPlanInfo("TestNode", "test string", Nil,
Map(), Nil), 0)
+ assert(reconstructedEvent == expectedOldEvent)
Review comment:
this test ensure we keep the backward compatibility @gengliangwang
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
