JoshRosen commented on code in PR #39763:
URL: https://github.com/apache/spark/pull/39763#discussion_r1088589702
##########
core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala:
##########
@@ -778,12 +778,117 @@ class JsonProtocolSuite extends SparkFunSuite {
|}""".stripMargin
assert(JsonProtocol.sparkEventFromJson(unknownFieldsJson) === expected)
}
+
+ test("SPARK-42204: spark.eventLog.includeTaskMetricsAccumulators config") {
+ val includeConf = new JsonProtocolOptions(
+ new SparkConf().set(EVENT_LOG_INCLUDE_TASK_METRICS_ACCUMULATORS, true))
+ val excludeConf = new JsonProtocolOptions(
+ new SparkConf().set(EVENT_LOG_INCLUDE_TASK_METRICS_ACCUMULATORS, false))
+
+ val taskMetricsAccumulables = TaskMetrics
+ .empty
+ .nameToAccums
+ .filterKeys(!JsonProtocol.accumulableExcludeList.contains(_))
+ .values
+ .map(_.toInfo(Some(1), None))
+ .toSeq
+
+ val taskInfoWithTaskMetricsAccums = makeTaskInfo(222L, 333, 1, 333, 444L,
false)
+ taskInfoWithTaskMetricsAccums.setAccumulables(taskMetricsAccumulables)
+ val taskInfoWithoutTaskMetricsAccums = makeTaskInfo(222L, 333, 1, 333,
444L, false)
+ taskInfoWithoutTaskMetricsAccums.setAccumulables(Seq.empty)
+
+ val stageInfoWithTaskMetricsAccums = makeStageInfo(100, 200, 300, 400L,
500L)
+ stageInfoWithTaskMetricsAccums.accumulables.clear()
+ stageInfoWithTaskMetricsAccums.accumulables ++=
taskMetricsAccumulables.map(x => (x.id, x))
+ val stageInfoWithoutTaskMetricsAccums = makeStageInfo(100, 200, 300, 400L,
500L)
+ stageInfoWithoutTaskMetricsAccums.accumulables.clear()
+
+ // Test events which should be impacted by the config. Due to SPARK-42205
we need to test
Review Comment:
See https://issues.apache.org/jira/browse/SPARK-42205 for more details. Once
that issue is fixed, we can drop the testcases for `TaskStart`, `JobStart`, and
`StageSubmitted` here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
