This is an automated email from the ASF dual-hosted git repository.
mridulm80 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 74447ab5817 [SPARK-44162][CORE] Support G1GC in spark metrics
74447ab5817 is described below
commit 74447ab5817bcb76f8979ae21bf38a6d193a3e46
Author: Jia Fan <[email protected]>
AuthorDate: Thu Aug 31 23:16:24 2023 -0500
[SPARK-44162][CORE] Support G1GC in spark metrics
### What changes were proposed in this pull request?
As a part of support JDK21, add support `G1 Concurrent GC`
GarbageCollectorMXBean in Spark metrics.
Refer https://github.com/openjdk/jdk/pull/11341 and
https://bugs.openjdk.org/browse/JDK-8297247 , the `G1 Concurrent GC` not a part
of YoungGC or FullGC. So we follow the JDK definition, bring two new metrics:
`ConcurrentGCCount` and `ConcurrentGCTime`.
### Why are the changes needed?
add new builtin garbage collectors for metrics.
### Does this PR introduce _any_ user-facing change?
Yes, will receive new metrics.
### How was this patch tested?
Test in local, I will add test if necessary.
Closes #41808 from Hisoka-X/SPARK-44162_G1GC_on_JDK21.
Authored-by: Jia Fan <[email protected]>
Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com>
---
.../apache/spark/metrics/ExecutorMetricType.scala | 11 ++++++--
.../spark/status/api/v1/PrometheusResource.scala | 4 +--
.../complete_stage_list_json_expectation.json | 12 ++++++---
.../excludeOnFailure_for_stage_expectation.json | 12 ++++++---
...xcludeOnFailure_node_for_stage_expectation.json | 24 ++++++++++++-----
.../executor_list_json_expectation.json | 4 ++-
...ist_with_executor_metrics_json_expectation.json | 16 +++++++++---
.../executor_memory_usage_expectation.json | 16 +++++++++---
...executor_node_excludeOnFailure_expectation.json | 16 +++++++++---
...e_excludeOnFailure_unexcluding_expectation.json | 16 +++++++++---
.../failed_stage_list_json_expectation.json | 4 ++-
..._json_details_with_failed_task_expectation.json | 8 ++++--
.../one_stage_attempt_json_expectation.json | 8 ++++--
.../one_stage_json_expectation.json | 8 ++++--
.../one_stage_json_with_details_expectation.json | 8 ++++--
...ne_stage_json_with_partitionId_expectation.json | 8 ++++--
.../stage_list_json_expectation.json | 16 +++++++++---
...age_list_with_accumulable_json_expectation.json | 4 ++-
.../stage_list_with_peak_metrics_expectation.json | 12 ++++++---
.../stage_with_accumulable_json_expectation.json | 8 ++++--
.../stage_with_peak_metrics_expectation.json | 12 ++++++---
...stage_with_speculation_summary_expectation.json | 20 +++++++++++----
.../stage_with_summaries_expectation.json | 16 +++++++++---
.../org/apache/spark/util/JsonProtocolSuite.scala | 30 ++++++++++++++--------
24 files changed, 217 insertions(+), 76 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
index 648532faa3a..1e80eb66dc5 100644
--- a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
@@ -110,10 +110,12 @@ case object GarbageCollectionMetrics extends
ExecutorMetricType with Logging {
"MinorGCTime",
"MajorGCCount",
"MajorGCTime",
- "TotalGCTime"
+ "TotalGCTime",
+ "ConcurrentGCCount",
+ "ConcurrentGCTime"
)
- /* We builtin some common GC collectors which categorized as young
generation and old */
+ /* We builtin some common GC collectors */
private[spark] val YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS = Seq(
"Copy",
"PS Scavenge",
@@ -128,6 +130,8 @@ case object GarbageCollectionMetrics extends
ExecutorMetricType with Logging {
"G1 Old Generation"
)
+ private[spark] val BUILTIN_CONCURRENT_GARBAGE_COLLECTOR = "G1 Concurrent GC"
+
private lazy val youngGenerationGarbageCollector: Seq[String] = {
SparkEnv.get.conf.get(config.EVENT_LOG_GC_METRICS_YOUNG_GENERATION_GARBAGE_COLLECTORS)
}
@@ -147,6 +151,9 @@ case object GarbageCollectionMetrics extends
ExecutorMetricType with Logging {
} else if (oldGenerationGarbageCollector.contains(mxBean.getName)) {
gcMetrics(2) = mxBean.getCollectionCount
gcMetrics(3) = mxBean.getCollectionTime
+ } else if (BUILTIN_CONCURRENT_GARBAGE_COLLECTOR.equals(mxBean.getName)) {
+ gcMetrics(5) = mxBean.getCollectionCount
+ gcMetrics(6) = mxBean.getCollectionTime
} else if (!nonBuiltInCollectors.contains(mxBean.getName)) {
nonBuiltInCollectors = mxBean.getName +: nonBuiltInCollectors
// log it when first seen
diff --git
a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
index 9658e5e6277..ca088dc8055 100644
---
a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
+++
b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
@@ -97,10 +97,10 @@ private[v1] class PrometheusResource extends
ApiRequestContext {
names.foreach { name =>
sb.append(s"$prefix${name}_bytes$labels ${m.getMetricValue(name)}\n")
}
- Seq("MinorGCCount", "MajorGCCount").foreach { name =>
+ Seq("MinorGCCount", "MajorGCCount", "ConcurrentGCCount").foreach {
name =>
sb.append(s"$prefix${name}_total$labels ${m.getMetricValue(name)}\n")
}
- Seq("MinorGCTime", "MajorGCTime").foreach { name =>
+ Seq("MinorGCTime", "MajorGCTime", "ConcurrentGCTime").foreach { name =>
sb.append(s"$prefix${name}_seconds_total$labels
${m.getMetricValue(name) * 0.001}\n")
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
index 850c3777ec4..ac0f2ce2605 100644
---
a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
@@ -76,7 +76,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
}
}, {
"status" : "COMPLETE",
@@ -156,7 +158,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
}
}, {
"status" : "COMPLETE",
@@ -236,6 +240,8 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
}
} ]
diff --git
a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
index ed4ed9ad871..d614bb000e4 100644
---
a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
@@ -887,7 +887,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : true
},
@@ -928,7 +930,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -956,6 +960,8 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
index f96a59fae53..475dee00a26 100644
---
a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
@@ -1021,7 +1021,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : true
},
@@ -1062,7 +1064,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : true
},
@@ -1103,7 +1107,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -1144,7 +1150,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -1185,7 +1193,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : true
}
@@ -1213,6 +1223,8 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
index ec3fc280b0a..a860682ca2e 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
@@ -42,7 +42,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
index 9b7498d9e91..2833cdcfde5 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
@@ -48,7 +48,9 @@
"MinorGCTime" : 55,
"MajorGCCount" : 3,
"MajorGCTime" : 144,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -108,7 +110,9 @@
"MinorGCTime" : 145,
"MajorGCCount" : 2,
"MajorGCTime" : 63,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : {
"NM_HTTP_ADDRESS" : "test-3.vpc.company.com:8042",
@@ -178,7 +182,9 @@
"MinorGCTime" : 106,
"MajorGCCount" : 2,
"MajorGCTime" : 75,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : {
"NM_HTTP_ADDRESS" : "test-4.vpc.company.com:8042",
@@ -248,7 +254,9 @@
"MinorGCTime" : 140,
"MajorGCCount" : 2,
"MajorGCTime" : 60,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : {
"NM_HTTP_ADDRESS" : "test-2.vpc.company.com:8042",
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index fbb7b6631f0..8a96858a201 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -85,7 +85,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -145,7 +147,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -205,7 +209,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -265,7 +271,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
index fbb7b6631f0..8a96858a201 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
@@ -85,7 +85,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -145,7 +147,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -205,7 +209,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -265,7 +271,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json
index b72ed0a6254..0e5e73f36fa 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json
@@ -73,7 +73,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -127,7 +129,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -181,7 +185,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -235,7 +241,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"attributes" : { },
"resources" : { },
diff --git
a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
index fee7377f181..dc1bcd6a396 100644
---
a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
@@ -75,7 +75,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
diff --git
a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
index 9e390a995c3..e24ac4f82b8 100644
---
a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
@@ -92,7 +92,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -120,6 +122,8 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index 887d2678e61..659e3c41d92 100644
---
a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -595,7 +595,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -625,6 +627,8 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index 3bb59aaf5b5..f84cf26fcf1 100644
---
a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -595,7 +595,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -625,6 +627,8 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
}
} ]
diff --git
a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
index b688b72b04d..564f3eadd1c 100644
---
a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
@@ -597,7 +597,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -625,6 +627,8 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
}
} ]
diff --git
a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
index 83ffb7da8e7..2bf7f348037 100644
---
a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
@@ -721,7 +721,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -749,7 +751,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
index e3cd9809434..8df41bfcc8d 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
@@ -74,7 +74,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
@@ -155,7 +157,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
@@ -235,7 +239,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
@@ -315,7 +321,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
index e4caffcf107..730df3fbd53 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
@@ -78,7 +78,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
index d3459be777d..16d92244ee0 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
@@ -74,7 +74,9 @@
"MinorGCTime" : 115,
"MajorGCCount" : 4,
"MajorGCTime" : 339,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
@@ -155,7 +157,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
@@ -236,7 +240,9 @@
"MinorGCTime" : 33,
"MajorGCCount" : 3,
"MajorGCTime" : 110,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index 3880818a7b5..e38741b7bf6 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -639,7 +639,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -667,7 +669,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
index d3eb7d55e0e..630b0512e8f 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
@@ -1147,7 +1147,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -1188,7 +1190,9 @@
"MinorGCTime" : 115,
"MajorGCCount" : 4,
"MajorGCTime" : 339,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -1216,7 +1220,9 @@
"MinorGCTime" : 115,
"MajorGCCount" : 4,
"MajorGCTime" : 339,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isShufflePushEnabled" : false,
"shuffleMergersCount" : 0
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
index 3ad18f816fb..23770480ad6 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
@@ -424,7 +424,9 @@
"MinorGCTime" : 280,
"MajorGCCount" : 2,
"MajorGCTime" : 1116,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -465,7 +467,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -506,7 +510,9 @@
"MinorGCTime" : 587,
"MajorGCCount" : 2,
"MajorGCTime" : 906,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -547,7 +553,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -584,6 +592,8 @@
"MinorGCTime" : 587,
"MajorGCCount" : 2,
"MajorGCTime" : 1116,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
index c89b82caf38..c8458a40958 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
@@ -1147,7 +1147,9 @@
"MinorGCTime" : 0,
"MajorGCCount" : 0,
"MajorGCTime" : 0,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -1188,7 +1190,9 @@
"MinorGCTime" : 115,
"MajorGCCount" : 4,
"MajorGCTime" : 339,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -1216,7 +1220,9 @@
"MinorGCTime" : 115,
"MajorGCCount" : 4,
"MajorGCTime" : 339,
- "TotalGCTime" : 0
+ "TotalGCTime" : 0,
+ "ConcurrentGCCount" : 0,
+ "ConcurrentGCTime" : 0
},
"taskMetricsDistributions" : {
"quantiles" : [ 0.0, 0.25, 0.5, 0.75, 1.0 ],
@@ -1306,7 +1312,9 @@
"MinorGCTime" : [ 0.0, 0.0, 115.0, 115.0, 115.0 ],
"MajorGCCount" : [ 0.0, 0.0, 4.0, 4.0, 4.0 ],
"MajorGCTime" : [ 0.0, 0.0, 339.0, 339.0, 339.0 ],
- "TotalGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+ "TotalGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "ConcurrentGCCount" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "ConcurrentGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
}
},
"isShufflePushEnabled" : false,
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index 8105df64705..e8d41c4d46e 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -58,21 +58,21 @@ class JsonProtocolSuite extends SparkFunSuite {
makeTaskInfo(123L, 234, 67, 234, 345L, false),
new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L,
- 0, 0, 0, 0, 80001L)),
+ 0, 0, 0, 0, 80001L, 3, 3)),
makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0,
hasHadoopInput = false, hasOutput = false))
val taskEndWithHadoopInput = SparkListenerTaskEnd(1, 0, "ShuffleMapTask",
Success,
makeTaskInfo(123L, 234, 67, 234, 345L, false),
new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L,
- 0, 0, 0, 0, 80001L)),
+ 0, 0, 0, 0, 80001L, 3, 3)),
makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0,
hasHadoopInput = true, hasOutput = false))
val taskEndWithOutput = SparkListenerTaskEnd(1, 0, "ResultTask", Success,
makeTaskInfo(123L, 234, 67, 234, 345L, false),
new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L,
- 0, 0, 0, 0, 80001L)),
+ 0, 0, 0, 0, 80001L, 3, 3)),
makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0,
hasHadoopInput = true, hasOutput = true))
val jobStart = {
@@ -136,7 +136,7 @@ class JsonProtocolSuite extends SparkFunSuite {
val executorUpdates = new ExecutorMetrics(
Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L,
- 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L))
+ 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L, 3, 3))
SparkListenerExecutorMetricsUpdate("exec3", Seq((1L, 2, 3,
accumUpdates)),
Map((0, 0) -> executorUpdates))
}
@@ -147,7 +147,7 @@ class JsonProtocolSuite extends SparkFunSuite {
SparkListenerStageExecutorMetrics("1", 2, 3,
new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L,
432L,
321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L,
- 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L)))
+ 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L, 3, 3)))
val rprofBuilder = new ResourceProfileBuilder()
val taskReq = new TaskResourceRequests()
.cpus(1)
@@ -1754,7 +1754,9 @@ private[spark] object JsonProtocolSuite extends
Assertions {
| "MinorGCTime" : 0,
| "MajorGCCount" : 0,
| "MajorGCTime" : 0,
- | "TotalGCTime" : 80001
+ | "TotalGCTime": 80001,
+ | "ConcurrentGCCount" : 3,
+ | "ConcurrentGCTime" : 3
| },
| "Task Metrics": {
| "Executor Deserialize Time": 300,
@@ -1893,7 +1895,9 @@ private[spark] object JsonProtocolSuite extends
Assertions {
| "MinorGCTime" : 0,
| "MajorGCCount" : 0,
| "MajorGCTime" : 0,
- | "TotalGCTime" : 80001
+ | "TotalGCTime": 80001,
+ | "ConcurrentGCCount" : 3,
+ | "ConcurrentGCTime" : 3
| },
| "Task Metrics": {
| "Executor Deserialize Time": 300,
@@ -2032,7 +2036,9 @@ private[spark] object JsonProtocolSuite extends
Assertions {
| "MinorGCTime" : 0,
| "MajorGCCount" : 0,
| "MajorGCTime" : 0,
- | "TotalGCTime" : 80001
+ | "TotalGCTime": 80001,
+ | "ConcurrentGCCount" : 3,
+ | "ConcurrentGCTime" : 3
| },
| "Task Metrics": {
| "Executor Deserialize Time": 300,
@@ -2933,7 +2939,9 @@ private[spark] object JsonProtocolSuite extends
Assertions {
| "MinorGCTime": 90,
| "MajorGCCount": 2,
| "MajorGCTime": 20,
- | "TotalGCTime" : 80001
+ | "TotalGCTime": 80001,
+ | "ConcurrentGCCount" : 3,
+ | "ConcurrentGCTime" : 3
| }
| }
| ]
@@ -2968,7 +2976,9 @@ private[spark] object JsonProtocolSuite extends
Assertions {
| "MinorGCTime": 90,
| "MajorGCCount": 2,
| "MajorGCTime": 20,
- | "TotalGCTime" : 80001
+ | "TotalGCTime": 80001,
+ | "ConcurrentGCCount" : 3,
+ | "ConcurrentGCTime" : 3
| }
|}
""".stripMargin
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
