gaborgsomogyi commented on a change in pull request #30336:
URL: https://github.com/apache/spark/pull/30336#discussion_r526029940
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
##########
@@ -236,12 +239,61 @@ private[ui] class StreamingQueryStatisticsPage(parent:
StreamingQueryTab)
<td
class={"aggregated-num-state-rows-dropped-by-watermark-timeline"}>{graphUIDataForNumRowsDroppedByWatermark.generateTimelineHtml(jsCollector)}</td>
<td
class={"aggregated-num-state-rows-dropped-by-watermark-histogram"}>{graphUIDataForNumRowsDroppedByWatermark.generateHistogramHtml(jsCollector)}</td>
</tr>
- // scalastyle:on
+ // scalastyle:on
+
+ result ++= generateAggregatedCustomMetrics(query, minBatchTime,
maxBatchTime, jsCollector)
+ result
} else {
new NodeBuffer()
}
}
+ def generateAggregatedCustomMetrics(
+ query: StreamingQueryUIData,
+ minBatchTime: Long,
+ maxBatchTime: Long,
+ jsCollector: JsCollector): NodeBuffer = {
+ val result: NodeBuffer = new NodeBuffer
+
+ // This is made sure on caller side but put it here to be defensive
+ require(query.lastProgress.stateOperators.nonEmpty)
+ val enabledCustomMetrics =
parent.parent.conf.get(ENABLED_STREAMING_UI_CUSTOM_METRIC_LIST)
+ logDebug(s"Enabled custom metrics: $enabledCustomMetrics")
+ query.lastProgress.stateOperators.head.customMetrics.keySet().asScala
+ .filter(enabledCustomMetrics.contains(_)).map { metricName =>
+ val data = query.recentProgress.map(p =>
(parseProgressTimestamp(p.timestamp),
+ p.stateOperators.map(_.customMetrics.get(metricName).toDouble).sum))
+ val max = data.maxBy(_._2)._2
+
+ val graphUIData =
+ new GraphUIData(
+ s"aggregated-$metricName-timeline",
+ s"aggregated-$metricName-histogram",
+ data,
+ minBatchTime,
+ maxBatchTime,
+ 0,
+ max,
+ "")
Review comment:
I've added it and working fine so closing this thread.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
