gaborgsomogyi commented on a change in pull request #30336:
URL: https://github.com/apache/spark/pull/30336#discussion_r525182879
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
##########
@@ -236,12 +239,61 @@ private[ui] class StreamingQueryStatisticsPage(parent:
StreamingQueryTab)
<td
class={"aggregated-num-state-rows-dropped-by-watermark-timeline"}>{graphUIDataForNumRowsDroppedByWatermark.generateTimelineHtml(jsCollector)}</td>
<td
class={"aggregated-num-state-rows-dropped-by-watermark-histogram"}>{graphUIDataForNumRowsDroppedByWatermark.generateHistogramHtml(jsCollector)}</td>
</tr>
- // scalastyle:on
+ // scalastyle:on
+
+ result ++= generateAggregatedCustomMetrics(query, minBatchTime,
maxBatchTime, jsCollector)
+ result
} else {
new NodeBuffer()
}
}
+ def generateAggregatedCustomMetrics(
+ query: StreamingQueryUIData,
+ minBatchTime: Long,
+ maxBatchTime: Long,
+ jsCollector: JsCollector): NodeBuffer = {
+ val result: NodeBuffer = new NodeBuffer
+
+ // This is made sure on caller side but put it here to be defensive
+ require(query.lastProgress.stateOperators.nonEmpty)
+ val enabledCustomMetrics =
parent.parent.conf.get(ENABLED_STREAMING_UI_CUSTOM_METRIC_LIST)
+ logDebug(s"Enabled custom metrics: $enabledCustomMetrics")
+ query.lastProgress.stateOperators.head.customMetrics.keySet().asScala
+ .filter(enabledCustomMetrics.contains(_)).map { metricName =>
+ val data = query.recentProgress.map(p =>
(parseProgressTimestamp(p.timestamp),
+ p.stateOperators.map(_.customMetrics.get(metricName).toDouble).sum))
+ val max = data.maxBy(_._2)._2
+
+ val graphUIData =
+ new GraphUIData(
+ s"aggregated-$metricName-timeline",
+ s"aggregated-$metricName-histogram",
+ data,
+ minBatchTime,
+ maxBatchTime,
+ 0,
+ max,
+ "")
Review comment:
It would be definitely good to add something better, but I have similar
problem like in tooltip. Every custom metric can have its own unit.
I've just double checked and custom metrics stored in the following way:
`java.util.Map[String, java.lang.Long]`
Since this structure doesn't contain any unit what the metric unit is I'm
not sure from where it could come from.
Any suggestion?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
