marin-ma commented on code in PR #6475:
URL: https://github.com/apache/incubator-gluten/pull/6475#discussion_r1688973227
##########
backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala:
##########
@@ -258,27 +258,33 @@ class VeloxMetricsApi extends MetricsApi with Logging {
sparkContext: SparkContext,
isSort: Boolean): Map[String, SQLMetric] = {
val baseMetrics = Map(
- "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size"),
"numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of
partitions"),
+ "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size"),
"bytesSpilled" -> SQLMetrics.createSizeMetric(sparkContext, "shuffle
bytes spilled"),
- "splitBufferSize" -> SQLMetrics.createSizeMetric(sparkContext, "split
buffer size"),
- "splitTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to
split"),
- "spillTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to
spill"),
- "deserializeTime" -> SQLMetrics.createNanoTimingMetric(sparkContext,
"time to deserialize"),
"avgReadBatchNumRows" -> SQLMetrics
.createAverageMetric(sparkContext, "avg read batch num rows"),
"numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input
rows"),
"numOutputRows" -> SQLMetrics
.createMetric(sparkContext, "number of output rows"),
"inputBatches" -> SQLMetrics
- .createMetric(sparkContext, "number of input batches")
+ .createMetric(sparkContext, "number of input batches"),
+ "spillTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to
spill"),
+ "compressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time
to compress"),
+ "decompressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext,
"time to decompress"),
+ "deserializeTime" -> SQLMetrics.createNanoTimingMetric(sparkContext,
"time to deserialize"),
+ // For hash shuffle writer, the peak bytes represents the maximum split
buffer size.
+ // For sort shuffle writer, the peak bytes represents the maximum
+ // row buffer + sort buffer size.
+ "peakBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak bytes
allocated")
)
if (isSort) {
- baseMetrics
+ baseMetrics ++ Map(
+ "sortTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to
shuffle sort"),
+ "c2rTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to
shuffle c2r")
Review Comment:
> Perhaps we can introduce a new metrics shuffle write wall time to
calculate the real time for both hash and sort shuffle, and modify the analysis
script to use this one.
Will address in follow-up PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
