This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 8817dfd75 [GLUTEN-6531] Minor polish for metrics code (#6532)
8817dfd75 is described below
commit 8817dfd752b5686ddd6e141678114040030ea4ab
Author: James Xu <[email protected]>
AuthorDate: Tue Aug 6 11:53:14 2024 +0800
[GLUTEN-6531] Minor polish for metrics code (#6532)
---
.../gluten/backendsapi/velox/VeloxMetricsApi.scala | 10 ++++------
.../scala/org/apache/gluten/metrics/MetricsUtil.scala | 17 +++++++++--------
2 files changed, 13 insertions(+), 14 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
index c05ce7cdc..49d45a396 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
@@ -34,7 +34,7 @@ class VeloxMetricsApi extends MetricsApi with Logging {
relMap: JMap[JLong, JList[JLong]],
joinParamsMap: JMap[JLong, JoinParams],
aggParamsMap: JMap[JLong, AggregationParams]): IMetrics => Unit = {
- MetricsUtil.updateNativeMetrics(child, relMap, joinParamsMap, aggParamsMap)
+ MetricsUtil.genMetricsUpdatingFunction(child, relMap, joinParamsMap,
aggParamsMap)
}
override def genInputIteratorTransformerMetrics(
@@ -356,9 +356,7 @@ class VeloxMetricsApi extends MetricsApi with Logging {
"numMemoryAllocations" -> SQLMetrics.createMetric(
sparkContext,
"number of memory allocations"),
- "spilledBytes" -> SQLMetrics.createSizeMetric(
- sparkContext,
- "total bytes written for spilling"),
+ "spilledBytes" -> SQLMetrics.createSizeMetric(sparkContext, "bytes
written for spilling"),
"spilledRows" -> SQLMetrics.createMetric(sparkContext, "total rows
written for spilling"),
"spilledPartitions" -> SQLMetrics.createMetric(sparkContext, "total
spilled partitions"),
"spilledFiles" -> SQLMetrics.createMetric(sparkContext, "total spilled
files")
@@ -445,7 +443,7 @@ class VeloxMetricsApi extends MetricsApi with Logging {
"number of hash build memory allocations"),
"hashBuildSpilledBytes" -> SQLMetrics.createSizeMetric(
sparkContext,
- "total bytes written for spilling of hash build"),
+ "bytes written for spilling of hash build"),
"hashBuildSpilledRows" -> SQLMetrics.createMetric(
sparkContext,
"total rows written for spilling of hash build"),
@@ -479,7 +477,7 @@ class VeloxMetricsApi extends MetricsApi with Logging {
"number of hash probe memory allocations"),
"hashProbeSpilledBytes" -> SQLMetrics.createSizeMetric(
sparkContext,
- "total bytes written for spilling of hash probe"),
+ "bytes written for spilling of hash probe"),
"hashProbeSpilledRows" -> SQLMetrics.createMetric(
sparkContext,
"total rows written for spilling of hash probe"),
diff --git
a/gluten-data/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala
b/gluten-data/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala
index 0c387b429..eff4245d5 100644
--- a/gluten-data/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala
+++ b/gluten-data/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala
@@ -28,7 +28,7 @@ import java.util.{ArrayList => JArrayList, List => JList, Map
=> JMap}
object MetricsUtil extends Logging {
/**
- * Update metrics fetched from certain iterator to transformers.
+ * Generate the function which updates metrics fetched from certain iterator
to transformers.
*
* @param child
* the child spark plan
@@ -39,7 +39,7 @@ object MetricsUtil extends Logging {
* @param aggParamsMap
* the map between operator index and aggregation parameters
*/
- def updateNativeMetrics(
+ def genMetricsUpdatingFunction(
child: SparkPlan,
relMap: JMap[JLong, JList[JLong]],
joinParamsMap: JMap[JLong, JoinParams],
@@ -66,7 +66,7 @@ object MetricsUtil extends Logging {
val mut: MetricsUpdaterTree = treeifyMetricsUpdaters(child)
- updateTransformerMetrics(
+ genMetricsUpdatingFunction(
mut,
relMap,
JLong.valueOf(relMap.size() - 1),
@@ -269,24 +269,25 @@ object MetricsUtil extends Logging {
}
/**
- * A recursive function updating the metrics of one transformer and its
child.
+ * Get a function which would update the metrics of transformers.
*
- * @param mut
+ * @param mutNode
* the metrics updater tree built from the original plan
* @param relMap
* the map between operator index and its rels
* @param operatorIdx
* the index of operator
- * @param metrics
- * the metrics fetched from native
* @param metricsIdx
* the index of metrics
* @param joinParamsMap
* the map between operator index and join parameters
* @param aggParamsMap
* the map between operator index and aggregation parameters
+ *
+ * @return
+ * A recursive function updating the metrics of operator(transformer) and
its children.
*/
- def updateTransformerMetrics(
+ def genMetricsUpdatingFunction(
mutNode: MetricsUpdaterTree,
relMap: JMap[JLong, JList[JLong]],
operatorIdx: JLong,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
