tgravescs commented on code in PR #34622:
URL: https://github.com/apache/spark/pull/34622#discussion_r855300474
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListener.scala:
##########
@@ -202,6 +227,49 @@ class SQLAppStatusListener(
}
}
+ /* Connects Operators to Stages by doing the following:
+ * 1. Read SparkGraph to get every Node's name and respective AccumulatorIDs.
+ * 2. Gets each stage's AccumulatorIDs.
+ * 3. Maps Operators to stages by checking for non-zero intersection of 1
and 2's AccumulatorIDs.
+ * 4. Connect SparkGraphNodes to respective StageIDs for rendering in SQL UI.
+ */
+ private def connectOperatorToStage(exec: LiveExecutionData): Map[Long,
List[Int]] = {
+ // Reads in SparkPlanGraph to get each Node's name and respective
accumulatorIDs
+ // and saves it inside nodeNameToAccumulatorIds.
+ val planGraph = kvstore.read(classOf[SparkPlanGraphWrapper],
exec.executionId)
+ .toSparkPlanGraph()
+ val nodeNameToAccumulatorIds = planGraph.allNodes.map { node =>
+ (node.name, node.metrics.map(_.accumulatorId)) }
+ logDebug("each Operator's Metrics represented by AccumulatorIds are: \n
%s\n"
+ .format(nodeNameToAccumulatorIds.mkString("\n ")))
+
+ // Gets each stage and its list of distinct accumulatorIDs
+ // which is retrieved upon completion of a stage.
+ val stageIdToAccumulatorIDs = stageAccumulators.asScala
+ logDebug("each Stage's Metrics represented by AccumulatorIds are: \n %s\n"
+ .format(stageIdToAccumulatorIDs.mkString("\n ")))
+
+ // Maps stages to operators by checking for non-zero intersection
+ // between nodeMetrics and stageAccumulateIDs
+ val operatorToStage = nodeNameToAccumulatorIds.map { case (nodeName,
accumulatorIds1) =>
+ val mappedStages = stageIdToAccumulatorIDs.flatMap { case (stageId,
accumulatorIds2) =>
Review Comment:
nit rename accumulatorIds2 to stageAccumIds and above accumulatorIds1 to be
sqlNodeAccumIds.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
