erenavsarogullari commented on a change in pull request #28208:
URL: https://github.com/apache/spark/pull/28208#discussion_r426323971
##########
File path:
sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
##########
@@ -21,46 +21,92 @@ import java.util.Date
import javax.ws.rs._
import javax.ws.rs.core.MediaType
+import scala.util.{Failure, Success, Try}
+
import org.apache.spark.JobExecutionStatus
-import org.apache.spark.sql.execution.ui.{SQLAppStatusStore,
SQLExecutionUIData, SQLPlanMetric}
+import org.apache.spark.sql.execution.ui.{SparkPlanGraph,
SparkPlanGraphCluster, SparkPlanGraphEdge, SparkPlanGraphNode,
SQLAppStatusStore, SQLExecutionUIData}
import org.apache.spark.status.api.v1.{BaseAppResource, NotFoundException}
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class SqlResource extends BaseAppResource {
+ val WHOLE_STAGE_CODEGEN = "WholeStageCodegen"
+
@GET
def sqlList(
- @DefaultValue("false") @QueryParam("details") details: Boolean,
+ @DefaultValue("true") @QueryParam("details") details: Boolean,
+ @DefaultValue("true") @QueryParam("planDescription") planDescription:
Boolean,
@DefaultValue("0") @QueryParam("offset") offset: Int,
@DefaultValue("20") @QueryParam("length") length: Int):
Seq[ExecutionData] = {
withUI { ui =>
val sqlStore = new SQLAppStatusStore(ui.store.store)
- sqlStore.executionsList(offset, length).map(prepareExecutionData(_,
details))
+ sqlStore.executionsList(offset, length).map { exec =>
+ val (allNodes, edges, nodeIdAndWSCGIdMap) =
+ computeDetailsIfTrue(sqlStore, exec.executionId, details)
+ prepareExecutionData(exec, allNodes, edges, nodeIdAndWSCGIdMap,
+ details = details, planDescription = planDescription)
+ }
}
}
@GET
@Path("{executionId:\\d+}")
def sql(
@PathParam("executionId") execId: Long,
- @DefaultValue("false") @QueryParam("details") details: Boolean):
ExecutionData = {
+ @DefaultValue("true") @QueryParam("details") details: Boolean,
+ @DefaultValue("true") @QueryParam("planDescription")
+ planDescription: Boolean): ExecutionData = {
withUI { ui =>
val sqlStore = new SQLAppStatusStore(ui.store.store)
+ val (allNodes, edges, nodeIdAndWSCGIdMap) =
computeDetailsIfTrue(sqlStore, execId, details)
sqlStore
.execution(execId)
- .map(prepareExecutionData(_, details))
- .getOrElse(throw new NotFoundException("unknown id: " + execId))
+ .map(prepareExecutionData(_, allNodes, edges, nodeIdAndWSCGIdMap,
details, planDescription))
+ .getOrElse(throw new NotFoundException("unknown query execution id: "
+ execId))
}
}
- private def printableMetrics(
- metrics: Seq[SQLPlanMetric],
- metricValues: Map[Long, String]): Seq[Metrics] = {
- metrics.map(metric =>
- Metrics(metric.name,
metricValues.get(metric.accumulatorId).getOrElse("")))
+ private def computeDetailsIfTrue(sqlStore: SQLAppStatusStore,
+ executionId: Long,
+ details: Boolean):
+ (Seq[SparkPlanGraphNode], Seq[SparkPlanGraphEdge], Map[Long, Option[Long]])
= {
+ if (details) {
+ val graph = sqlStore.planGraph(executionId)
+ val nodeIdAndWSCGIdMap: Map[Long, Option[Long]] =
getNodeIdAndWSCGIdMap(graph)
+ (graph.allNodes, graph.edges, nodeIdAndWSCGIdMap)
+ } else {
+ (Seq.empty, Seq.empty, Map.empty)
+ }
}
- private def prepareExecutionData(exec: SQLExecutionUIData, details:
Boolean): ExecutionData = {
+ private def getNodeIdAndWSCGIdMap(graph: SparkPlanGraph): Map[Long,
Option[Long]] = {
Review comment:
WSCG node has both `nodeId` and WSCG `index` as follows. WSCG `index`
comes as part of WSCG `nodeName` and needs to be parsed before populating of
other nodes' for their `wholeStageCodegenId` attribute so this map is useful
for computation and readability by reducing complexity. Also, i cleaned up
existing implementation by covering this part. Please find final patch:
2743296ee44083eedc743a9572de0121af825c88
```
{
"nodeId": 2,
"nodeName": "WholeStageCodegen (1)",
"metrics": [...]
}
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
