wangyum commented on code in PR #40744:
URL: https://github.com/apache/spark/pull/40744#discussion_r1212534490
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala:
##########
@@ -714,6 +717,121 @@ case class UnionExec(children: Seq[SparkPlan]) extends
SparkPlan {
copy(children = newChildren)
}
+/**
+ * The physical node for recursion.
+ *
+ * @param loopId The id of the loop.
+ * @param anchor The logical plan of the initial element of the loop.
+ * @param recursion The logical plan that describes the recursion with an
[[UnionLoopRef]] node.
+ * @param output The output attributes of this loop.
+ * @param limit An optional limit that can be pushed down to the node to stop
the loop earlier.
+ */
+case class UnionLoopExec(
+ loopId: Long,
+ @transient anchor: LogicalPlan,
+ @transient recursion: LogicalPlan,
+ override val output: Seq[Attribute],
+ limit: Option[Int] = None) extends LeafExecNode {
+
+ val levelLimit = conf.getConf(SQLConf.CTE_RECURSION_LEVEL_LIMIT)
+
+ // We store the initial and generated children of the loop in this buffer.
+ // Please note that all elements are cached because of performance reasons
as they are needed for
+ // next iteration.
+ @transient private val unionDFs = mutable.ArrayBuffer.empty[DataFrame]
+
+ override def innerChildren: Seq[QueryPlan[_]] = Seq(anchor, recursion)
+
+ override lazy val metrics = Map(
+ "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output
rows"))
+
+ private def cacheAndCount(plan: LogicalPlan, limit: Option[Long]) = {
+ val limitedPlan = limit.map(l => Limit(Literal(l.toInt),
plan)).getOrElse(plan)
+ val df = Dataset.ofRows(session, limitedPlan).persist()
Review Comment:
Could we replace `persist()` with `repartition()` to avoid stack overflow
issue?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
