cloud-fan commented on code in PR #37612:
URL: https://github.com/apache/spark/pull/37612#discussion_r953537151
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ValidateSparkPlan.scala:
##########
@@ -39,30 +45,41 @@ object ValidateSparkPlan extends Rule[SparkPlan] {
* - BroadcastQueryStage only appears as the immediate child and the build
side of a broadcast
* hash join or broadcast nested loop join.
*/
- private def validate(plan: SparkPlan): Unit = plan match {
+ private def validateBroadcastQueryStage(plan: SparkPlan): Unit = plan match {
case b: BroadcastHashJoinExec =>
val (buildPlan, probePlan) = b.buildSide match {
case BuildLeft => (b.left, b.right)
case BuildRight => (b.right, b.left)
}
if (!buildPlan.isInstanceOf[BroadcastQueryStageExec]) {
- validate(buildPlan)
+ validateBroadcastQueryStage(buildPlan)
}
- validate(probePlan)
+ validateBroadcastQueryStage(probePlan)
case b: BroadcastNestedLoopJoinExec =>
val (buildPlan, probePlan) = b.buildSide match {
case BuildLeft => (b.left, b.right)
case BuildRight => (b.right, b.left)
}
if (!buildPlan.isInstanceOf[BroadcastQueryStageExec]) {
- validate(buildPlan)
+ validateBroadcastQueryStage(buildPlan)
}
- validate(probePlan)
+ validateBroadcastQueryStage(probePlan)
case q: BroadcastQueryStageExec => errorOnInvalidBroadcastQueryStage(q)
- case _ => plan.children.foreach(validate)
+ case _ => plan.children.foreach(validateBroadcastQueryStage)
}
private def errorOnInvalidBroadcastQueryStage(plan: SparkPlan): Unit = {
throw InvalidAQEPlanException("Invalid broadcast query stage", plan)
}
+
+ /**
+ * Validate that the output partitioning of plan satisfies the
`userSpecifiedDistribution`.
+ */
+ private def validateOutputPartitioning(plan: SparkPlan): Unit = {
+ if (userSpecifiedDistribution.isDefined &&
+ !plan.outputPartitioning.satisfies(userSpecifiedDistribution.get)) {
+ throw InvalidAQEPlanException(s"Output partitioning does not satisfy the
specified, " +
Review Comment:
do we have a test case to cover this case, or it's just a safe guard for now?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
