Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/17636#discussion_r111829450
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
---
@@ -1210,16 +1210,41 @@ class Analyzer(
private def checkAndGetOuterReferences(sub: LogicalPlan):
Seq[Expression] = {
val outerReferences = ArrayBuffer.empty[Expression]
+ // Validate that correlated aggregate expression do not contain a
mixture
+ // of outer and local references.
+ def checkMixedReferencesInsideAggregateExpr(expr: Expression): Unit
= {
+ expr.foreach {
+ case a: AggregateExpression if containsOuter(a) =>
+ val outer = a.collect { case OuterReference(e) =>
e.toAttribute }
+ val local = a.references -- outer
+ if (local.nonEmpty) {
+ val msg =
+ s"""
+ |Found an aggregate expression in a correlated
predicate that has both
+ |outer and local references, which is not supported yet.
+ |Aggregate expression: ${a.sql},
+ |Outer references: ${outer.map(_.sql).mkString(", ")},
+ |Local references: ${local.map(_.sql).mkString(", ")}.
+ """.stripMargin.replace("\n", " ").trim()
+ failAnalysis(msg)
+ }
+ case _ =>
+ }
+ }
+
// Make sure a plan's subtree does not contain outer references
def failOnOuterReferenceInSubTree(p: LogicalPlan): Unit = {
if (hasOuterReferences(p)) {
failAnalysis(s"Accessing outer query column is not allowed
in:\n$p")
}
}
- // Make sure a plan's expressions do not contain outer references
- def failOnOuterReference(p: LogicalPlan): Unit = {
- if (p.expressions.exists(containsOuter)) {
+ // Make sure a plan's expressions do not contain :
+ // 1. Aggregate expressions that has mixture of outer and local
references.
--- End diff --
Nit: `has` -> `have`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
