Github user hvanhovell commented on a diff in the pull request:
https://github.com/apache/spark/pull/16046#discussion_r90064101
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
---
@@ -1120,47 +1173,54 @@ class Analyzer(
} else {
a
}
- case w : Window =>
- failOnOuterReference(w)
- failOnNonEqualCorrelatedPredicate(foundNonEqualCorrelatedPred, w)
- w
- case j @ Join(left, _, RightOuter, _) =>
- failOnOuterReference(j)
- failOnOuterReferenceInSubTree(left, "a RIGHT OUTER JOIN")
- j
- // SPARK-18578: Do not allow any correlated predicate
- // in a Full (Outer) Join operator and its descendants
- case j @ Join(_, _, FullOuter, _) =>
- failOnOuterReferenceInSubTree(j, "a FULL OUTER JOIN")
- j
- case j @ Join(_, right, jt, _) if !jt.isInstanceOf[InnerLike] =>
- failOnOuterReference(j)
- failOnOuterReferenceInSubTree(right, "a LEFT (OUTER) JOIN")
+
+ // Join can host correlated expressions.
+ case j @ Join(left, right, joinType, _) =>
+ joinType match {
+ // Inner join, like Filter, can be anywhere.
+ // LeftSemi is a special case of Inner join which returns
+ // only the first matched row to the right table.
+ case _: InnerLike | LeftSemi =>
+ failOnOuterReference(j)
+
+ // Left outer join's right operand cannot be on a correlation
path.
+ // LeftAnti and ExistenceJoin are special cases of LeftOuter.
+ // Note that ExistenceJoin cannot be expressed externally in
both SQL and DataFrame
+ // so it should not show up here in Analysis phase. This is
just a safety net.
+ case LeftOuter | LeftAnti | ExistenceJoin(_) =>
+ failOnOuterReference(j)
+ failOnOuterReferenceInSubTree(right)
+
+ // Likewise, Right outer join's left operand cannot be on a
correlation path.
+ case RightOuter =>
+ failOnOuterReference(j)
+ failOnOuterReferenceInSubTree(left)
+
+ // Any other join types not explicitly listed above,
+ // including Full outer join, are treated as Category 4.
+ case _ =>
+ failOnOuterReferenceInSubTree(j)
+ }
j
- case u: Union =>
- failOnOuterReferenceInSubTree(u, "a UNION")
- u
- case s: SetOperation =>
- failOnOuterReferenceInSubTree(s.right, "an INTERSECT/EXCEPT")
- s
- case e: Expand =>
- failOnOuterReferenceInSubTree(e, "an EXPAND")
- e
- case l : LocalLimit =>
- failOnOuterReferenceInSubTree(l, "a LIMIT")
- l
- // Since LIMIT <n> is represented as GlobalLimit(<n>, (LocalLimit
(<n>, child))
- // and we are walking bottom up, we will fail on LocalLimit before
- // reaching GlobalLimit.
- // The code below is just a safety net.
- case g : GlobalLimit =>
- failOnOuterReferenceInSubTree(g, "a LIMIT")
- g
- case s : Sample =>
- failOnOuterReferenceInSubTree(s, "a TABLESAMPLE")
- s
+
+ // Generator with join=true, i.e., expressed with
+ // LATERAL VIEW [OUTER], similar to inner join,
+ // allows to have correlation under it
+ // but must not host any outer references.
+ // Note:
+ // Generator with join=false is treated as Category 4.
+ case p @ Generate(generator, join, _, _, _, _) if (join) =>
--- End diff --
If you only use the columns generated by generator, the optimizer will set
`join=false`. There is no way to do this in SQL.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
