Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/418#discussion_r11936670
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
---
@@ -102,6 +105,55 @@ object PhysicalOperation extends PredicateHelper {
}
/**
+ * A pattern that finds joins with equality conditions that can be
evaluated using hashing
+ * techniques. For inner joins, any filters on top of the join operator
are also matched.
+ */
+object HashFilteredJoin extends Logging with PredicateHelper {
+ /** (joinType, rightKeys, leftKeys, condition, left, right) */
+ type ReturnType =
+ (JoinType, Seq[Expression], Seq[Expression], Option[Expression],
LogicalPlan, LogicalPlan)
+
+ def unapply(plan: LogicalPlan): Option[ReturnType] = plan match {
+ // All predicates can be evaluated for inner join (i.e., those that
are in the ON
+ // clause and WHERE clause.)
+ case FilteredOperation(predicates, join @ Join(left, right, Inner,
condition)) =>
+ logger.debug(s"Considering hash inner join on: ${predicates ++
condition}")
+ splitPredicates(predicates ++ condition, join)
+ case join @ Join(left, right, joinType, condition) =>
+ logger.debug(s"Considering hash join on: $condition")
+ splitPredicates(condition.toSeq, join)
+ case _ => None
+ }
+
+ // Find equi-join predicates that can be evaluated before the join, and
thus can be used
+ // as join keys.
+ def splitPredicates(allPredicates: Seq[Expression], join: Join):
Option[ReturnType] = {
+ val Join(left, right, joinType, _) = join
+ val (joinPredicates, otherPredicates) = allPredicates.partition {
+ case Equals(l, r) if (canEvaluate(l, left) && canEvaluate(r, right))
||
+ (canEvaluate(l, right) && canEvaluate(r, left)) => true
+ case _ => false
+ }
+
+ val joinKeys = joinPredicates.map {
+ case Equals(l,r) if canEvaluate(l, left) && canEvaluate(r, right) =>
(l, r)
--- End diff --
nit here: space after comma; also for the following line.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
