TAJO-1350: Refactor FilterPushDownRule::visitJoin() into well-defined, small methods. (jihoon)
Closes #384 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/b1e174ee Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/b1e174ee Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/b1e174ee Branch: refs/heads/index_support Commit: b1e174eec4c15142b6fa518b6803579bb0788e8e Parents: cad5442 Author: Jihoon Son <[email protected]> Authored: Fri Mar 27 11:49:42 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri Mar 27 11:49:42 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../planner/physical/HashLeftOuterJoinExec.java | 3 +- .../apache/tajo/engine/query/TestJoinQuery.java | 23 +- .../testComplexJoinsWithCaseWhen.sql | 11 + .../testComplexJoinsWithCaseWhen2.sql | 9 + .../TestJoinQuery/testCrossJoinAndCaseWhen.sql | 18 - .../TestJoinQuery/testInnerJoinAndCaseWhen.sql | 18 + .../TestJoinQuery/testJoinWithOrPredicates.sql | 6 + .../testComplexJoinsWithCaseWhen.result | 27 ++ .../testComplexJoinsWithCaseWhen2.result | 27 ++ .../testCrossJoinAndCaseWhen.result | 27 -- .../testInnerJoinAndCaseWhen.result | 27 ++ .../testJoinWithOrPredicates.result | 4 + .../org/apache/tajo/plan/LogicalPlanner.java | 60 ++- .../plan/rewrite/rules/FilterPushDownRule.java | 383 +++++++++---------- .../rewrite/rules/ProjectionPushDownRule.java | 5 +- .../org/apache/tajo/plan/util/PlannerUtil.java | 25 +- 17 files changed, 428 insertions(+), 248 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 029735f..60cb0f7 100644 --- a/CHANGES +++ b/CHANGES @@ -9,6 +9,9 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-1350: Refactor FilterPushDownRule::visitJoin() into well-defined, + small methods. (jihoon) + TAJO-1426: Support "explain global" to get physical plan. (Contributed by navis, Committed by jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java index e78cb20..81ac02c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java @@ -181,7 +181,7 @@ public class HashLeftOuterJoinExec extends BinaryPhysicalExec { boolean satisfiedWithJoinCondition = joinQual.eval(inSchema, frameTuple).isTrue(); // if a composited tuple satisfies with both join filter and join condition - if (satisfiedWithFilter && satisfiedWithJoinCondition) { + if (satisfiedWithJoinCondition && satisfiedWithFilter) { projector.eval(frameTuple, outTuple); return outTuple; } else { @@ -195,6 +195,7 @@ public class HashLeftOuterJoinExec extends BinaryPhysicalExec { // null padding Tuple nullPaddedTuple = TupleUtil.createNullPaddedTuple(rightNumCols); frameTuple.set(leftTuple, nullPaddedTuple); + projector.eval(frameTuple, outTuple); return outTuple; } http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java index 9ab32ff..1078943 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java @@ -338,7 +338,21 @@ public class TestJoinQuery extends QueryTestCaseBase { } @Test - public void testCrossJoinAndCaseWhen() throws Exception { + public void testInnerJoinAndCaseWhen() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public void testComplexJoinsWithCaseWhen() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public void testComplexJoinsWithCaseWhen2() throws Exception { ResultSet res = executeQuery(); assertResultSet(res); cleanupQuery(res); @@ -1170,4 +1184,11 @@ public class TestJoinQuery extends QueryTestCaseBase { assertResultSet(res); cleanupQuery(res); } + + @Test + public final void testJoinWithOrPredicates() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinsWithCaseWhen.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinsWithCaseWhen.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinsWithCaseWhen.sql new file mode 100644 index 0000000..b2c49a4 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinsWithCaseWhen.sql @@ -0,0 +1,11 @@ +select + r_name, + case when + s_name is null then 'N/O' + else + s_name + end as s1 +from + region inner join nation on n_regionkey = r_regionkey + left outer join supplier on s_nationkey = n_nationkey +order by r_name, s1; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinsWithCaseWhen2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinsWithCaseWhen2.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinsWithCaseWhen2.sql new file mode 100644 index 0000000..5e2918f --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinsWithCaseWhen2.sql @@ -0,0 +1,9 @@ +select + r_name, + case when s_name is null then 'N/O' + else s_name end as s1 +from region inner join ( + select * from nation + left outer join supplier on s_nationkey = n_nationkey +) t on n_regionkey = r_regionkey +order by r_name, s1; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/test/resources/queries/TestJoinQuery/testCrossJoinAndCaseWhen.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testCrossJoinAndCaseWhen.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testCrossJoinAndCaseWhen.sql deleted file mode 100644 index d058aba..0000000 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/testCrossJoinAndCaseWhen.sql +++ /dev/null @@ -1,18 +0,0 @@ -select - r_regionkey, - n_regionkey, - case - when r_regionkey = 1 then 'one' - when r_regionkey = 2 then 'two' - when r_regionkey = 3 then 'three' - when r_regionkey = 4 then 'four' - else 'zero' - end as cond -from - region, - nation -where - r_regionkey = n_regionkey -order by - r_regionkey, - n_regionkey \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/test/resources/queries/TestJoinQuery/testInnerJoinAndCaseWhen.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testInnerJoinAndCaseWhen.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testInnerJoinAndCaseWhen.sql new file mode 100644 index 0000000..d058aba --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testInnerJoinAndCaseWhen.sql @@ -0,0 +1,18 @@ +select + r_regionkey, + n_regionkey, + case + when r_regionkey = 1 then 'one' + when r_regionkey = 2 then 'two' + when r_regionkey = 3 then 'three' + when r_regionkey = 4 then 'four' + else 'zero' + end as cond +from + region, + nation +where + r_regionkey = n_regionkey +order by + r_regionkey, + n_regionkey \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/test/resources/queries/TestJoinQuery/testJoinWithOrPredicates.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testJoinWithOrPredicates.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testJoinWithOrPredicates.sql new file mode 100644 index 0000000..b388233 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testJoinWithOrPredicates.sql @@ -0,0 +1,6 @@ +select + n1.n_nationkey, + n1.n_name, + n2.n_name +from nation n1, nation n2 where n1.n_name = n2.n_name and (n1.n_nationkey in (1, 2) or n2.n_nationkey in (2)) +order by n1.n_nationkey; http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinsWithCaseWhen.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinsWithCaseWhen.result b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinsWithCaseWhen.result new file mode 100644 index 0000000..046a7c1 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinsWithCaseWhen.result @@ -0,0 +1,27 @@ +r_name,s1 +------------------------------- +AFRICA,N/O +AFRICA,N/O +AFRICA,N/O +AFRICA,Supplier#000000002 +AFRICA,Supplier#000000004 +AMERICA,N/O +AMERICA,N/O +AMERICA,N/O +AMERICA,N/O +AMERICA,Supplier#000000003 +ASIA,N/O +ASIA,N/O +ASIA,N/O +ASIA,N/O +ASIA,N/O +EUROPE,N/O +EUROPE,N/O +EUROPE,N/O +EUROPE,N/O +EUROPE,N/O +MIDDLE EAST,N/O +MIDDLE EAST,N/O +MIDDLE EAST,N/O +MIDDLE EAST,N/O +MIDDLE EAST,N/O \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinsWithCaseWhen2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinsWithCaseWhen2.result b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinsWithCaseWhen2.result new file mode 100644 index 0000000..046a7c1 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinsWithCaseWhen2.result @@ -0,0 +1,27 @@ +r_name,s1 +------------------------------- +AFRICA,N/O +AFRICA,N/O +AFRICA,N/O +AFRICA,Supplier#000000002 +AFRICA,Supplier#000000004 +AMERICA,N/O +AMERICA,N/O +AMERICA,N/O +AMERICA,N/O +AMERICA,Supplier#000000003 +ASIA,N/O +ASIA,N/O +ASIA,N/O +ASIA,N/O +ASIA,N/O +EUROPE,N/O +EUROPE,N/O +EUROPE,N/O +EUROPE,N/O +EUROPE,N/O +MIDDLE EAST,N/O +MIDDLE EAST,N/O +MIDDLE EAST,N/O +MIDDLE EAST,N/O +MIDDLE EAST,N/O \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/test/resources/results/TestJoinQuery/testCrossJoinAndCaseWhen.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testCrossJoinAndCaseWhen.result b/tajo-core/src/test/resources/results/TestJoinQuery/testCrossJoinAndCaseWhen.result deleted file mode 100644 index a4c33f8..0000000 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testCrossJoinAndCaseWhen.result +++ /dev/null @@ -1,27 +0,0 @@ -r_regionkey,n_regionkey,cond -------------------------------- -0,0,zero -0,0,zero -0,0,zero -0,0,zero -0,0,zero -1,1,one -1,1,one -1,1,one -1,1,one -1,1,one -2,2,two -2,2,two -2,2,two -2,2,two -2,2,two -3,3,three -3,3,three -3,3,three -3,3,three -3,3,three -4,4,four -4,4,four -4,4,four -4,4,four -4,4,four \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/test/resources/results/TestJoinQuery/testInnerJoinAndCaseWhen.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testInnerJoinAndCaseWhen.result b/tajo-core/src/test/resources/results/TestJoinQuery/testInnerJoinAndCaseWhen.result new file mode 100644 index 0000000..a4c33f8 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testInnerJoinAndCaseWhen.result @@ -0,0 +1,27 @@ +r_regionkey,n_regionkey,cond +------------------------------- +0,0,zero +0,0,zero +0,0,zero +0,0,zero +0,0,zero +1,1,one +1,1,one +1,1,one +1,1,one +1,1,one +2,2,two +2,2,two +2,2,two +2,2,two +2,2,two +3,3,three +3,3,three +3,3,three +3,3,three +3,3,three +4,4,four +4,4,four +4,4,four +4,4,four +4,4,four \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-core/src/test/resources/results/TestJoinQuery/testJoinWithOrPredicates.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testJoinWithOrPredicates.result b/tajo-core/src/test/resources/results/TestJoinQuery/testJoinWithOrPredicates.result new file mode 100644 index 0000000..ede3464 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testJoinWithOrPredicates.result @@ -0,0 +1,4 @@ +n_nationkey,n_name,n_name +------------------------------- +1,ARGENTINA,ARGENTINA +2,BRAZIL,BRAZIL \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index ff3d6c2..5e91b0c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -1122,7 +1122,10 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex joinCondition = context.evalOptimizer.optimize(context, evalNode); } - List<String> newlyEvaluatedExprs = getNewlyEvaluatedExprsForJoin(context, joinNode, stack); + // If the query involves a subquery, the stack can be empty. + // In this case, this join is the top most one within a query block. + boolean isTopMostJoin = stack.isEmpty() ? true : stack.peek().getType() != OpType.Join; + List<String> newlyEvaluatedExprs = getNewlyEvaluatedExprsForJoin(context, joinNode, isTopMostJoin); List<Target> targets = TUtil.newList(PlannerUtil.schemaToTargets(merged)); for (String newAddedExpr : newlyEvaluatedExprs) { @@ -1141,7 +1144,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex return joinNode; } - private List<String> getNewlyEvaluatedExprsForJoin(PlanContext context, JoinNode joinNode, Stack<Expr> stack) { + private List<String> getNewlyEvaluatedExprsForJoin(PlanContext context, JoinNode joinNode, boolean isTopMostJoin) { QueryBlock block = context.queryBlock; EvalNode evalNode; @@ -1150,7 +1153,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex NamedExpr namedExpr = it.next(); try { evalNode = exprAnnotator.createEvalNode(context, namedExpr.getExpr(), NameResolvingMode.LEGACY); - if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, evalNode, joinNode, stack.peek().getType() != OpType.Join)) { + // the predicates specified in the on clause are already processed in visitJoin() + if (LogicalPlanner.checkIfBeEvaluatedAtJoin(context.queryBlock, evalNode, joinNode, isTopMostJoin)) { block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode); newlyEvaluatedExprs.add(namedExpr.getAlias()); } @@ -1972,6 +1976,52 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex return true; } + public static boolean isEvaluatableJoinQual(QueryBlock block, EvalNode evalNode, JoinNode node, + boolean isOnPredicate, boolean isTopMostJoin) { + + if (checkIfBeEvaluatedAtJoin(block, evalNode, node, isTopMostJoin)) { + + if (isNonEquiThetaJoinQual(block, node, evalNode)) { + return false; + } + + if (PlannerUtil.isOuterJoin(node.getJoinType())) { + /* + * For outer joins, only predicates which are specified at the on clause can be evaluated during processing join. + * Other predicates from the where clause must be evaluated after the join. + * The below code will be modified after improving join operators to keep join filters by themselves (TAJO-1310). + */ + if (!isOnPredicate) { + return false; + } + } else { + /* + * Only join predicates should be evaluated at join if the join type is inner or cross. (TAJO-1445) + */ + if (!EvalTreeUtil.isJoinQual(block, node.getLeftChild().getOutSchema(), node.getRightChild().getOutSchema(), + evalNode, false)) { + return false; + } + } + + return true; + } + + return false; + } + + public static boolean isNonEquiThetaJoinQual(final LogicalPlan.QueryBlock block, + final JoinNode joinNode, + final EvalNode evalNode) { + if (EvalTreeUtil.isJoinQual(block, joinNode.getLeftChild().getOutSchema(), + joinNode.getRightChild().getOutSchema(), evalNode, true) && + evalNode.getType() != EvalType.EQUAL) { + return true; + } else { + return false; + } + } + public static boolean checkIfBeEvaluatedAtJoin(QueryBlock block, EvalNode evalNode, JoinNode node, boolean isTopMostJoin) { Set<Column> columnRefs = EvalTreeUtil.findUniqueColumns(evalNode); @@ -2004,10 +2054,6 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex return true; } - public static boolean isOuterJoin(JoinType joinType) { - return joinType == JoinType.LEFT_OUTER || joinType == JoinType.RIGHT_OUTER || joinType==JoinType.FULL_OUTER; - } - public static boolean containsOuterJoin(QueryBlock block) { return block.containsJoinType(JoinType.LEFT_OUTER) || block.containsJoinType(JoinType.RIGHT_OUTER) || block.containsJoinType(JoinType.FULL_OUTER); http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index 4cd008a..a6a7c78 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -18,7 +18,9 @@ package org.apache.tajo.plan.rewrite.rules; -import com.google.common.collect.*; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.OverridableConf; @@ -36,7 +38,11 @@ import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.util.TUtil; -import java.util.*; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Stack; /** * This rule tries to push down all filter conditions into logical nodes as lower as possible. @@ -48,7 +54,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo private static final String NAME = "FilterPushDown"; static class FilterPushDownContext { - Set<EvalNode> pushingDownFilters = new HashSet<EvalNode>(); + Set<EvalNode> pushingDownFilters = TUtil.newHashSet(); public void clear() { pushingDownFilters.clear(); @@ -63,7 +69,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo public void setToOrigin(Map<EvalNode, EvalNode> evalMap) { //evalMap: copy -> origin - List<EvalNode> origins = new ArrayList<EvalNode>(); + List<EvalNode> origins = TUtil.newList(); for (EvalNode eval : pushingDownFilters) { EvalNode origin = evalMap.get(eval); if (origin != null) { @@ -118,7 +124,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo @Override public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, SelectionNode selNode, Stack<LogicalNode> stack) throws PlanningException { - context.pushingDownFilters.addAll(Sets.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(selNode.getQual()))); + context.pushingDownFilters.addAll(TUtil.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(selNode.getQual()))); stack.push(selNode); visit(context, plan, block, selNode.getChild(), stack); @@ -158,159 +164,24 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode joinNode, Stack<LogicalNode> stack) throws PlanningException { - // here we should stop selection pushdown on the null supplying side(s) of an outer join - // get the two operands of the join operation as well as the join type - JoinType joinType = joinNode.getJoinType(); - EvalNode joinQual = joinNode.getJoinQual(); - if (joinQual != null && LogicalPlanner.isOuterJoin(joinType)) { - BinaryEval binaryEval = (BinaryEval) joinQual; - // if both are fields - if (binaryEval.getLeftExpr().getType() == EvalType.FIELD && - binaryEval.getRightExpr().getType() == EvalType.FIELD) { - - String leftTableName = ((FieldEval) binaryEval.getLeftExpr()).getQualifier(); - String rightTableName = ((FieldEval) binaryEval.getRightExpr()).getQualifier(); - List<String> nullSuppliers = Lists.newArrayList(); - Set<String> leftTableSet = Sets.newHashSet(PlannerUtil.getRelationLineageWithinQueryBlock(plan, - joinNode.getLeftChild())); - Set<String> rightTableSet = Sets.newHashSet(PlannerUtil.getRelationLineageWithinQueryBlock(plan, - joinNode.getRightChild())); - - // some verification - if (joinType == JoinType.FULL_OUTER) { - nullSuppliers.add(leftTableName); - nullSuppliers.add(rightTableName); - - // verify that these null suppliers are indeed in the left and right sets - if (!rightTableSet.contains(nullSuppliers.get(0)) && !leftTableSet.contains(nullSuppliers.get(0))) { - throw new InvalidQueryException("Incorrect Logical Query Plan with regard to outer join"); - } - if (!rightTableSet.contains(nullSuppliers.get(1)) && !leftTableSet.contains(nullSuppliers.get(1))) { - throw new InvalidQueryException("Incorrect Logical Query Plan with regard to outer join"); - } - - } else if (joinType == JoinType.LEFT_OUTER) { - nullSuppliers.add(((RelationNode)joinNode.getRightChild()).getCanonicalName()); - //verify that this null supplier is indeed in the right sub-tree - if (!rightTableSet.contains(nullSuppliers.get(0))) { - throw new InvalidQueryException("Incorrect Logical Query Plan with regard to outer join"); - } - } else if (joinType == JoinType.RIGHT_OUTER) { - if (((RelationNode)joinNode.getRightChild()).getCanonicalName().equals(rightTableName)) { - nullSuppliers.add(leftTableName); - } else { - nullSuppliers.add(rightTableName); - } - - // verify that this null supplier is indeed in the left sub-tree - if (!leftTableSet.contains(nullSuppliers.get(0))) { - throw new InvalidQueryException("Incorrect Logical Query Plan with regard to outer join"); - } - } - } - } - - /* non-equi filter should not be push down as a join qualifier until theta join is implemented - * TODO this code SHOULD be restored after TAJO-742 is resolved. */ - List<EvalNode> thetaJoinFilter = new ArrayList<EvalNode>(); - for (EvalNode eachEval: context.pushingDownFilters) { - if (eachEval.getType() != EvalType.EQUAL) { - if (EvalTreeUtil.isJoinQual(block, - joinNode.getLeftChild().getOutSchema(), - joinNode.getRightChild().getOutSchema(), - eachEval, true)) { - thetaJoinFilter.add(eachEval); - } - } - } - context.pushingDownFilters.removeAll(thetaJoinFilter); - - // get evals from ON clause - List<EvalNode> onConditions = new ArrayList<EvalNode>(); + Set<EvalNode> onPredicates = TUtil.newHashSet(); if (joinNode.hasJoinQual()) { - onConditions.addAll(Sets.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual()))); - } - - boolean isTopMostJoin = stack.peek().getType() != NodeType.JOIN; - - List<EvalNode> outerJoinPredicationEvals = new ArrayList<EvalNode>(); - List<EvalNode> outerJoinFilterEvalsExcludePredication = new ArrayList<EvalNode>(); - if (LogicalPlanner.isOuterJoin(joinNode.getJoinType())) { - // TAJO-853 - // In the case of top most JOIN, all filters except JOIN condition aren't pushed down. - // That filters are processed by SELECTION NODE. - Set<String> nullSupplyingTableNameSet; - if (joinNode.getJoinType() == JoinType.RIGHT_OUTER) { - nullSupplyingTableNameSet = TUtil.newHashSet(PlannerUtil.getRelationLineage(joinNode.getLeftChild())); - } else { - nullSupplyingTableNameSet = TUtil.newHashSet(PlannerUtil.getRelationLineage(joinNode.getRightChild())); - } - - Set<String> preservedTableNameSet; - if (joinNode.getJoinType() == JoinType.RIGHT_OUTER) { - preservedTableNameSet = TUtil.newHashSet(PlannerUtil.getRelationLineage(joinNode.getRightChild())); - } else { - preservedTableNameSet = TUtil.newHashSet(PlannerUtil.getRelationLineage(joinNode.getLeftChild())); - } - - List<EvalNode> removedFromFilter = new ArrayList<EvalNode>(); - for (EvalNode eachEval: context.pushingDownFilters) { - if (EvalTreeUtil.isJoinQual(block, - joinNode.getLeftChild().getOutSchema(), - joinNode.getRightChild().getOutSchema(), - eachEval, true)) { - outerJoinPredicationEvals.add(eachEval); - removedFromFilter.add(eachEval); - } else { - Set<Column> columns = EvalTreeUtil.findUniqueColumns(eachEval); - boolean canPushDown = true; - for (Column eachColumn: columns) { - if (nullSupplyingTableNameSet.contains(eachColumn.getQualifier())) { - canPushDown = false; - break; - } - } - if (!canPushDown) { - outerJoinFilterEvalsExcludePredication.add(eachEval); - removedFromFilter.add(eachEval); - } - } - } - - context.pushingDownFilters.removeAll(removedFromFilter); - - for (EvalNode eachOnEval: onConditions) { - if (EvalTreeUtil.isJoinQual(eachOnEval, true)) { - // If join condition, processing in the JoinNode. - outerJoinPredicationEvals.add(eachOnEval); - } else { - // If Eval has a column which belong to Preserved Row table, not using to push down but using JoinCondition - Set<Column> columns = EvalTreeUtil.findUniqueColumns(eachOnEval); - boolean canPushDown = true; - for (Column eachColumn: columns) { - if (preservedTableNameSet.contains(eachColumn.getQualifier())) { - canPushDown = false; - break; - } - } - if (canPushDown) { - context.pushingDownFilters.add(eachOnEval); - } else { - outerJoinPredicationEvals.add(eachOnEval); - } - } - } - } else { - context.pushingDownFilters.addAll(onConditions); + onPredicates.addAll(TUtil.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual()))); } + // we assume all the quals in pushingDownFilters as where predicates + Set<EvalNode> nonPushableQuals = extractNonPushableJoinQuals(plan, block, joinNode, onPredicates, + context.pushingDownFilters); + // add every predicate and remove non-pushable ones + context.pushingDownFilters.addAll(onPredicates); + context.pushingDownFilters.removeAll(nonPushableQuals); LogicalNode left = joinNode.getLeftChild(); LogicalNode right = joinNode.getRightChild(); - List<EvalNode> notMatched = new ArrayList<EvalNode>(); + List<EvalNode> notMatched = TUtil.newList(); // Join's input schema = right child output columns + left child output columns Map<EvalNode, EvalNode> transformedMap = findCanPushdownAndTransform(context, block, joinNode, left, notMatched, - null, true, 0); + null, 0); context.setFiltersTobePushed(transformedMap.keySet()); visit(context, plan, block, left, stack); @@ -318,9 +189,9 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo context.addFiltersTobePushed(notMatched); notMatched.clear(); - transformedMap = findCanPushdownAndTransform(context, block, joinNode, right, notMatched, null, true, + transformedMap = findCanPushdownAndTransform(context, block, joinNode, right, notMatched, null, left.getOutSchema().size()); - context.setFiltersTobePushed(new HashSet<EvalNode>(transformedMap.keySet())); + context.setFiltersTobePushed(transformedMap.keySet()); visit(context, plan, block, right, stack); @@ -328,14 +199,18 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo context.addFiltersTobePushed(notMatched); notMatched.clear(); - List<EvalNode> matched = Lists.newArrayList(); - if(LogicalPlanner.isOuterJoin(joinNode.getJoinType())) { - matched.addAll(outerJoinPredicationEvals); - } else { - for (EvalNode eval : context.pushingDownFilters) { - if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, eval, joinNode, isTopMostJoin)) { - matched.add(eval); - } + context.addFiltersTobePushed(nonPushableQuals); + List<EvalNode> matched = TUtil.newList(); + + // If the query involves a subquery, the stack can be empty. + // In this case, this join is the top most one within a query block. + boolean isTopMostJoin = stack.isEmpty() ? true : stack.peek().getType() != NodeType.JOIN; + + for (EvalNode evalNode : context.pushingDownFilters) { + // TODO: currently, non-equi theta join is not supported yet. + if (LogicalPlanner.isEvaluatableJoinQual(block, evalNode, joinNode, onPredicates.contains(evalNode), + isTopMostJoin)) { + matched.add(evalNode); } } @@ -355,20 +230,150 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo if (joinNode.getJoinType() == JoinType.CROSS) { joinNode.setJoinType(JoinType.INNER); } - context.pushingDownFilters.removeAll(matched); } - context.pushingDownFilters.addAll(outerJoinFilterEvalsExcludePredication); - context.pushingDownFilters.addAll(thetaJoinFilter); + context.pushingDownFilters.removeAll(matched); return joinNode; } + private static Set<EvalNode> extractNonPushableJoinQuals(final LogicalPlan plan, + final LogicalPlan.QueryBlock block, + final JoinNode joinNode, + final Set<EvalNode> onPredicates, + final Set<EvalNode> wherePredicates) + throws PlanningException { + Set<EvalNode> nonPushableQuals = TUtil.newHashSet(); + // TODO: non-equi theta join quals must not be pushed until TAJO-742 is resolved. + nonPushableQuals.addAll(extractNonEquiThetaJoinQuals(wherePredicates, block, joinNode)); + + // for outer joins + if (PlannerUtil.isOuterJoin(joinNode.getJoinType())) { + nonPushableQuals.addAll(extractNonPushableOuterJoinQuals(plan, onPredicates, wherePredicates, joinNode)); + } + return nonPushableQuals; + } + + /** + * For outer joins, pushable predicates can be decided based on their locations in the SQL and types of referencing + * relations. + * + * <h3>Table types</h3> + * <ul> + * <li>Preserved Row table : The preserved row table refers to the table that preserves rows when there is no match + * in the join operation. Therefore, all rows from the preserved row table that qualify against the WHERE clause + * will be returned, regardless of whether there is a matched row in the join. For a left/right table, the preserved + * row table is the left/right table. For a full outer join, both tables are preserved row tables.</li> + * <li>Null Supplying table : The NULL-supplying table supplies NULLs when there is an unmatched row. Any column + * from the NULL-supplying table referred to in the SELECT list or subsequent WHERE or ON clause will contain NULL + * if there was no match in the join operation. For a left/right outer join, the NULL-supplying + * table is the right/left table. For a full outer join, both tables are NULL-supplying + * table. In a full outer join, both tables can preserve rows, and also can supply NULLs. This is significant, + * because there are rules that apply to purely preserved row tables that do not apply if the table can also supply + * NULLs.</li> + * </ul> + * + * <h3>Predicate types</h3> + * <ul> + * <li>During Join predicate : A predicate that is in the JOIN ON clause.</li> + * <li>After Join predicate : A predicate that is in the WHERE clause.</li> + * </ul> + * + * <h3>Predicate Pushdown Rules</h3> + * <ol> + * <li>During Join predicates cannot be pushed past Preserved Row tables.</li> + * <li>After Join predicates cannot be pushed past Null Supplying tables.</li> + * </ol> + */ + private static Set<EvalNode> extractNonPushableOuterJoinQuals(final LogicalPlan plan, + final Set<EvalNode> onPredicates, + final Set<EvalNode> wherePredicates, + final JoinNode joinNode) throws PlanningException { + Set<String> nullSupplyingTableNameSet = TUtil.newHashSet(); + Set<String> preservedTableNameSet = TUtil.newHashSet(); + String leftRelation = PlannerUtil.getTopRelationInLineage(plan, joinNode.getLeftChild()); + String rightRelation = PlannerUtil.getTopRelationInLineage(plan, joinNode.getRightChild()); + + if (joinNode.getJoinType() == JoinType.LEFT_OUTER) { + nullSupplyingTableNameSet.add(rightRelation); + preservedTableNameSet.add(leftRelation); + } else if (joinNode.getJoinType() == JoinType.RIGHT_OUTER) { + nullSupplyingTableNameSet.add(leftRelation); + preservedTableNameSet.add(rightRelation); + } else { + // full outer join + preservedTableNameSet.add(leftRelation); + preservedTableNameSet.add(rightRelation); + nullSupplyingTableNameSet.add(leftRelation); + nullSupplyingTableNameSet.add(rightRelation); + } + + Set<EvalNode> nonPushableQuals = TUtil.newHashSet(); + for (EvalNode eachQual : onPredicates) { + for (String relName : preservedTableNameSet) { + if (isEvalNeedRelation(eachQual, relName)) { + nonPushableQuals.add(eachQual); + } + } + } + + for (EvalNode eachQual : wherePredicates) { + for (String relName : nullSupplyingTableNameSet) { + if (isEvalNeedRelation(eachQual, relName)) { + nonPushableQuals.add(eachQual); + } + } + } + + return nonPushableQuals; + } + + private static boolean isEvalNeedRelation(final EvalNode evalNode, final String relationName) { + Set<Column> columns = EvalTreeUtil.findUniqueColumns(evalNode); + for (Column column : columns) { + if (isColumnFromRelation(column, relationName)) { + return true; + } + } + return false; + } + + private static boolean isColumnFromRelation(final Column column, final String relationName) { + if (relationName.equals(column.getQualifier())) { + return true; + } + return false; + } + + private static boolean isNonEquiThetaJoinQual(final LogicalPlan.QueryBlock block, + final JoinNode joinNode, + final EvalNode evalNode) { + if (EvalTreeUtil.isJoinQual(block, joinNode.getLeftChild().getOutSchema(), + joinNode.getRightChild().getOutSchema(), evalNode, true) && + evalNode.getType() != EvalType.EQUAL) { + return true; + } else { + return false; + } + } + + private static List<EvalNode> extractNonEquiThetaJoinQuals(final Set<EvalNode> predicates, + final LogicalPlan.QueryBlock block, + final JoinNode joinNode) { + List<EvalNode> nonEquiThetaJoinQuals = TUtil.newList(); + for (EvalNode eachEval: predicates) { + if (isNonEquiThetaJoinQual(block, joinNode, eachEval)) { + nonEquiThetaJoinQuals.add(eachEval); + } + } + return nonEquiThetaJoinQuals; + } + private Map<EvalNode, EvalNode> transformEvalsWidthByPassNode( Collection<EvalNode> originEvals, LogicalPlan plan, LogicalPlan.QueryBlock block, LogicalNode node, LogicalNode childNode) throws PlanningException { // transformed -> pushingDownFilters - Map<EvalNode, EvalNode> transformedMap = new HashMap<EvalNode, EvalNode>(); + Map<EvalNode, EvalNode> transformedMap = TUtil.newHashMap(); if (originEvals.isEmpty()) { return transformedMap; @@ -426,7 +431,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo } // node in column -> child out column - Map<String, String> columnMap = new HashMap<String, String>(); + Map<String, String> columnMap = TUtil.newHashMap(); for (int i = 0; i < node.getInSchema().size(); i++) { String inColumnName = node.getInSchema().getColumn(i).getQualifiedName(); @@ -474,7 +479,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo @Override public LogicalNode visitTableSubQuery(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, TableSubQueryNode node, Stack<LogicalNode> stack) throws PlanningException { - List<EvalNode> matched = Lists.newArrayList(); + List<EvalNode> matched = TUtil.newList(); for (EvalNode eval : context.pushingDownFilters) { if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, node)) { matched.add(eval); @@ -484,8 +489,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo // transformed -> pushingDownFilters Map<EvalNode, EvalNode> transformedMap = transformEvalsWidthByPassNode(matched, plan, block, node, node.getSubQuery()); - - context.setFiltersTobePushed(new HashSet<EvalNode>(transformedMap.keySet())); + context.setFiltersTobePushed(transformedMap.keySet()); visit(context, plan, plan.getBlock(node.getSubQuery())); context.setToOrigin(transformedMap); @@ -498,11 +502,11 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo Stack<LogicalNode> stack) throws PlanningException { LogicalNode leftNode = unionNode.getLeftChild(); - List<EvalNode> origins = new ArrayList<EvalNode>(context.pushingDownFilters); + List<EvalNode> origins = TUtil.newList(context.pushingDownFilters); // transformed -> pushingDownFilters Map<EvalNode, EvalNode> transformedMap = transformEvalsWidthByPassNode(origins, plan, block, unionNode, leftNode); - context.setFiltersTobePushed(new HashSet<EvalNode>(transformedMap.keySet())); + context.setFiltersTobePushed(transformedMap.keySet()); visit(context, plan, plan.getBlock(leftNode)); if (!context.pushingDownFilters.isEmpty()) { @@ -511,7 +515,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo LogicalNode rightNode = unionNode.getRightChild(); transformedMap = transformEvalsWidthByPassNode(origins, plan, block, unionNode, rightNode); - context.setFiltersTobePushed(new HashSet<EvalNode>(transformedMap.keySet())); + context.setFiltersTobePushed(transformedMap.keySet()); visit(context, plan, plan.getBlock(rightNode), rightNode, stack); if (!context.pushingDownFilters.isEmpty()) { @@ -531,11 +535,11 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo Stack<LogicalNode> stack) throws PlanningException { LogicalNode childNode = projectionNode.getChild(); - List<EvalNode> notMatched = new ArrayList<EvalNode>(); + List<EvalNode> notMatched = TUtil.newList(); //copy -> origin BiMap<EvalNode, EvalNode> transformedMap = findCanPushdownAndTransform( - context, block,projectionNode, childNode, notMatched, null, false, 0); + context, block,projectionNode, childNode, notMatched, null, 0); context.setFiltersTobePushed(transformedMap.keySet()); @@ -583,7 +587,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo } private Collection<EvalNode> reverseTransform(BiMap<EvalNode, EvalNode> map, Set<EvalNode> remainFilters) { - Set<EvalNode> reversed = Sets.newHashSet(); + Set<EvalNode> reversed = TUtil.newHashSet(); for (EvalNode evalNode : remainFilters) { reversed.add(map.get(evalNode)); } @@ -593,10 +597,9 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo private BiMap<EvalNode, EvalNode> findCanPushdownAndTransform( FilterPushDownContext context, LogicalPlan.QueryBlock block, Projectable node, LogicalNode childNode, List<EvalNode> notMatched, - Set<String> partitionColumns, - boolean ignoreJoin, int columnOffset) throws PlanningException { + Set<String> partitionColumns, int columnOffset) throws PlanningException { // canonical name -> target - Map<String, Target> nodeTargetMap = new HashMap<String, Target>(); + Map<String, Target> nodeTargetMap = TUtil.newHashMap(); for (Target target : node.getTargets()) { nodeTargetMap.put(target.getCanonicalName(), target); } @@ -605,10 +608,6 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo BiMap<EvalNode, EvalNode> matched = HashBiMap.create(); for (EvalNode eval : context.pushingDownFilters) { - if (ignoreJoin && EvalTreeUtil.isJoinQual(block, null, null, eval, true)) { - notMatched.add(eval); - continue; - } // If all column is field eval, can push down. Set<Column> evalColumns = EvalTreeUtil.findUniqueColumns(eval); boolean columnMatched = true; @@ -725,16 +724,16 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo HavingNode havingNode, GroupbyNode groupByNode) throws PlanningException { // find aggregation column - Set<Column> groupingColumns = new HashSet<Column>(Arrays.asList(groupByNode.getGroupingColumns())); - Set<String> aggrFunctionOutColumns = new HashSet<String>(); + Set<Column> groupingColumns = TUtil.newHashSet(groupByNode.getGroupingColumns()); + Set<String> aggrFunctionOutColumns = TUtil.newHashSet(); for (Column column : groupByNode.getOutSchema().getColumns()) { if (!groupingColumns.contains(column)) { aggrFunctionOutColumns.add(column.getQualifiedName()); } } - List<EvalNode> aggrEvalOrigins = new ArrayList<EvalNode>(); - List<EvalNode> aggrEvals = new ArrayList<EvalNode>(); + List<EvalNode> aggrEvalOrigins = TUtil.newList(); + List<EvalNode> aggrEvals = TUtil.newList(); for (EvalNode eval : context.pushingDownFilters) { EvalNode copy = null; @@ -818,10 +817,10 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo context.pushingDownFilters.removeAll(aggrEvals); } - List<EvalNode> notMatched = new ArrayList<EvalNode>(); + List<EvalNode> notMatched = TUtil.newList(); // transform Map<EvalNode, EvalNode> transformed = - findCanPushdownAndTransform(context, block, groupbyNode,groupbyNode.getChild(), notMatched, null, false, 0); + findCanPushdownAndTransform(context, block, groupbyNode,groupbyNode.getChild(), notMatched, null, 0); context.setFiltersTobePushed(transformed.keySet()); LogicalNode current = super.visitGroupBy(context, plan, block, groupbyNode, stack); @@ -836,10 +835,10 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scanNode, Stack<LogicalNode> stack) throws PlanningException { - List<EvalNode> matched = Lists.newArrayList(); + List<EvalNode> matched = TUtil.newList(); // find partition column and check matching - Set<String> partitionColumns = new HashSet<String>(); + Set<String> partitionColumns = TUtil.newHashSet(); TableDesc table = scanNode.getTableDesc(); boolean hasQualifiedName = false; if (table.hasPartition()) { @@ -848,7 +847,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo hasQualifiedName = c.hasQualifier(); } } - Set<EvalNode> partitionEvals = new HashSet<EvalNode>(); + Set<EvalNode> partitionEvals = TUtil.newHashSet(); for (EvalNode eval : context.pushingDownFilters) { if (table.hasPartition()) { Set<Column> columns = EvalTreeUtil.findUniqueColumns(eval); @@ -882,11 +881,11 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo context.pushingDownFilters.removeAll(partitionEvals); - List<EvalNode> notMatched = new ArrayList<EvalNode>(); + List<EvalNode> notMatched = TUtil.newList(); // transform Map<EvalNode, EvalNode> transformed = - findCanPushdownAndTransform(context, block, scanNode, null, notMatched, partitionColumns, true, 0); + findCanPushdownAndTransform(context, block, scanNode, null, notMatched, partitionColumns, 0); for (EvalNode eval : transformed.keySet()) { if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, scanNode)) { http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java index abd2814..9a6e625 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java @@ -850,7 +850,8 @@ public class ProjectionPushDownRule extends // So, we should prevent dividing the binary operator into more subexpressions. if (term.getType() != EvalType.FIELD && !(term instanceof BinaryEval) && - !(term.getType() == EvalType.ROW_CONSTANT)) { + term.getType() != EvalType.ROW_CONSTANT && + term.getType() != EvalType.CONST) { String refName = ctx.addExpr(term); EvalTreeUtil.replace(cnf, term, new FieldEval(refName, term.getValueType())); } @@ -914,6 +915,8 @@ public class ProjectionPushDownRule extends if (context.targetListMgr.isEvaluated(referenceName)) { Target fieldReference = new Target(new FieldEval(target.getNamedColumn())); + // here, we assume that every exprs are specified at the on clause + // because all filters have been moved to appropriate logical nodes during the filter push down phase if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, fieldReference.getEvalTree(), node, stack.peek().getType() != NodeType.JOIN)) { projectedTargets.add(fieldReference); http://git-wip-us.apache.org/repos/asf/tajo/blob/b1e174ee/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index b09fc9e..763f938 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -218,6 +218,16 @@ public class PlannerUtil { return tableNames; } + public static String getTopRelationInLineage(LogicalPlan plan, LogicalNode from) throws PlanningException { + RelationFinderVisitor visitor = new RelationFinderVisitor(true); + visitor.visit(null, plan, null, from, new Stack<LogicalNode>()); + if (visitor.getFoundRelations().isEmpty()) { + return null; + } else { + return visitor.getFoundRelations().iterator().next(); + } + } + /** * Get all RelationNodes which are descendant of a given LogicalNode. * The finding is restricted within a query block. @@ -227,13 +237,18 @@ public class PlannerUtil { */ public static Collection<String> getRelationLineageWithinQueryBlock(LogicalPlan plan, LogicalNode from) throws PlanningException { - RelationFinderVisitor visitor = new RelationFinderVisitor(); + RelationFinderVisitor visitor = new RelationFinderVisitor(false); visitor.visit(null, plan, null, from, new Stack<LogicalNode>()); return visitor.getFoundRelations(); } public static class RelationFinderVisitor extends BasicLogicalPlanVisitor<Object, LogicalNode> { private Set<String> foundRelNameSet = Sets.newHashSet(); + private boolean topOnly = false; + + public RelationFinderVisitor(boolean topOnly) { + this.topOnly = topOnly; + } public Set<String> getFoundRelations() { return foundRelNameSet; @@ -242,6 +257,10 @@ public class PlannerUtil { @Override public LogicalNode visit(Object context, LogicalPlan plan, @Nullable LogicalPlan.QueryBlock block, LogicalNode node, Stack<LogicalNode> stack) throws PlanningException { + if (topOnly && foundRelNameSet.size() > 0) { + return node; + } + if (node.getType() != NodeType.TABLE_SUBQUERY) { super.visit(context, plan, block, node, stack); } @@ -759,6 +778,10 @@ public class PlannerUtil { return joinType == JoinType.INNER; } + public static boolean isOuterJoin(JoinType joinType) { + return joinType == JoinType.LEFT_OUTER || joinType == JoinType.RIGHT_OUTER || joinType==JoinType.FULL_OUTER; + } + public static boolean existsAggregationFunction(Expr expr) throws PlanningException { AggregationFunctionFinder finder = new AggregationFunctionFinder(); AggFunctionFoundResult result = new AggFunctionFoundResult();
