Repository: tajo Updated Branches: refs/heads/master f87f66729 -> caf8cca94
TAJO-1732: During filter push down phase, join conditions are not set properly. Closes #670 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/caf8cca9 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/caf8cca9 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/caf8cca9 Branch: refs/heads/master Commit: caf8cca94699a687f28e7fcf2c8f6efb67773983 Parents: f87f667 Author: Jihoon Son <[email protected]> Authored: Tue Aug 11 13:49:43 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Tue Aug 11 13:49:43 2015 +0900 ---------------------------------------------------------------------- CHANGES | 4 +- .../org/apache/tajo/benchmark/TestTPCH.java | 7 + .../queries/TestTPCH/testFirstJoinInQ7.sql | 13 ++ .../results/TestTPCH/testFirstJoinInQ7.plan | 156 +++++++++++++++++++ .../results/TestTPCH/testFirstJoinInQ7.result | 4 + .../org/apache/tajo/plan/logical/JoinNode.java | 4 + .../plan/rewrite/rules/FilterPushDownRule.java | 3 + 7 files changed, 190 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 2b873b4..18de625 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,5 @@ Tajo Change Log - Release 0.11.0 - unreleased NEW FEATURES @@ -212,6 +211,9 @@ Release 0.11.0 - unreleased BUG FIXES + TAJO-1732: During filter push down phase, join conditions are not set properly. + (jihoon) + TAJO-1742: Remove hadoop dependency in DatumFactory. (jinho) TAJO-1733: Finished query occasionally does not appear in Web-UI. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-core/src/test/java/org/apache/tajo/benchmark/TestTPCH.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/benchmark/TestTPCH.java b/tajo-core/src/test/java/org/apache/tajo/benchmark/TestTPCH.java index 9b7db2d..6390f90 100644 --- a/tajo-core/src/test/java/org/apache/tajo/benchmark/TestTPCH.java +++ b/tajo-core/src/test/java/org/apache/tajo/benchmark/TestTPCH.java @@ -60,4 +60,11 @@ public class TestTPCH extends QueryTestCaseBase { public void testTPCHQ5() throws Exception { runSimpleTests(); } + + @Test + @Option(withExplain = true, withExplainGlobal = true) + @SimpleTest + public void testFirstJoinInQ7() throws Exception { + runSimpleTests(); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-core/src/test/resources/queries/TestTPCH/testFirstJoinInQ7.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestTPCH/testFirstJoinInQ7.sql b/tajo-core/src/test/resources/queries/TestTPCH/testFirstJoinInQ7.sql new file mode 100644 index 0000000..2dfb83d --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestTPCH/testFirstJoinInQ7.sql @@ -0,0 +1,13 @@ +select + n1.n_name as supp_nation, n2.n_name as cust_nation, n1.n_nationkey as s_nationkey, n2.n_nationkey as c_nationkey +from + nation n1 join nation n2 + on + n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY' +UNION ALL +select + n1.n_name as supp_nation, n2.n_name as cust_nation, n1.n_nationkey as s_nationkey, n2.n_nationkey as c_nationkey +from + nation n1 join nation n2 + on + n2.n_name = 'FRANCE' and n1.n_name = 'GERMANY' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan b/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan new file mode 100644 index 0000000..d17f26c --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan @@ -0,0 +1,156 @@ +explain +------------------------------- +UNION(8) + PROJECTION(7) + => Targets: supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4) + => out schema: {(4) supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4)} + => in schema: {(4) supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4)} + JOIN(11)(CROSS) + => target list: c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT) + => out schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)} + => in schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)} + SCAN(4) on default.nation as n1 + => filter: default.n1.n_name (TEXT) = GERMANY + => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey + => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} + => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)} + SCAN(5) on default.nation as n2 + => filter: default.n2.n_name (TEXT) = FRANCE + => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey + => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} + => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)} + PROJECTION(3) + => Targets: supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4) + => out schema: {(4) supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4)} + => in schema: {(4) supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4)} + JOIN(10)(CROSS) + => target list: c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT) + => out schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)} + => in schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)} + SCAN(0) on default.nation as n1 + => filter: default.n1.n_name (TEXT) = FRANCE + => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey + => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} + => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)} + SCAN(1) on default.nation as n2 + => filter: default.n2.n_name (TEXT) = GERMANY + => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey + => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} + => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)} +explain +------------------------------- +------------------------------------------------------------------------------- +Execution Block Graph (TERMINAL - eb_0000000000000_0000_000007) +------------------------------------------------------------------------------- +|-eb_0000000000000_0000_000007 + |-eb_0000000000000_0000_000006 + |-eb_0000000000000_0000_000005 + |-eb_0000000000000_0000_000004 + |-eb_0000000000000_0000_000003 + |-eb_0000000000000_0000_000002 + |-eb_0000000000000_0000_000001 +------------------------------------------------------------------------------- +Order of Execution +------------------------------------------------------------------------------- +1: eb_0000000000000_0000_000001 +2: eb_0000000000000_0000_000002 +3: eb_0000000000000_0000_000004 +4: eb_0000000000000_0000_000005 +5: eb_0000000000000_0000_000003 +6: eb_0000000000000_0000_000006 +7: eb_0000000000000_0000_000007 +------------------------------------------------------------------------------- + +======================================================= +Block Id: eb_0000000000000_0000_000001 [LEAF] +======================================================= + +[Outgoing] +[q_0000000000000_0000] 1 => 3 (type=HASH_SHUFFLE) + +SCAN(1) on default.nation as n2 + => filter: default.n2.n_name (TEXT) = GERMANY + => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey + => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} + => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000002 [LEAF] +======================================================= + +[Outgoing] +[q_0000000000000_0000] 2 => 3 (type=HASH_SHUFFLE) + +SCAN(0) on default.nation as n1 + => filter: default.n1.n_name (TEXT) = FRANCE + => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey + => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} + => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000004 [LEAF] +======================================================= + +[Outgoing] +[q_0000000000000_0000] 4 => 6 (type=HASH_SHUFFLE) + +SCAN(5) on default.nation as n2 + => filter: default.n2.n_name (TEXT) = FRANCE + => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey + => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} + => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000005 [LEAF] +======================================================= + +[Outgoing] +[q_0000000000000_0000] 5 => 6 (type=HASH_SHUFFLE) + +SCAN(4) on default.nation as n1 + => filter: default.n1.n_name (TEXT) = GERMANY + => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey + => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} + => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000003 [ROOT] +======================================================= + +[Incoming] +[q_0000000000000_0000] 1 => 3 (type=HASH_SHUFFLE) +[q_0000000000000_0000] 2 => 3 (type=HASH_SHUFFLE) + +JOIN(10)(CROSS) + => target list: c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT) + => out schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)} + => in schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)} + SCAN(14) on eb_0000000000000_0000_000002 + => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} + => in schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} + SCAN(13) on eb_0000000000000_0000_000001 + => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} + => in schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} + +======================================================= +Block Id: eb_0000000000000_0000_000006 [ROOT] +======================================================= + +[Incoming] +[q_0000000000000_0000] 4 => 6 (type=HASH_SHUFFLE) +[q_0000000000000_0000] 5 => 6 (type=HASH_SHUFFLE) + +JOIN(11)(CROSS) + => target list: c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT) + => out schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)} + => in schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)} + SCAN(16) on eb_0000000000000_0000_000005 + => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} + => in schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} + SCAN(15) on eb_0000000000000_0000_000004 + => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} + => in schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} + +======================================================= +Block Id: eb_0000000000000_0000_000007 [TERMINAL] +======================================================= http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.result b/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.result new file mode 100644 index 0000000..9f5b85a --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.result @@ -0,0 +1,4 @@ +supp_nation,cust_nation,s_nationkey,c_nationkey +------------------------------- +FRANCE,GERMANY,6,7 +GERMANY,FRANCE,7,6 http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java index 4f36026..8b843a9 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java @@ -61,6 +61,10 @@ public class JoinNode extends BinaryNode implements Projectable, Cloneable { this.joinSpec.setSingletonPredicate(joinQual); } + public void clearJoinQual() { + this.joinSpec.setPredicates(null); + } + public boolean hasJoinQual() { return this.joinSpec.hasPredicates(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index 4fb8aac..30ef7a8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -177,6 +177,9 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo if (joinNode.hasJoinQual()) { onPredicates.addAll(TUtil.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual()))); } + // clear join qual + joinNode.clearJoinQual(); + // we assume all the quals in pushingDownFilters as where predicates Set<EvalNode> nonPushableQuals = extractNonPushableJoinQuals(plan, block, joinNode, onPredicates, context.pushingDownFilters);
