TAJO-680: Improve the IN operator to support sub queries. Closes #620
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/042c3e88 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/042c3e88 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/042c3e88 Branch: refs/heads/master Commit: 042c3e882fbb45fffc6fc2988588282ed085614c Parents: f0ab0ca Author: Jihoon Son <[email protected]> Authored: Fri Aug 14 12:33:02 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri Aug 14 12:33:46 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/tajo/algebra/CommonSubquery.java | 60 ++++++ .../apache/tajo/algebra/ExistsPredicate.java | 6 +- .../java/org/apache/tajo/algebra/OpType.java | 2 +- .../java/org/apache/tajo/algebra/Relation.java | 8 +- .../org/apache/tajo/algebra/RelationList.java | 3 +- .../tajo/algebra/SimpleTableSubQuery.java | 16 +- .../tajo/algebra/TablePrimarySubQuery.java | 54 +----- .../apache/tajo/engine/parser/SQLAnalyzer.java | 7 +- .../org/apache/tajo/master/GlobalEngine.java | 8 +- .../java/org/apache/tajo/QueryTestCaseBase.java | 2 +- .../apache/tajo/engine/eval/ExprTestBase.java | 4 +- .../tajo/engine/planner/TestLogicalPlan.java | 2 +- .../tajo/engine/query/TestInSubquery.java | 177 +++++++++++++++++ .../TestInSubquery/testInAndNotInSubQuery.sql | 3 + .../queries/TestInSubquery/testInSubQuery.sql | 1 + .../queries/TestInSubquery/testInSubQuery2.sql | 2 + .../TestInSubquery/testInSubQueryWithJoin.sql | 2 + .../testInSubQueryWithOtherConditions.sql | 2 + .../testInSubQueryWithTableSubQuery.sql | 2 + .../TestInSubquery/testMultipleInSubQuery.sql | 5 + .../testMultipleNotInSubQuery.sql | 3 + .../testNestedInAndNotInSubQuery.sql | 5 + .../TestInSubquery/testNestedInSubQuery.sql | 4 + .../TestInSubquery/testNestedInSubQuery2.sql | 4 + .../TestInSubquery/testNestedNotInSubQuery.sql | 4 + .../TestInSubquery/testNotInSubQuery.sql | 1 + .../testSameKeyNameOfOuterAndInnerQueries.sql | 22 +++ .../TestInSubquery/testWithAsteriskAndJoin.sql | 7 + .../testInAndNotInSubQuery.result | 24 +++ .../TestInSubquery/testInSubQuery.result | 27 +++ .../TestInSubquery/testInSubQuery2.result | 3 + .../testInSubQueryWithJoin.result | 5 + .../testInSubQueryWithOtherConditions.result | 25 +++ .../testInSubQueryWithTableSubQuery.result | 4 + .../testMultipleInSubQuery.result | 5 + .../testMultipleNotInSubQuery.result | 20 ++ .../testNestedInAndNotInSubQuery.result | 3 + .../TestInSubquery/testNestedInSubQuery.result | 3 + .../TestInSubquery/testNestedInSubQuery2.result | 3 + .../testNestedNotInSubQuery.result | 7 + .../TestInSubquery/testNotInSubQuery.result | 22 +++ ...testSameKeyNameOfOuterAndInnerQueries.result | 3 + .../testWithAsteriskAndJoin.result | 6 + .../org/apache/tajo/plan/ExprAnnotator.java | 17 +- .../org/apache/tajo/plan/LogicalOptimizer.java | 101 ++++++---- .../java/org/apache/tajo/plan/LogicalPlan.java | 15 +- .../tajo/plan/LogicalPlanPreprocessor.java | 38 +++- .../org/apache/tajo/plan/LogicalPlanner.java | 45 ++++- .../tajo/plan/algebra/AlgebraVisitor.java | 2 +- .../tajo/plan/algebra/BaseAlgebraVisitor.java | 11 +- .../tajo/plan/expr/BasicEvalNodeVisitor.java | 9 + .../apache/tajo/plan/expr/EvalNodeVisitor2.java | 2 + .../org/apache/tajo/plan/expr/EvalType.java | 4 +- .../java/org/apache/tajo/plan/expr/InEval.java | 9 +- .../apache/tajo/plan/expr/RowConstantEval.java | 34 +--- .../tajo/plan/expr/SimpleEvalNodeVisitor.java | 8 + .../org/apache/tajo/plan/expr/SubqueryEval.java | 99 ++++++++++ .../org/apache/tajo/plan/expr/ValueSetEval.java | 54 ++++++ .../GreedyHeuristicJoinOrderAlgorithm.java | 7 +- .../apache/tajo/plan/joinorder/JoinGraph.java | 12 +- .../tajo/plan/joinorder/JoinOrderingUtil.java | 18 +- .../apache/tajo/plan/logical/RelationNode.java | 10 + .../tajo/plan/nameresolver/NameResolver.java | 39 ++-- .../BaseLogicalPlanRewriteRuleProvider.java | 2 + .../rewrite/rules/InSubqueryRewriteRule.java | 189 +++++++++++++++++++ .../rewrite/rules/ProjectionPushDownRule.java | 2 +- .../tajo/plan/serder/EvalNodeDeserializer.java | 9 +- .../tajo/plan/serder/EvalNodeSerializer.java | 15 ++ .../plan/serder/LogicalNodeDeserializer.java | 2 + .../tajo/plan/serder/LogicalNodeSerializer.java | 2 + .../org/apache/tajo/plan/util/ExprFinder.java | 14 +- .../org/apache/tajo/plan/util/PlannerUtil.java | 17 ++ .../tajo/plan/verifier/LogicalPlanVerifier.java | 11 +- .../tajo/plan/visitor/SimpleAlgebraVisitor.java | 6 + tajo-plan/src/main/proto/Plan.proto | 9 + 76 files changed, 1168 insertions(+), 222 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 66d9bee..0b705e8 100644 --- a/CHANGES +++ b/CHANGES @@ -32,6 +32,8 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-680: Improve the IN operator to support sub queries. (jihoon) + TAJO-1751: Reduce the client connection timeout. (jinho) TAJO-1746: Improve resource usage at first request of DefaultTaskScheduler. http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-algebra/src/main/java/org/apache/tajo/algebra/CommonSubquery.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CommonSubquery.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CommonSubquery.java new file mode 100644 index 0000000..ec567f0 --- /dev/null +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CommonSubquery.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.algebra; + +import com.google.common.base.Objects; +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; + +public abstract class CommonSubquery extends Relation { + @Expose + @SerializedName("SubPlan") + protected Expr subquery; + + protected CommonSubquery(OpType type, String relationName, Expr subquery) { + super(type, relationName); + this.subquery = subquery; + } + + public Expr getSubQuery() { + return subquery; + } + + public int hashCode() { + return Objects.hashCode(subquery); + } + + @Override + boolean equalsTo(Expr expr) { + CommonSubquery another = (CommonSubquery) expr; + return subquery.equals(another.subquery); + } + + public String toJson() { + return JsonHelper.toJson(this); + } + + @Override + public Object clone() throws CloneNotSupportedException { + CommonSubquery subQuery = (CommonSubquery) super.clone(); + subQuery.subquery = (Expr) subquery.clone(); + return subQuery; + } + +} http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-algebra/src/main/java/org/apache/tajo/algebra/ExistsPredicate.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ExistsPredicate.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ExistsPredicate.java index fa8b3d4..5ee997d 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ExistsPredicate.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ExistsPredicate.java @@ -26,7 +26,7 @@ public class ExistsPredicate extends UnaryOperator { @Expose @SerializedName("IsNot") private boolean not; - public ExistsPredicate(SimpleTableSubQuery simpleTableSubQuery, boolean not) { + public ExistsPredicate(SimpleTableSubquery simpleTableSubQuery, boolean not) { super(OpType.ExistsPredicate); this.not = not; setChild(simpleTableSubQuery); @@ -36,8 +36,8 @@ public class ExistsPredicate extends UnaryOperator { return this.not; } - public SimpleTableSubQuery getSubQuery() { - return (SimpleTableSubQuery) getChild(); + public SimpleTableSubquery getSubQuery() { + return (SimpleTableSubquery) getChild(); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java index 47fea64..f3efde5 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java @@ -38,8 +38,8 @@ public enum OpType { Union(SetOperation.class), Except(SetOperation.class), Intersect(SetOperation.class), - SimpleTableSubQuery(SimpleTableSubQuery.class), TablePrimaryTableSubQuery(TablePrimarySubQuery.class), + SimpleTableSubquery(SimpleTableSubquery.class), RelationList(RelationList.class), Relation(Relation.class), ScalarSubQuery(ScalarSubQuery.class), http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-algebra/src/main/java/org/apache/tajo/algebra/Relation.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Relation.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Relation.java index 2092b67..6769f8a 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Relation.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Relation.java @@ -25,9 +25,9 @@ import org.apache.tajo.util.TUtil; public class Relation extends Expr { @Expose @SerializedName("TableName") - private String tableName; + protected String tableName; @Expose @SerializedName("TableAlias") - private String alias; + protected String alias; protected Relation(OpType type, String relationName) { super(type); @@ -46,6 +46,10 @@ public class Relation extends Expr { return tableName; } + public void setTableName(String tableName) { + this.tableName = tableName; + } + public boolean hasAlias() { return alias != null; } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-algebra/src/main/java/org/apache/tajo/algebra/RelationList.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/RelationList.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/RelationList.java index ad7315b..1ae8ead 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/RelationList.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/RelationList.java @@ -37,7 +37,8 @@ public class RelationList extends Expr { Preconditions.checkArgument( rel.getType() == OpType.Relation || rel.getType() == OpType.Join || - rel.getType() == OpType.TablePrimaryTableSubQuery, + rel.getType() == OpType.TablePrimaryTableSubQuery || + rel.getType() == OpType.SimpleTableSubquery, "Only Relation, Join, or TablePrimarySubQuery can be given to RelationList, but this expr " + " is " + rel.getType()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-algebra/src/main/java/org/apache/tajo/algebra/SimpleTableSubQuery.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/SimpleTableSubQuery.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/SimpleTableSubQuery.java index 2332be1..fd2f777 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/SimpleTableSubQuery.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/SimpleTableSubQuery.java @@ -18,19 +18,11 @@ package org.apache.tajo.algebra; -public class SimpleTableSubQuery extends UnaryOperator { +public class SimpleTableSubquery extends CommonSubquery { - public SimpleTableSubQuery(Expr subquery) { - super(OpType.SimpleTableSubQuery); - setChild(subquery); - } - - public Expr getSubQuery() { - return getChild(); - } + public final static String TEMP_RELATION_NAME = "TempSubqueryName"; - @Override - boolean equalsTo(Expr expr) { - return true; + public SimpleTableSubquery(Expr subquery) { + super(OpType.SimpleTableSubquery, TEMP_RELATION_NAME, subquery); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-algebra/src/main/java/org/apache/tajo/algebra/TablePrimarySubQuery.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/TablePrimarySubQuery.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/TablePrimarySubQuery.java index 6f08b0d..22f49ca 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/TablePrimarySubQuery.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/TablePrimarySubQuery.java @@ -18,59 +18,9 @@ package org.apache.tajo.algebra; -import com.google.common.base.Objects; -import com.google.gson.annotations.Expose; -import com.google.gson.annotations.SerializedName; -import org.apache.tajo.util.TUtil; - -public class TablePrimarySubQuery extends Relation { - @Expose @SerializedName("SubPlan") - private Expr subquery; - @Expose @SerializedName("ColumnNames") - private String [] columnNames; +public class TablePrimarySubQuery extends CommonSubquery { public TablePrimarySubQuery(String relName, Expr subquery) { - super(OpType.TablePrimaryTableSubQuery, relName); - this.subquery = subquery; - } - - public boolean hasColumnNames() { - return this.columnNames != null; - } - - public void setColumnNames(String[] aliasList) { - this.columnNames = aliasList; - } - - public String [] getColumnNames() { - return columnNames; - } - - public Expr getSubQuery() { - return subquery; - } - - public int hashCode() { - return Objects.hashCode(subquery, Objects.hashCode(columnNames)); - } - - @Override - boolean equalsTo(Expr expr) { - TablePrimarySubQuery another = (TablePrimarySubQuery) expr; - return subquery.equals(another.subquery) && TUtil.checkEquals(columnNames, another.columnNames); - } - - public String toJson() { - return JsonHelper.toJson(this); - } - - @Override - public Object clone() throws CloneNotSupportedException { - TablePrimarySubQuery subQuery = (TablePrimarySubQuery) super.clone(); - subQuery.subquery = (Expr) subquery.clone(); - if (columnNames != null) { - subQuery.columnNames = columnNames.clone(); - } - return subQuery; + super(OpType.TablePrimaryTableSubQuery, relName, subquery); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index d89a404..6d00dde 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -46,9 +46,6 @@ import static org.apache.tajo.engine.parser.SQLParser.*; public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { - public SQLAnalyzer() { - } - public Expr parse(String sql) { ANTLRInputStream input = new ANTLRInputStream(sql); SQLLexer lexer = new SQLLexer(input); @@ -987,7 +984,7 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { } return new ValueListExpr(exprs); } else { - return new SimpleTableSubQuery(visitChildren(ctx.table_subquery())); + return new SimpleTableSubquery(visitChildren(ctx.table_subquery())); } } @@ -1047,7 +1044,7 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { @Override public ExistsPredicate visitExists_predicate(SQLParser.Exists_predicateContext ctx) { - return new ExistsPredicate(new SimpleTableSubQuery(visitTable_subquery(ctx.table_subquery())), ctx.NOT() != null); + return new ExistsPredicate(new SimpleTableSubquery(visitTable_subquery(ctx.table_subquery())), ctx.NOT() != null); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 20780ec..f1f1e3e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -100,7 +100,7 @@ public class GlobalEngine extends AbstractService { planner = new LogicalPlanner(context.getCatalog(), TablespaceManager.getInstance()); // Access path rewriter is enabled only in QueryMasterTask optimizer = new LogicalOptimizer(context.getConf(), context.getCatalog()); - annotatedPlanVerifier = new LogicalPlanVerifier(context.getConf(), context.getCatalog()); + annotatedPlanVerifier = new LogicalPlanVerifier(); } catch (Throwable t) { LOG.error(t.getMessage(), t); throw new RuntimeException(t); @@ -283,8 +283,8 @@ public class GlobalEngine extends AbstractService { LOG.info("Optimized Query: \n" + plan.toString()); LOG.info("============================================="); - annotatedPlanVerifier.verify(queryContext, state, plan); - verifyInsertTableSchema(queryContext, state, plan); + annotatedPlanVerifier.verify(state, plan); + verifyInsertTableSchema(state, plan); if (!state.verified()) { for (Throwable error : state.getErrors()) { @@ -295,7 +295,7 @@ public class GlobalEngine extends AbstractService { return plan; } - private void verifyInsertTableSchema(QueryContext queryContext, VerificationState state, LogicalPlan plan) { + private void verifyInsertTableSchema(VerificationState state, LogicalPlan plan) { String storeType = PlannerUtil.getStoreType(plan); if (storeType != null) { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index bcce612..89e53d5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -295,7 +295,7 @@ public class QueryTestCaseBase { } LogicalPlan plan = planner.createPlan(context, expr); optimizer.optimize(plan); - postVerifier.verify(context, state, plan); + postVerifier.verify(state, plan); return state; } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 6fe1510..f2b6477 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -104,7 +104,7 @@ public class ExprTestBase { preLogicalPlanVerifier = new PreLogicalPlanVerifier(cat); planner = new LogicalPlanner(cat, TablespaceManager.getInstance()); optimizer = new LogicalOptimizer(util.getConfiguration(), cat); - annotatedPlanVerifier = new LogicalPlanVerifier(util.getConfiguration(), cat); + annotatedPlanVerifier = new LogicalPlanVerifier(); } @AfterClass @@ -149,7 +149,7 @@ public class ExprTestBase { } LogicalPlan plan = planner.createPlan(context, expr, true); optimizer.optimize(context, plan); - annotatedPlanVerifier.verify(context, state, plan); + annotatedPlanVerifier.verify(state, plan); if (state.getErrors().size() > 0) { assertFalse(state.getErrors().get(0).getMessage(), true); http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java index dc9e2b0..d49c43e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java @@ -49,7 +49,7 @@ public class TestLogicalPlan { @Test public final void testQueryBlockGraph() { - LogicalPlan plan = new LogicalPlan(planner); + LogicalPlan plan = new LogicalPlan(); LogicalPlan.QueryBlock root = plan.newAndGetBlock(LogicalPlan.ROOT_BLOCK); LogicalPlan.QueryBlock new1 = plan.newQueryBlock(); LogicalPlan.QueryBlock new2 = plan.newQueryBlock(); http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInSubquery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInSubquery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInSubquery.java new file mode 100644 index 0000000..fe465f1 --- /dev/null +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInSubquery.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.query; + +import org.apache.tajo.IntegrationTest; +import org.apache.tajo.NamedTest; +import org.apache.tajo.error.Errors.ResultCode; +import org.apache.tajo.exception.TajoRuntimeException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.sql.SQLException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@Category(IntegrationTest.class) +@RunWith(Parameterized.class) +@NamedTest("TestJoinQuery") +public class TestInSubquery extends TestJoinQuery { + + public TestInSubquery(String joinOption) throws Exception { + super(joinOption); + } + + @BeforeClass + public static void setup() throws Exception { + TestJoinQuery.setup(); + } + + @AfterClass + public static void classTearDown() throws SQLException { + TestJoinQuery.classTearDown(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testInSubQuery() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testInSubQuery2() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testNestedInSubQuery() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testInSubQueryWithOtherConditions() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testMultipleInSubQuery() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testInSubQueryWithJoin() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testInSubQueryWithTableSubQuery() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testNotInSubQuery() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testMultipleNotInSubQuery() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testNestedNotInSubQuery() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testInAndNotInSubQuery() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testNestedInAndNotInSubQuery() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testNestedInSubQuery2() throws Exception { + // select c_name from customer + // where c_nationkey in ( + // select n_nationkey from nation where n_name like 'C%' and n_regionkey in ( + // select count(*)-1 from region where r_regionkey > 0 and r_regionkey < 3)) + runSimpleTests(); + } + + @Test() + public final void testCorrelatedSubQuery() throws Exception { + // Use try-catch clause to verify the exact error message + try { + executeString("select * from nation where n_regionkey in (select r_regionkey from region where n_name > r_name)"); + fail("Correlated subquery must raise the UnimplementedException."); + } catch (TajoRuntimeException e) { + assertEquals(ResultCode.NOT_IMPLEMENTED, e.getErrorCode()); + } + } + + @Test + @Option(withExplain = false, withExplainGlobal = false, parameterized = true, sort = true) + @SimpleTest() + public final void testSameKeyNameOfOuterAndInnerQueries() throws Exception { + runSimpleTests(); + } + + @Test + @Option(parameterized = true, sort = true) + @SimpleTest() + public final void testWithAsteriskAndJoin() throws Exception { + // select * from lineitem, orders where l_orderkey = o_orderkey and l_partkey in + // (select l_partkey from lineitem where l_linenumber in (1, 3, 5, 7, 9)) + runSimpleTests(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testInAndNotInSubQuery.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testInAndNotInSubQuery.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testInAndNotInSubQuery.sql new file mode 100644 index 0000000..8ef5077 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testInAndNotInSubQuery.sql @@ -0,0 +1,3 @@ +select n_name from nation +where n_regionkey in (select r_regionkey from region) + and n_nationkey not in (select s_nationkey from supplier) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQuery.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQuery.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQuery.sql new file mode 100644 index 0000000..48928a1 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQuery.sql @@ -0,0 +1 @@ +select n_name from nation where n_regionkey in (select r_regionkey from region) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQuery2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQuery2.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQuery2.sql new file mode 100644 index 0000000..eff1e7b --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQuery2.sql @@ -0,0 +1,2 @@ +select n_name from nation +where n_nationkey in (select count(*) from region) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithJoin.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithJoin.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithJoin.sql new file mode 100644 index 0000000..178b601 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithJoin.sql @@ -0,0 +1,2 @@ +select n_name from nation, supplier +where n_regionkey in (select r_regionkey from region) and n_nationkey = s_nationkey \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithOtherConditions.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithOtherConditions.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithOtherConditions.sql new file mode 100644 index 0000000..47a23fa --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithOtherConditions.sql @@ -0,0 +1,2 @@ +select n_name from nation where n_regionkey in ( + select r_regionkey from region) and n_nationkey > 1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithTableSubQuery.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithTableSubQuery.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithTableSubQuery.sql new file mode 100644 index 0000000..8645df1 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testInSubQueryWithTableSubQuery.sql @@ -0,0 +1,2 @@ +select n_name from (select * from nation where n_nationkey > 1 and n_nationkey < 10) as T +where n_regionkey in (select r_regionkey from region where r_regionkey > 1 and r_regionkey < 3); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testMultipleInSubQuery.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testMultipleInSubQuery.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testMultipleInSubQuery.sql new file mode 100644 index 0000000..b93041f --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testMultipleInSubQuery.sql @@ -0,0 +1,5 @@ +select n_name from nation +where + n_regionkey in (select r_regionkey from region) + and + n_nationkey in (select s_nationkey from supplier) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testMultipleNotInSubQuery.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testMultipleNotInSubQuery.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testMultipleNotInSubQuery.sql new file mode 100644 index 0000000..bd85ded --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testMultipleNotInSubQuery.sql @@ -0,0 +1,3 @@ +select n_name from nation +where n_nationkey not in (select r_regionkey from region) + and n_nationkey not in (select s_nationkey from supplier) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInAndNotInSubQuery.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInAndNotInSubQuery.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInAndNotInSubQuery.sql new file mode 100644 index 0000000..8e4c6ba --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInAndNotInSubQuery.sql @@ -0,0 +1,5 @@ +select c_name from customer + where c_nationkey in ( + select n_nationkey from nation where n_name like 'C%' and n_regionkey + not in ( + select count(*) from region where r_regionkey > 0 and r_regionkey < 3)) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInSubQuery.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInSubQuery.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInSubQuery.sql new file mode 100644 index 0000000..902dfd7 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInSubQuery.sql @@ -0,0 +1,4 @@ +select c_name from customer +where c_nationkey in ( + select n_nationkey from nation where n_name like 'C%' and n_regionkey in ( + select r_regionkey from region where r_regionkey > 0 and r_regionkey < 3)) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInSubQuery2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInSubQuery2.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInSubQuery2.sql new file mode 100644 index 0000000..4965ec2 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testNestedInSubQuery2.sql @@ -0,0 +1,4 @@ +select c_name from customer + where c_nationkey in ( + select n_nationkey from nation where n_name like 'C%' and n_regionkey in ( + select count(*) - 1 from region where r_regionkey > 0 and r_regionkey < 3)) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testNestedNotInSubQuery.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testNestedNotInSubQuery.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testNestedNotInSubQuery.sql new file mode 100644 index 0000000..a2cea67 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testNestedNotInSubQuery.sql @@ -0,0 +1,4 @@ +select c_name from customer +where c_nationkey not in ( + select n_nationkey from nation where n_name like 'C%' and n_regionkey not in ( + select r_regionkey from region where r_regionkey > 0 and r_regionkey < 3)) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testNotInSubQuery.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testNotInSubQuery.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testNotInSubQuery.sql new file mode 100644 index 0000000..cb05624 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testNotInSubQuery.sql @@ -0,0 +1 @@ +select n_name from nation where n_nationkey not in (select r_regionkey from region) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testSameKeyNameOfOuterAndInnerQueries.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testSameKeyNameOfOuterAndInnerQueries.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testSameKeyNameOfOuterAndInnerQueries.sql new file mode 100644 index 0000000..c8d3bf4 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testSameKeyNameOfOuterAndInnerQueries.sql @@ -0,0 +1,22 @@ +select + n_regionkey, count(*) +from + customer, lineitem, orders, supplier, nation +where + l_orderkey = o_orderkey and + c_custkey = o_custkey and + l_linenumber = s_suppkey and + l_partkey in ( + select + l_partkey + from + lineitem + where + l_linenumber in (1, 3, 5, 7, 9) + ) and + n_nationkey = c_nationkey +group by + n_regionkey +order by + n_regionkey +limit 100; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/queries/TestInSubquery/testWithAsteriskAndJoin.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInSubquery/testWithAsteriskAndJoin.sql b/tajo-core/src/test/resources/queries/TestInSubquery/testWithAsteriskAndJoin.sql new file mode 100644 index 0000000..d5e2bfa --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInSubquery/testWithAsteriskAndJoin.sql @@ -0,0 +1,7 @@ +select + * +from + lineitem, orders +where + l_orderkey = o_orderkey and + l_partkey in (select l_partkey from lineitem where l_linenumber in (1, 3, 5, 7, 9)) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testInAndNotInSubQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testInAndNotInSubQuery.result b/tajo-core/src/test/resources/results/TestInSubquery/testInAndNotInSubQuery.result new file mode 100644 index 0000000..1acab75 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testInAndNotInSubQuery.result @@ -0,0 +1,24 @@ +n_name +------------------------------- +ALGERIA +BRAZIL +CANADA +CHINA +EGYPT +FRANCE +GERMANY +INDIA +INDONESIA +IRAN +IRAQ +JAPAN +JORDAN +KENYA +MOZAMBIQUE +PERU +ROMANIA +RUSSIA +SAUDI ARABIA +UNITED KINGDOM +UNITED STATES +VIETNAM http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testInSubQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testInSubQuery.result b/tajo-core/src/test/resources/results/TestInSubquery/testInSubQuery.result new file mode 100644 index 0000000..7644296 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testInSubQuery.result @@ -0,0 +1,27 @@ +n_name +------------------------------- +ALGERIA +ARGENTINA +BRAZIL +CANADA +CHINA +EGYPT +ETHIOPIA +FRANCE +GERMANY +INDIA +INDONESIA +IRAN +IRAQ +JAPAN +JORDAN +KENYA +MOROCCO +MOZAMBIQUE +PERU +ROMANIA +RUSSIA +SAUDI ARABIA +UNITED KINGDOM +UNITED STATES +VIETNAM http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testInSubQuery2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testInSubQuery2.result b/tajo-core/src/test/resources/results/TestInSubquery/testInSubQuery2.result new file mode 100644 index 0000000..fbab93d --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testInSubQuery2.result @@ -0,0 +1,3 @@ +n_name +------------------------------- +ETHIOPIA http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithJoin.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithJoin.result b/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithJoin.result new file mode 100644 index 0000000..1deee15 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithJoin.result @@ -0,0 +1,5 @@ +n_name +------------------------------- +ARGENTINA +ETHIOPIA +MOROCCO http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithOtherConditions.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithOtherConditions.result b/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithOtherConditions.result new file mode 100644 index 0000000..8367869 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithOtherConditions.result @@ -0,0 +1,25 @@ +n_name +------------------------------- +BRAZIL +CANADA +CHINA +EGYPT +ETHIOPIA +FRANCE +GERMANY +INDIA +INDONESIA +IRAN +IRAQ +JAPAN +JORDAN +KENYA +MOROCCO +MOZAMBIQUE +PERU +ROMANIA +RUSSIA +SAUDI ARABIA +UNITED KINGDOM +UNITED STATES +VIETNAM http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithTableSubQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithTableSubQuery.result b/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithTableSubQuery.result new file mode 100644 index 0000000..d386836 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testInSubQueryWithTableSubQuery.result @@ -0,0 +1,4 @@ +n_name +------------------------------- +INDIA +INDONESIA http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testMultipleInSubQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testMultipleInSubQuery.result b/tajo-core/src/test/resources/results/TestInSubquery/testMultipleInSubQuery.result new file mode 100644 index 0000000..1deee15 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testMultipleInSubQuery.result @@ -0,0 +1,5 @@ +n_name +------------------------------- +ARGENTINA +ETHIOPIA +MOROCCO http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testMultipleNotInSubQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testMultipleNotInSubQuery.result b/tajo-core/src/test/resources/results/TestInSubquery/testMultipleNotInSubQuery.result new file mode 100644 index 0000000..a1e17d5 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testMultipleNotInSubQuery.result @@ -0,0 +1,20 @@ +n_name +------------------------------- +CHINA +FRANCE +GERMANY +INDIA +INDONESIA +IRAN +IRAQ +JAPAN +JORDAN +KENYA +MOZAMBIQUE +PERU +ROMANIA +RUSSIA +SAUDI ARABIA +UNITED KINGDOM +UNITED STATES +VIETNAM http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testNestedInAndNotInSubQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testNestedInAndNotInSubQuery.result b/tajo-core/src/test/resources/results/TestInSubquery/testNestedInAndNotInSubQuery.result new file mode 100644 index 0000000..51e570a --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testNestedInAndNotInSubQuery.result @@ -0,0 +1,3 @@ +c_name +------------------------------- +Customer#000000005 http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testNestedInSubQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testNestedInSubQuery.result b/tajo-core/src/test/resources/results/TestInSubquery/testNestedInSubQuery.result new file mode 100644 index 0000000..51e570a --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testNestedInSubQuery.result @@ -0,0 +1,3 @@ +c_name +------------------------------- +Customer#000000005 http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testNestedInSubQuery2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testNestedInSubQuery2.result b/tajo-core/src/test/resources/results/TestInSubquery/testNestedInSubQuery2.result new file mode 100644 index 0000000..51e570a --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testNestedInSubQuery2.result @@ -0,0 +1,3 @@ +c_name +------------------------------- +Customer#000000005 http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testNestedNotInSubQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testNestedNotInSubQuery.result b/tajo-core/src/test/resources/results/TestInSubquery/testNestedNotInSubQuery.result new file mode 100644 index 0000000..e746b35 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testNestedNotInSubQuery.result @@ -0,0 +1,7 @@ +c_name +------------------------------- +Customer#000000001 +Customer#000000002 +Customer#000000003 +Customer#000000004 +Customer#000000005 http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testNotInSubQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testNotInSubQuery.result b/tajo-core/src/test/resources/results/TestInSubquery/testNotInSubQuery.result new file mode 100644 index 0000000..50b69bd --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testNotInSubQuery.result @@ -0,0 +1,22 @@ +n_name +------------------------------- +CHINA +ETHIOPIA +FRANCE +GERMANY +INDIA +INDONESIA +IRAN +IRAQ +JAPAN +JORDAN +KENYA +MOROCCO +MOZAMBIQUE +PERU +ROMANIA +RUSSIA +SAUDI ARABIA +UNITED KINGDOM +UNITED STATES +VIETNAM http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testSameKeyNameOfOuterAndInnerQueries.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testSameKeyNameOfOuterAndInnerQueries.result b/tajo-core/src/test/resources/results/TestInSubquery/testSameKeyNameOfOuterAndInnerQueries.result new file mode 100644 index 0000000..e3d9398 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testSameKeyNameOfOuterAndInnerQueries.result @@ -0,0 +1,3 @@ +n_regionkey,?count +------------------------------- +1,1 http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-core/src/test/resources/results/TestInSubquery/testWithAsteriskAndJoin.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInSubquery/testWithAsteriskAndJoin.result b/tajo-core/src/test/resources/results/TestInSubquery/testWithAsteriskAndJoin.result new file mode 100644 index 0000000..734ce36 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInSubquery/testWithAsteriskAndJoin.result @@ -0,0 +1,6 @@ +l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment,o_orderkey,o_custkey,o_orderstatus,o_totalprice,o_orderdate,o_orderpriority,o_clerk,o_shippriority,o_comment +------------------------------- +1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold ,1,3,O,173665.47,1996-01-02,5-LOW,Clerk#000000951,0,nstructions sleep furiously among +1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the,1,3,O,173665.47,1996-01-02,5-LOW,Clerk#000000951,0,nstructions sleep furiously among +2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a,2,4,O,46929.18,1996-12-01,1-URGENT,Clerk#000000880,0, foxes. pending accounts at the pending, silent asymptot +3,2,1798,1,45.0,54058.05,0.06,0.0,R,F,1994-02-02,1994-01-04,1994-02-23,NONE,AIR,ongside of the furiously brave acco,3,2,F,193846.25,1993-10-14,5-LOW,Clerk#000000955,0,sly final accounts boost. carefully regular ideas cajole carefully. depos http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java index a44b526..b062e8e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java @@ -37,6 +37,7 @@ import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.TableSubQueryNode; import org.apache.tajo.plan.nameresolver.NameResolver; import org.apache.tajo.plan.nameresolver.NameResolvingMode; import org.apache.tajo.util.Pair; @@ -363,12 +364,12 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva public EvalNode visitInPredicate(Context ctx, Stack<Expr> stack, InPredicate expr) throws TajoException { stack.push(expr); EvalNode lhs = visit(ctx, stack, expr.getLeft()); - RowConstantEval rowConstantEval = (RowConstantEval) visit(ctx, stack, expr.getInValue()); + ValueSetEval valueSetEval = (ValueSetEval) visit(ctx, stack, expr.getInValue()); stack.pop(); - Pair<EvalNode, EvalNode> pair = convertTypesIfNecessary(ctx, lhs, rowConstantEval); + Pair<EvalNode, EvalNode> pair = convertTypesIfNecessary(ctx, lhs, valueSetEval); - return new InEval(pair.getFirst(), (RowConstantEval) pair.getSecond(), expr.isNot()); + return new InEval(pair.getFirst(), (ValueSetEval) pair.getSecond(), expr.isNot()); } @Override @@ -386,6 +387,16 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva } @Override + public EvalNode visitSimpleTableSubquery(Context ctx, Stack<Expr> stack, SimpleTableSubquery expr) + throws TajoException { + if (stack.peek().getType() == OpType.InPredicate) { + // In the case of in-subquery, stop visiting because the subquery expr is not expression. + return new SubqueryEval((TableSubQueryNode) ctx.currentBlock.getNodeFromExpr(expr)); + } else { + return super.visitSimpleTableSubquery(ctx, stack, expr); + } + } + public EvalNode visitExistsPredicate(Context ctx, Stack<Expr> stack, ExistsPredicate expr) throws TajoException { throw new NotImplementedException("EXISTS clause"); } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index b1d8ce5..96617d1 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -19,6 +19,7 @@ package org.apache.tajo.plan; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceStability; @@ -29,6 +30,7 @@ import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.plan.expr.*; import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.expr.AlgebraicUtil; import org.apache.tajo.plan.expr.EvalNode; @@ -274,51 +276,74 @@ public class LogicalOptimizer { throws TajoException { super.visitJoin(context, plan, block, joinNode, stack); - // given a join node, find the relations which are nearest to the join in the query. - RelationNode leftChild = findMostRightRelation(plan, block, joinNode.getLeftChild()); - RelationNode rightChild = findMostLeftRelation(plan, block, joinNode.getRightChild()); - RelationVertex leftVertex = new RelationVertex(leftChild); - RelationVertex rightVertex = new RelationVertex(rightChild); + if (joinNode.getJoinType() == JoinType.LEFT_SEMI || joinNode.getJoinType() == JoinType.LEFT_ANTI) { + // In case of in-subquery, the left vertex must be the relation of the left column of the in qual. + // In addition, the join qual can be evaluated only at the join node for in-subquery, + // we don't need to consider moving it to other joins. + + BinaryEval joinQual = (BinaryEval) joinNode.getJoinQual(); + Preconditions.checkArgument(joinQual.getLeftExpr().getType() == EvalType.FIELD || + joinQual.getLeftExpr().getType() == EvalType.CAST); + FieldEval leftColumn = null; + if (joinQual.getLeftExpr().getType() == EvalType.FIELD) { + leftColumn = joinQual.getLeftExpr(); + } else if (joinQual.getLeftExpr().getType() == EvalType.CAST) { + leftColumn = (FieldEval) ((CastEval)joinQual.getLeftExpr()).getOperand(); + } + RelationNode leftChild = block.getRelation(leftColumn.getQualifier()); + RelationNode rightChild = joinNode.getRightChild(); + RelationVertex leftVertex = new RelationVertex(leftChild); + RelationVertex rightVertex = new RelationVertex(rightChild); - JoinEdge edge = context.getJoinGraph().addJoin(context, joinNode.getJoinSpec(), leftVertex, rightVertex); + context.getJoinGraph().addJoin(context, joinNode.getJoinSpec(), leftVertex, rightVertex); + } else { - // find all possible predicates for this join edge - Set<EvalNode> joinConditions = TUtil.newHashSet(); - if (joinNode.hasJoinQual()) { - Set<EvalNode> originPredicates = joinNode.getJoinSpec().getPredicates(); - for (EvalNode predicate : joinNode.getJoinSpec().getPredicates()) { - if (EvalTreeUtil.isJoinQual(block, leftVertex.getSchema(), rightVertex.getSchema(), predicate, false)) { - if (JoinOrderingUtil.checkIfEvaluatedAtEdge(predicate, edge, true)) { + // given a join node, find the relations which are nearest to the join in the query. + RelationNode leftChild = findMostRightRelation(plan, block, joinNode.getLeftChild()); + RelationNode rightChild = findMostLeftRelation(plan, block, joinNode.getRightChild()); + RelationVertex leftVertex = new RelationVertex(leftChild); + RelationVertex rightVertex = new RelationVertex(rightChild); + + JoinEdge edge = context.getJoinGraph().addJoin(context, joinNode.getJoinSpec(), leftVertex, rightVertex); + + // find all possible predicates for this join edge + Set<EvalNode> joinConditions = TUtil.newHashSet(); + if (joinNode.hasJoinQual()) { + Set<EvalNode> originPredicates = joinNode.getJoinSpec().getPredicates(); + for (EvalNode predicate : joinNode.getJoinSpec().getPredicates()) { + if (EvalTreeUtil.isJoinQual(block, leftVertex.getSchema(), rightVertex.getSchema(), predicate, false)) { + if (JoinOrderingUtil.checkIfEvaluatedAtEdge(predicate, edge, true)) { + joinConditions.add(predicate); + } + } else { joinConditions.add(predicate); } - } else { - joinConditions.add(predicate); } + // find predicates which cannot be evaluated at this join + originPredicates.removeAll(joinConditions); + context.addCandidateJoinConditions(originPredicates); + originPredicates.clear(); + originPredicates.addAll(joinConditions); } - // find predicates which cannot be evaluated at this join - originPredicates.removeAll(joinConditions); - context.addCandidateJoinConditions(originPredicates); - originPredicates.clear(); - originPredicates.addAll(joinConditions); - } - joinConditions.addAll(JoinOrderingUtil.findJoinConditionForJoinVertex(context.getCandidateJoinConditions(), edge, - true)); - joinConditions.addAll(JoinOrderingUtil.findJoinConditionForJoinVertex(context.getCandidateJoinFilters(), edge, - false)); - context.markAsEvaluatedJoinConditions(joinConditions); - context.markAsEvaluatedJoinFilters(joinConditions); - edge.addJoinPredicates(joinConditions); - if (edge.getJoinType() == JoinType.INNER && edge.getJoinQual().isEmpty()) { - edge.getJoinSpec().setType(JoinType.CROSS); - } - - if (PlannerUtil.isCommutativeJoinType(edge.getJoinType())) { - JoinEdge commutativeEdge = context.getCachedOrNewJoinEdge(edge.getJoinSpec(), edge.getRightVertex(), - edge.getLeftVertex()); - commutativeEdge.addJoinPredicates(joinConditions); - context.getJoinGraph().addEdge(commutativeEdge.getLeftVertex(), commutativeEdge.getRightVertex(), - commutativeEdge); + joinConditions.addAll(JoinOrderingUtil.findJoinConditionForJoinVertex(context.getCandidateJoinConditions(), edge, + true)); + joinConditions.addAll(JoinOrderingUtil.findJoinConditionForJoinVertex(context.getCandidateJoinFilters(), edge, + false)); + context.markAsEvaluatedJoinConditions(joinConditions); + context.markAsEvaluatedJoinFilters(joinConditions); + edge.addJoinPredicates(joinConditions); + if (edge.getJoinType() == JoinType.INNER && edge.getJoinQual().isEmpty()) { + edge.getJoinSpec().setType(JoinType.CROSS); + } + + if (PlannerUtil.isCommutativeJoinType(edge.getJoinType())) { + JoinEdge commutativeEdge = context.getCachedOrNewJoinEdge(edge.getJoinSpec(), edge.getRightVertex(), + edge.getLeftVertex()); + commutativeEdge.addJoinPredicates(joinConditions); + context.getJoinGraph().addEdge(commutativeEdge.getLeftVertex(), commutativeEdge.getRightVertex(), + commutativeEdge); + } } return joinNode; http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index eab939d..b7df810 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@ -57,12 +57,11 @@ public class LogicalPlan { /** it indicates the root block */ public static final String ROOT_BLOCK = VIRTUAL_TABLE_PREFIX + "ROOT"; public static final String NONAME_BLOCK_PREFIX = VIRTUAL_TABLE_PREFIX + "QB_"; - public static final String NONAME_SUBQUERY_PREFIX = "?SubQuery_"; + public static final String NONAME_SUBQUERY_PREFIX = VIRTUAL_TABLE_PREFIX + "SQ_"; private static final int NO_SEQUENCE_PID = -1; private int nextPid = 0; private Integer noNameBlockId = 0; private Integer noNameColumnId = 0; - private Integer noNameSubqueryId = 0; /** a map from between a block name to a block plan */ private Map<String, QueryBlock> queryBlocks = new LinkedHashMap<String, QueryBlock>(); @@ -74,16 +73,13 @@ public class LogicalPlan { /** planning and optimization log */ private List<String> planingHistory = Lists.newArrayList(); - private static enum ExplainType { + private enum ExplainType { NOT_EXPLAIN, EXPLAIN_LOGICAL, EXPLAIN_GLOBAL } private ExplainType explainType = ExplainType.NOT_EXPLAIN; - public LogicalPlan(LogicalPlanner planner) { - } - /** * Create a LogicalNode instance for a type. Each a LogicalNode instance is given an unique plan node id (PID). * @@ -161,13 +157,6 @@ public class LogicalPlan { } /** - * It generates a unique table subquery name - */ - public String generateUniqueSubQueryName() { - return NONAME_SUBQUERY_PREFIX + noNameSubqueryId++; - } - - /** * It generates an unique column name from Expr. It is usually used for an expression or predicate without * a specified name (i.e., alias). * Here, some expressions require to be identified with their names in the future. http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java index 76907f2..f2c15ac 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java @@ -33,7 +33,6 @@ import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.nameresolver.NameResolver; import org.apache.tajo.plan.nameresolver.NameResolvingMode; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; import org.apache.tajo.util.TUtil; @@ -124,8 +123,10 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P while (iterator.hasNext()) { relationOp = iterator.next(); - schema = relationOp.getLogicalSchema(); - resolvedColumns.addAll(schema.getRootColumns()); + if (relationOp.isNameResolveBase()) { + schema = relationOp.getLogicalSchema(); + resolvedColumns.addAll(schema.getRootColumns()); + } } if (resolvedColumns.size() == 0) { @@ -346,6 +347,13 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P public LogicalNode visitFilter(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Selection expr) throws TajoException { stack.push(expr); + // Since filter push down will be done later, it is guaranteed that in-subqueries are found at only selection. + for (Expr eachQual : PlannerUtil.extractInSubquery(expr.getQual())) { + InPredicate inPredicate = (InPredicate) eachQual; + stack.push(inPredicate); + visit(ctx, stack, inPredicate.getRight()); + stack.pop(); + } LogicalNode child = visit(ctx, stack, expr.getChild()); stack.pop(); @@ -415,6 +423,30 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P TableSubQueryNode node = ctx.plan.createNode(TableSubQueryNode.class); node.init(CatalogUtil.buildFQName(ctx.queryContext.get(SessionVars.CURRENT_DATABASE), expr.getName()), child); ctx.queryBlock.addRelation(node); + + return node; + } + + @Override + public LogicalNode visitSimpleTableSubquery(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, SimpleTableSubquery expr) + throws TajoException { + LogicalPlanner.PlanContext newContext; + // Note: TableSubQuery always has a table name. + // SELECT .... FROM (SELECT ...) TB_NAME <- + QueryBlock queryBlock = ctx.plan.newQueryBlock(); + newContext = new LogicalPlanner.PlanContext(ctx, queryBlock); + LogicalNode child = super.visitSimpleTableSubquery(newContext, stack, expr); + queryBlock.setRoot(child); + + // a table subquery should be dealt as a relation. + TableSubQueryNode node = ctx.plan.createNode(TableSubQueryNode.class); + node.init(CatalogUtil.buildFQName(ctx.queryContext.get(SessionVars.CURRENT_DATABASE), + ctx.generateUniqueSubQueryName()), child); + ctx.queryBlock.addRelation(node); + if (stack.peek().getType() == OpType.InPredicate) { + // In-subquery and scalar subquery cannot be the base for name resolution. + node.setNameResolveBase(false); + } return node; } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 9b114f1..4b17b0e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -99,7 +99,9 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex QueryBlock queryBlock; EvalTreeOptimizer evalOptimizer; TimeZone timeZone; + List<Expr> unplannedExprs = TUtil.newList(); boolean debugOrUnitTests; + Integer noNameSubqueryId = 0; public PlanContext(OverridableConf context, LogicalPlan plan, QueryBlock block, EvalTreeOptimizer evalOptimizer, boolean debugOrUnitTests) { @@ -137,6 +139,13 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex return "block=" + queryBlock.getName() + ", relNum=" + queryBlock.getRelations().size() + ", "+ queryBlock.namedExprsMgr.toString(); } + + /** + * It generates a unique table subquery name + */ + public String generateUniqueSubQueryName() { + return LogicalPlan.NONAME_SUBQUERY_PREFIX + noNameSubqueryId++; + } } /** @@ -152,7 +161,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex @VisibleForTesting public LogicalPlan createPlan(OverridableConf queryContext, Expr expr, boolean debug) throws TajoException { - LogicalPlan plan = new LogicalPlan(this); + LogicalPlan plan = new LogicalPlan(); QueryBlock rootBlock = plan.newAndGetBlock(LogicalPlan.ROOT_BLOCK); PlanContext context = new PlanContext(queryContext, plan, rootBlock, evalOptimizer, debug); @@ -234,8 +243,6 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex public LogicalNode visitProjection(PlanContext context, Stack<Expr> stack, Projection projection) throws TajoException { - - LogicalPlan plan = context.plan; QueryBlock block = context.queryBlock; // If a non-from statement is given @@ -279,6 +286,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex projectionNode.init(projection.isDistinct(), targets); projectionNode.setChild(child); projectionNode.setInSchema(child.getOutSchema()); + projectionNode.setOutSchema(PlannerUtil.targetToSchema(targets)); if (projection.isDistinct() && block.hasNode(NodeType.GROUP_BY)) { throw makeSyntaxError("Cannot support grouping and distinct at the same time yet"); @@ -373,7 +381,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex } private interface Matcher { - public boolean isMatch(Expr expr); + boolean isMatch(Expr expr); } public List<Integer> normalize(PlanContext context, Projection projection, ExprNormalizedResult [] normalizedExprList, @@ -1089,6 +1097,12 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex // Visit and Build Child Plan //////////////////////////////////////////////////////// stack.push(selection); + // Since filter push down will be done later, it is guaranteed that in-subqueries are found at only selection. + for (Expr eachQual : PlannerUtil.extractInSubquery(selection.getQual())) { + InPredicate inPredicate = (InPredicate) eachQual; + visit(context, stack, inPredicate.getInValue()); + context.unplannedExprs.add(inPredicate.getInValue()); + } LogicalNode child = visit(context, stack, selection.getChild()); stack.pop(); //////////////////////////////////////////////////////// @@ -1384,13 +1398,26 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex } } + @Override public TableSubQueryNode visitTableSubQuery(PlanContext context, Stack<Expr> stack, TablePrimarySubQuery expr) throws TajoException { + return visitCommonTableSubquery(context, stack, expr); + } + + @Override + public TableSubQueryNode visitSimpleTableSubquery(PlanContext context, Stack<Expr> stack, SimpleTableSubquery expr) + throws TajoException { + return visitCommonTableSubquery(context, stack, expr); + } + + private TableSubQueryNode visitCommonTableSubquery(PlanContext context, Stack<Expr> stack, CommonSubquery expr) + throws TajoException { QueryBlock currentBlock = context.queryBlock; QueryBlock childBlock = context.plan.getBlock(context.plan.getBlockNameByExpr(expr.getSubQuery())); context.plan.connectBlocks(childBlock, currentBlock, BlockType.TableSubQuery); PlanContext newContext = new PlanContext(context, childBlock); + context.plan.connectBlocks(childBlock, context.queryBlock, BlockType.TableSubQuery); LogicalNode child = visit(newContext, new Stack<Expr>(), expr.getSubQuery()); TableSubQueryNode subQueryNode = currentBlock.getNodeFromExpr(expr); @@ -1400,7 +1427,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex return subQueryNode; } - private void setTargetOfTableSubQuery (PlanContext context, QueryBlock block, TableSubQueryNode subQueryNode) throws TajoException { + private void setTargetOfTableSubQuery (PlanContext context, QueryBlock block, TableSubQueryNode subQueryNode) + throws TajoException { // Add additional expressions required in upper nodes. Set<String> newlyEvaluatedExprs = TUtil.newHashSet(); for (NamedExpr rawTarget : block.namedExprsMgr.getAllNamedExprs()) { @@ -1457,12 +1485,15 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex return resultingNode; } - private ProjectionNode insertProjectionGroupbyBeforeSetOperation(PlanContext context, SetOperationNode setOperationNode) throws TajoException { + private ProjectionNode insertProjectionGroupbyBeforeSetOperation(PlanContext context, + SetOperationNode setOperationNode) + throws TajoException { QueryBlock currentBlock = context.queryBlock; // make table subquery node which has set operation as its subquery TableSubQueryNode setOpTableSubQueryNode = context.plan.createNode(TableSubQueryNode.class); - setOpTableSubQueryNode.init(CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), context.plan.generateUniqueSubQueryName()), setOperationNode); + setOpTableSubQueryNode.init(CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), + context.generateUniqueSubQueryName()), setOperationNode); setTargetOfTableSubQuery(context, currentBlock, setOpTableSubQueryNode); currentBlock.registerNode(setOpTableSubQueryNode); currentBlock.addRelation(setOpTableSubQueryNode); http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java index 6149080..c795c09 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java @@ -38,7 +38,7 @@ public interface AlgebraVisitor<CONTEXT, RESULT> { RESULT visitUnion(CONTEXT ctx, Stack<Expr> stack, SetOperation expr) throws TajoException; RESULT visitExcept(CONTEXT ctx, Stack<Expr> stack, SetOperation expr) throws TajoException; RESULT visitIntersect(CONTEXT ctx, Stack<Expr> stack, SetOperation expr) throws TajoException; - RESULT visitSimpleTableSubQuery(CONTEXT ctx, Stack<Expr> stack, SimpleTableSubQuery expr) throws TajoException; + RESULT visitSimpleTableSubquery(CONTEXT ctx, Stack<Expr> stack, SimpleTableSubquery expr) throws TajoException; RESULT visitTableSubQuery(CONTEXT ctx, Stack<Expr> stack, TablePrimarySubQuery expr) throws TajoException; RESULT visitRelationList(CONTEXT ctx, Stack<Expr> stack, RelationList expr) throws TajoException; RESULT visitRelation(CONTEXT ctx, Stack<Expr> stack, Relation expr) throws TajoException; http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java index 2b4fb30..2d200fc 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java @@ -86,8 +86,8 @@ public class BaseAlgebraVisitor<CONTEXT, RESULT> implements AlgebraVisitor<CONTE case Intersect: current = visitIntersect(ctx, stack, (SetOperation) expr); break; - case SimpleTableSubQuery: - current = visitSimpleTableSubQuery(ctx, stack, (SimpleTableSubQuery) expr); + case SimpleTableSubquery: + current = visitSimpleTableSubquery(ctx, stack, (SimpleTableSubquery) expr); break; case TablePrimaryTableSubQuery: current = visitTableSubQuery(ctx, stack, (TablePrimarySubQuery) expr); @@ -404,9 +404,12 @@ public class BaseAlgebraVisitor<CONTEXT, RESULT> implements AlgebraVisitor<CONTE } @Override - public RESULT visitSimpleTableSubQuery(CONTEXT ctx, Stack<Expr> stack, SimpleTableSubQuery expr) + public RESULT visitSimpleTableSubquery(CONTEXT ctx, Stack<Expr> stack, SimpleTableSubquery expr) throws TajoException { - return visitDefaultUnaryExpr(ctx, stack, expr); + stack.push(expr); + RESULT child = visit(ctx, stack, expr.getSubQuery()); + stack.pop(); + return child; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java index 59be24a..84da79e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java @@ -136,6 +136,10 @@ public class BasicEvalNodeVisitor<CONTEXT, RESULT> implements EvalNodeVisitor2<C result = visitCast(context, (CastEval) evalNode, stack); break; + case SUBQUERY: + result = visitSubquery(context, (SubqueryEval) evalNode, stack); + break; + default: throw new UnsupportedException("Unknown EvalType: " + evalNode); } @@ -342,4 +346,9 @@ public class BasicEvalNodeVisitor<CONTEXT, RESULT> implements EvalNodeVisitor2<C public RESULT visitCast(CONTEXT context, CastEval castEval, Stack<EvalNode> stack) { return visitDefaultUnaryEval(context, castEval, stack); } + + @Override + public RESULT visitSubquery(CONTEXT context, SubqueryEval signedEval, Stack<EvalNode> stack) { + return null; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java index 43729ac..ed4e940 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java @@ -69,4 +69,6 @@ public interface EvalNodeVisitor2<CONTEXT, RESULT> { RESULT visitSigned(CONTEXT context, SignedEval signedEval, Stack<EvalNode> stack); RESULT visitCast(CONTEXT context, CastEval signedEval, Stack<EvalNode> stack); + + RESULT visitSubquery(CONTEXT context, SubqueryEval signedEval, Stack<EvalNode> stack); } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java index c1df658..2c2a52f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java @@ -65,7 +65,9 @@ public enum EvalType { CAST(CastEval.class), ROW_CONSTANT(RowConstantEval.class), FIELD(FieldEval.class), - CONST(ConstEval.class); + CONST(ConstEval.class), + + SUBQUERY(SubqueryEval.class); private Class<? extends EvalNode> baseClass; private String operatorName; http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/expr/InEval.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/InEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/InEval.java index 7052663..70346b4 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/InEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/InEval.java @@ -19,15 +19,14 @@ package org.apache.tajo.plan.expr; -import com.google.common.collect.Sets; import com.google.gson.annotations.Expose; - import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.TUtil; import java.util.Set; @@ -37,7 +36,7 @@ public class InEval extends BinaryEval { @Expose private boolean not; Set<Datum> values; - public InEval(EvalNode lhs, RowConstantEval valueList, boolean not) { + public InEval(EvalNode lhs, ValueSetEval valueList, boolean not) { super(EvalType.IN, lhs, valueList); this.not = not; } @@ -62,7 +61,7 @@ public class InEval extends BinaryEval { throw new IllegalStateException("bind() must be called before eval()"); } if (values == null) { - values = Sets.newHashSet(((RowConstantEval)rightExpr).getValues()); + values = TUtil.newHashSet(((ValueSetEval) rightExpr).getValues()); } Datum leftValue = leftExpr.eval(tuple); @@ -93,6 +92,6 @@ public class InEval extends BinaryEval { } public String toString() { - return leftExpr + " IN (" + rightExpr + ")"; + return leftExpr + (not? " NOT" : "") + " IN (" + rightExpr + ")"; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java index eddb022..d2dae5a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java @@ -18,10 +18,7 @@ package org.apache.tajo.plan.expr; -import java.util.Arrays; - import com.google.gson.annotations.Expose; - import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; @@ -29,9 +26,11 @@ import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; +import java.util.Arrays; + import static org.apache.tajo.common.TajoDataTypes.DataType; -public class RowConstantEval extends EvalNode { +public class RowConstantEval extends ValueSetEval { @Expose Datum [] values; public RowConstantEval(Datum [] values) { @@ -45,16 +44,6 @@ public class RowConstantEval extends EvalNode { } @Override - public int childNum() { - return 0; - } - - @Override - public EvalNode getChild(int idx) { - return null; - } - - @Override public String getName() { return "ROW"; } @@ -66,10 +55,6 @@ public class RowConstantEval extends EvalNode { return NullDatum.get(); } - public Datum [] getValues() { - return values; - } - @Override public int hashCode() { final int prime = 31; @@ -92,14 +77,6 @@ public class RowConstantEval extends EvalNode { return StringUtils.join(values); } - public void preOrder(EvalNodeVisitor visitor) { - visitor.visit(this); - } - - public void postOrder(EvalNodeVisitor visitor) { - visitor.visit(this); - } - @Override public Object clone() throws CloneNotSupportedException { RowConstantEval rowConstantEval = (RowConstantEval) super.clone(); @@ -109,4 +86,9 @@ public class RowConstantEval extends EvalNode { } return rowConstantEval; } + + @Override + public Datum[] getValues() { + return values; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java index 61a25a9..9515fe8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java @@ -76,6 +76,10 @@ public abstract class SimpleEvalNodeVisitor<CONTEXT> { result = visitFuncCall(context, (FunctionEval) evalNode, stack); break; + case SUBQUERY: + result = visitSubquery(context, (SubqueryEval) evalNode, stack); + break; + default: throw new TajoInternalError("Unknown EvalType: " + evalNode); } @@ -170,4 +174,8 @@ public abstract class SimpleEvalNodeVisitor<CONTEXT> { protected EvalNode visitFuncCall(CONTEXT context, FunctionEval evalNode, Stack<EvalNode> stack) { return visitDefaultFunctionEval(context, stack, evalNode); } + + protected EvalNode visitSubquery(CONTEXT context, SubqueryEval evalNode, Stack<EvalNode> stack) { + return evalNode; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/042c3e88/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SubqueryEval.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SubqueryEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SubqueryEval.java new file mode 100644 index 0000000..98c36a1 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SubqueryEval.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.expr; + +import org.apache.tajo.annotation.Nullable; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.plan.logical.TableSubQueryNode; +import org.apache.tajo.storage.Tuple; + +/** + * SubqueryEval is a temporal eval to keep subquery information when the subquery occurs in expressions, + * such as in subquery or scalar subquery, before {@link org.apache.tajo.plan.rewrite.rules.InSubqueryRewriteRule} is + * applied. + * During in subquery rewrite phase, A SubqueryEval is expected to be replaced with a Join. + * + */ +public class SubqueryEval extends ValueSetEval { + + private TableSubQueryNode subQueryNode; + + public SubqueryEval(TableSubQueryNode subQueryNode) { + super(EvalType.SUBQUERY); + this.subQueryNode = subQueryNode; + } + + @Override + public DataType getValueType() { + return subQueryNode.getOutSchema().getColumn(0).getDataType(); + } + + @Override + public String getName() { + return "SUBQUERY"; + } + + @Override + public EvalNode bind(@Nullable EvalContext evalContext, Schema schema) { + throw new UnsupportedException("Cannot call bind()"); + } + + @Override + public Datum eval(Tuple tuple) { + throw new UnsupportedException("Cannot call eval()"); + } + + public TableSubQueryNode getSubQueryNode() { + return subQueryNode; + } + + @Override + public int hashCode() { + return subQueryNode.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof SubqueryEval) { + SubqueryEval other = (SubqueryEval) o; + return this.subQueryNode.equals(other.subQueryNode); + } + return false; + } + + @Override + public Object clone() throws CloneNotSupportedException { + SubqueryEval clone = (SubqueryEval) super.clone(); + clone.subQueryNode = (TableSubQueryNode) this.subQueryNode.clone(); + return clone; + } + + @Override + public String toString() { + return subQueryNode.toString(); + } + + @Override + public Datum[] getValues() { + throw new UnsupportedException("Cannot call getValues()"); + } +}
