TAJO-1426: Support "explain global" to get physical plan. Closes #441
Signed-off-by: Jihoon Son <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3e9a2dd2 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3e9a2dd2 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3e9a2dd2 Branch: refs/heads/index_support Commit: 3e9a2dd2ba86653ec7b90394518a1a700bd937dc Parents: d7b5212 Author: Jihoon Son <[email protected]> Authored: Tue Mar 24 17:12:24 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Tue Mar 24 17:13:57 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../java/org/apache/tajo/algebra/Explain.java | 9 +- .../main/java/org/apache/tajo/SessionVars.java | 1 + .../java/org/apache/tajo/conf/TajoConf.java | 1 + .../java/org/apache/tajo/util/FileUtil.java | 25 ++- .../org/apache/tajo/engine/parser/SQLLexer.g4 | 1 + .../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +- .../apache/tajo/engine/parser/SQLAnalyzer.java | 2 +- .../engine/planner/global/GlobalPlanner.java | 4 + .../ExplainGlobalPlanPreprocessorForTest.java | 62 ++++++ .../exec/ExplainPlanPreprocessorForTest.java | 218 +++++++++++++++++++ .../apache/tajo/master/exec/QueryExecutor.java | 58 ++++- .../tajo/querymaster/QueryMasterTask.java | 1 + .../java/org/apache/tajo/QueryTestCaseBase.java | 75 ++++++- .../tajo/engine/query/TestSelectQuery.java | 16 ++ .../testExplainSelectPhysical.1.result | 26 +++ .../testExplainSelectPhysical.2.result | 88 ++++++++ .../testExplainSelectPhysical.3.result | 89 ++++++++ .../java/org/apache/tajo/plan/LogicalPlan.java | 17 +- .../org/apache/tajo/plan/LogicalPlanner.java | 2 +- 20 files changed, 681 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 58c061a..f573550 100644 --- a/CHANGES +++ b/CHANGES @@ -9,6 +9,9 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-1426: Support "explain global" to get physical plan. (Contributed by + navis, Committed by jihoon) + TAJO-1407: Minor performance improvement of MemSortExec. (Contributed by navis, Committed by jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-algebra/src/main/java/org/apache/tajo/algebra/Explain.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Explain.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Explain.java index ee76ea9..2e966b4 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Explain.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Explain.java @@ -22,11 +22,18 @@ import com.google.common.base.Objects; public class Explain extends UnaryOperator { - public Explain(Expr operand) { + private boolean isGlobal; + + public Explain(Expr operand, boolean isGlobal) { super(OpType.Explain); + this.isGlobal = isGlobal; setChild(operand); } + public boolean isGlobal() { + return isGlobal; + } + public int hashCode() { return Objects.hashCode(getChild()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-common/src/main/java/org/apache/tajo/SessionVars.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index b3233ed..5cca413 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -139,6 +139,7 @@ public enum SessionVars implements ConfigKey { TEST_JOIN_OPT_ENABLED(ConfVars.$TEST_JOIN_OPT_ENABLED, "(test only) join optimization enabled", TEST_VAR), TEST_FILTER_PUSHDOWN_ENABLED(ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED, "filter push down enabled", TEST_VAR), TEST_MIN_TASK_NUM(ConfVars.$TEST_MIN_TASK_NUM, "(test only) min task num", TEST_VAR), + TEST_PLAN_SHAPE_FIX_ENABLED(ConfVars.$TEST_PLAN_SHAPE_FIX_ENABLED, "(test only) plan shape fix enabled", TEST_VAR), ; public static final Map<String, SessionVars> SESSION_VARS = Maps.newHashMap(); http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 5b569d5..ecdb2ef 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -355,6 +355,7 @@ public class TajoConf extends Configuration { $TEST_JOIN_OPT_ENABLED("tajo.test.plan.join-optimization.enabled", true), $TEST_FILTER_PUSHDOWN_ENABLED("tajo.test.plan.filter-pushdown.enabled", true), $TEST_MIN_TASK_NUM("tajo.test.min-task-num", -1), + $TEST_PLAN_SHAPE_FIX_ENABLED("tajo.test.plan.shape.fix.enabled", false), // used for explain statement test // Behavior Control --------------------------------------------------------- $BEHAVIOR_ARITHMETIC_ABORT("tajo.behavior.arithmetic-abort", false), http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java index 9aa6af9..9403a2f 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java @@ -87,19 +87,23 @@ public class FileUtil { } public static String readTextFileFromResource(String resource) throws IOException { - StringBuilder fileData = new StringBuilder(1000); - InputStream inputStream = ClassLoader.getSystemResourceAsStream(resource); - byte[] buf = new byte[1024]; - int numRead; + return readTextFromStream(ClassLoader.getSystemResourceAsStream(resource)); + } + + public static String readTextFromStream(InputStream inputStream) + throws IOException { try { + StringBuilder fileData = new StringBuilder(1000); + byte[] buf = new byte[1024]; + int numRead; while ((numRead = inputStream.read(buf)) != -1) { String readData = new String(buf, 0, numRead, Charset.defaultCharset()); fileData.append(readData); } + return fileData.toString(); } finally { - IOUtils.cleanup(null, inputStream); + IOUtils.closeStream(inputStream); } - return fileData.toString(); } public static String readTextFile(File file) throws IOException { @@ -119,6 +123,15 @@ public class FileUtil { return fileData.toString(); } + public static void writeTextToStream(String text, OutputStream outputStream) + throws IOException { + try { + outputStream.write(text.getBytes()); + } finally { + IOUtils.closeStream(outputStream); + } + } + public static String humanReadableByteCount(long bytes, boolean si) { int unit = si ? 1000 : 1024; if (bytes < unit) return bytes + " B"; http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 index f42e114..6fccaad 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 @@ -237,6 +237,7 @@ FOLLOWING : F O L L O W I N G; FORMAT : F O R M A T; FUSION : F U S I O N; +GLOBAL : G L O B A L; GROUPING : G R O U P I N G; HASH : H A S H; http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index a05a060..9ac3f8c 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -39,7 +39,7 @@ sql ; explain_clause - : EXPLAIN + : EXPLAIN (GLOBAL)? ; statement http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 869c0eb..23c2eec 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -78,7 +78,7 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { public Expr visitSql(SqlContext ctx) { Expr statement = visit(ctx.statement()); if (checkIfExist(ctx.explain_clause())) { - return new Explain(statement); + return new Explain(statement, checkIfExist(ctx.explain_clause().GLOBAL())); } else { return statement; } http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index d2ac6cc..cd35d96 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -91,6 +91,10 @@ public class GlobalPlanner { this(conf, workerContext.getCatalog()); } + public TajoConf getConf() { + return conf; + } + public CatalogService getCatalog() { return catalog; } http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java new file mode 100644 index 0000000..c26e12c --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.master.exec; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.engine.planner.global.DataChannel; +import org.apache.tajo.engine.planner.global.ExecutionBlock; +import org.apache.tajo.engine.planner.global.ExecutionBlockCursor; +import org.apache.tajo.engine.planner.global.MasterPlan; + +import java.util.Arrays; +import java.util.List; + +/** + * Data channels of a global plan can have multiple shuffle keys, and their appearance order is basically not preserved. + * However, to test the equivalence of global plans, the appearance order of shuffle keys must be preserved. + * This class guarantees the consistency of the order of shuffle keys. + */ +public class ExplainGlobalPlanPreprocessorForTest { + private static final ExplainPlanPreprocessorForTest.ColumnComparator columnComparator = + new ExplainPlanPreprocessorForTest.ColumnComparator(); + + /** + * For all data channels, sort shuffle keys by their names. + * + * @param plan master plan + */ + public void prepareTest(MasterPlan plan) { + ExecutionBlockCursor cursor = new ExecutionBlockCursor(plan); + + while (cursor.hasNext()) { + ExecutionBlock block = cursor.nextBlock(); + List<DataChannel> outgoingChannels = plan.getOutgoingChannels(block.getId()); + if (outgoingChannels != null) { + for (DataChannel channel : outgoingChannels) { + if (channel.hasShuffleKeys()) { + Column[] shuffleKeys = channel.getShuffleKeys(); + Arrays.sort(shuffleKeys, columnComparator); + channel.setShuffleKeys(shuffleKeys); + } + } + } + } + } + +} http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java new file mode 100644 index 0000000..ab37e22 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.master.exec; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.AlgebraicUtil; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.Stack; + +/** + * Tajo's logical planner can generate different shapes of logical plans for the same query, + * especially when the query involves one or more joins. + * This class guarantees the consistency of the logical plan for the same query. + */ +public class ExplainPlanPreprocessorForTest { + private static final PlanShapeFixerContext shapeFixerContext = new PlanShapeFixerContext(); + private static final PlanShapeFixer shapeFixer = new PlanShapeFixer(); + private static final PidResetContext resetContext = new PidResetContext(); + private static final PidReseter pidReseter = new PidReseter(); + + public void prepareTest(LogicalPlan plan) throws PlanningException { + // Pid reseter + resetContext.reset(); + pidReseter.visit(resetContext, plan, plan.getRootBlock()); + + // Plan shape fixer + shapeFixerContext.reset(); + shapeFixer.visit(shapeFixerContext, plan, plan.getRootBlock()); + } + + private static class PlanShapeFixerContext { + + Stack<Integer> childNumbers = new Stack<Integer>(); + public void reset() { + childNumbers.clear(); + } + } + + /** + * Given a commutative join, two children of the join node are interchangeable. + * This class fix the logical plan according to the following rules. + * + * <h3>Rules</h3> + * <ul> + * <li>When one of the both children has more descendants, + * change the plan in order that the left child is the one who has more descendants.</li> + * <li>When both children have the same number of descendants, + * their order is decided based on their string representation.</li> + * </ul> + * + * In addition, in/out schemas, quals, and targets are sorted by their names. + */ + private static class PlanShapeFixer extends BasicLogicalPlanVisitor<PlanShapeFixerContext, LogicalNode> { + private static final ColumnComparator columnComparator = new ColumnComparator(); + private static final EvalNodeComparator evalNodeComparator = new EvalNodeComparator(); + private static final TargetComparator targetComparator = new TargetComparator(); + + @Override + public LogicalNode visit(PlanShapeFixerContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + LogicalNode node, Stack<LogicalNode> stack) throws PlanningException { + super.visit(context, plan, block, node, stack); + context.childNumbers.push(context.childNumbers.pop()+1); + return null; + } + + @Override + public LogicalNode visitScan(PlanShapeFixerContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + ScanNode node, Stack<LogicalNode> stack) throws PlanningException { + super.visitScan(context, plan, block, node, stack); + context.childNumbers.push(1); + node.setInSchema(sortSchema(node.getInSchema())); + if (node.hasQual()) { + node.setQual(sortQual(node.getQual())); + } + return null; + } + + @Override + public LogicalNode visitJoin(PlanShapeFixerContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + JoinNode node, Stack<LogicalNode> stack) throws PlanningException { + super.visitJoin(context, plan, block, node, stack); + int rightChildNum = context.childNumbers.pop(); + int leftChildNum = context.childNumbers.pop(); + + if (PlannerUtil.isCommutativeJoin(node.getJoinType())) { + + if (leftChildNum < rightChildNum) { + swapChildren(node); + } else if (leftChildNum == rightChildNum) { + if (node.getLeftChild().toString().compareTo(node.getRightChild().toString()) < + 0) { + swapChildren(node); + } + } + } + + node.setInSchema(sortSchema(node.getInSchema())); + node.setOutSchema(sortSchema(node.getOutSchema())); + + if (node.hasJoinQual()) { + node.setJoinQual(sortQual(node.getJoinQual())); + } + + if (node.hasTargets()) { + node.setTargets(sortTargets(node.getTargets())); + } + + context.childNumbers.push(rightChildNum + leftChildNum); + + return null; + } + + private Schema sortSchema(Schema schema) { + Column[] columns = schema.toArray(); + Arrays.sort(columns, columnComparator); + + Schema sorted = new Schema(); + for (Column col : columns) { + sorted.addColumn(col); + } + return sorted; + } + + private EvalNode sortQual(EvalNode qual) { + EvalNode[] cnf = AlgebraicUtil.toConjunctiveNormalFormArray(qual); + Arrays.sort(cnf, evalNodeComparator); + return AlgebraicUtil.createSingletonExprFromCNF(cnf); + } + + private Target[] sortTargets(Target[] targets) { + Arrays.sort(targets, targetComparator); + return targets; + } + + private static void swapChildren(JoinNode node) { + LogicalNode tmpChild = node.getLeftChild(); + int tmpId = tmpChild.getPID(); + tmpChild.setPID(node.getRightChild().getPID()); + node.getRightChild().setPID(tmpId); + node.setLeftChild(node.getRightChild()); + node.setRightChild(tmpChild); + } + } + + public static class ColumnComparator implements Comparator<Column> { + + @Override + public int compare(Column o1, Column o2) { + return o1.getQualifiedName().compareTo(o2.getQualifiedName()); + } + } + + private static class EvalNodeComparator implements Comparator<EvalNode> { + + @Override + public int compare(EvalNode o1, EvalNode o2) { + return o1.toJson().compareTo(o2.toJson()); + } + } + + private static class TargetComparator implements Comparator<Target> { + + @Override + public int compare(Target o1, Target o2) { + return o1.toJson().compareTo(o2.toJson()); + } + } + + private static class PidResetContext { + int seqId = 0; + public void reset() { + seqId = 0; + } + } + + /** + * During join order optimization, new join nodes are created based on the chosen join order. + * So, each join node has different pids. + * This class sequentially assigns unique pids to all logical nodes. + */ + private static class PidReseter extends BasicLogicalPlanVisitor<PidResetContext, LogicalNode> { + + @Override + public void preHook(LogicalPlan plan, LogicalNode node, Stack<LogicalNode> stack, PidResetContext context) + throws PlanningException { + node.setPID(context.seqId++); + } + } + +} http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index aa8b228..75e7762 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -36,6 +36,8 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.engine.planner.global.GlobalPlanner; +import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.planner.physical.EvalExprExec; import org.apache.tajo.engine.planner.physical.StoreTableExec; import org.apache.tajo.engine.query.QueryContext; @@ -45,6 +47,7 @@ import org.apache.tajo.master.*; import org.apache.tajo.master.exec.prehook.CreateTableHook; import org.apache.tajo.master.exec.prehook.DistributedQueryHookManager; import org.apache.tajo.master.exec.prehook.InsertIntoHook; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.querymaster.*; import org.apache.tajo.session.Session; import org.apache.tajo.plan.LogicalPlan; @@ -101,7 +104,7 @@ public class QueryExecutor { } else if (plan.isExplain()) { // explain query - execExplain(plan, response); + execExplain(plan, queryContext, plan.isExplainGlobal(), response); } else if (PlannerUtil.checkIfQueryTargetIsVirtualTable(plan)) { execQueryOnVirtualTable(queryContext, session, sql, plan, response); @@ -157,9 +160,28 @@ public class QueryExecutor { response.setResultCode(ClientProtos.ResultCode.OK); } - public void execExplain(LogicalPlan plan, SubmitQueryResponse.Builder response) throws IOException { + public void execExplain(LogicalPlan plan, QueryContext queryContext, boolean isGlobal, + SubmitQueryResponse.Builder response) + throws Exception { + String explainStr; + boolean isTest = queryContext.getBool(SessionVars.TEST_PLAN_SHAPE_FIX_ENABLED); + if (isTest) { + ExplainPlanPreprocessorForTest preprocessorForTest = new ExplainPlanPreprocessorForTest(); + preprocessorForTest.prepareTest(plan); + } + + if (isGlobal) { + GlobalPlanner planner = new GlobalPlanner(context.getConf(), context.getCatalog()); + MasterPlan masterPlan = compileMasterPlan(plan, queryContext, planner); + if (isTest) { + ExplainGlobalPlanPreprocessorForTest globalPlanPreprocessorForTest = new ExplainGlobalPlanPreprocessorForTest(); + globalPlanPreprocessorForTest.prepareTest(masterPlan); + } + explainStr = masterPlan.toString(); + } else { + explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot()); + } - String explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot()); Schema schema = new Schema(); schema.addColumn("explain", TajoDataTypes.Type.TEXT); RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(schema); @@ -416,4 +438,34 @@ public class QueryExecutor { " is forwarded to " + queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort()); } } + + public static MasterPlan compileMasterPlan(LogicalPlan plan, QueryContext context, GlobalPlanner planner) + throws Exception { + + CatalogProtos.StoreType storeType = PlannerUtil.getStoreType(plan); + if (storeType != null) { + StorageManager sm = StorageManager.getStorageManager(planner.getConf(), storeType); + StorageProperty storageProperty = sm.getStorageProperty(); + if (storageProperty.isSortedInsert()) { + String tableName = PlannerUtil.getStoreTableName(plan); + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + TableDesc tableDesc = PlannerUtil.getTableDesc(planner.getCatalog(), rootNode.getChild()); + if (tableDesc == null) { + throw new VerifyException("Can't get table meta data from catalog: " + tableName); + } + List<LogicalPlanRewriteRule> storageSpecifiedRewriteRules = sm.getRewriteRules( + context, tableDesc); + if (storageSpecifiedRewriteRules != null) { + for (LogicalPlanRewriteRule eachRule: storageSpecifiedRewriteRules) { + eachRule.rewrite(context, plan); + } + } + } + } + + MasterPlan masterPlan = new MasterPlan(QueryIdFactory.NULL_QUERY_ID, context, plan); + planner.build(masterPlan); + + return masterPlan; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index 0d1924b..f83cb1e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -38,6 +38,7 @@ import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.UnimplementedException; http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 15fbdae..4e104dd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -21,9 +21,12 @@ package org.apache.tajo; import com.google.protobuf.ServiceException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.CatalogService; @@ -50,10 +53,19 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TestName; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.lang.annotation.Annotation; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.Method; import java.net.URL; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -181,6 +193,8 @@ public class QueryTestCaseBase { protected Path currentResultPath; protected Path currentDatasetPath; + protected FileSystem currentResultFS; + // for getting a method name @Rule public TestName name = new TestName(); @@ -243,8 +257,10 @@ public class QueryTestCaseBase { client.updateQuery("CREATE DATABASE IF NOT EXISTS " + CatalogUtil.denormalizeIdentifier(currentDatabase)); } client.selectDatabase(currentDatabase); - } catch (ServiceException e) { - e.printStackTrace(); + currentResultFS = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration()); + + } catch (Exception e) { + throw new RuntimeException(e); } testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false"); } @@ -317,6 +333,61 @@ public class QueryTestCaseBase { return executeFile(getMethodName() + ".sql"); } + private volatile Description current; + + @Rule + public TestRule watcher = new TestWatcher() { + @Override + protected void starting(Description description) { + QueryTestCaseBase.this.current = description; + } + }; + + @Target({ElementType.METHOD}) + @Retention(RetentionPolicy.RUNTIME) + protected static @interface SimpleTest { + String[] queries(); + String[] cleanupTables() default {}; + } + + protected void runSimpleTests() throws Exception { + String methodName = getMethodName(); + Method method = current.getTestClass().getMethod(methodName); + SimpleTest annotation = method.getAnnotation(SimpleTest.class); + if (annotation == null) { + throw new IllegalStateException("Cannot find test annotation"); + } + String[] queries = annotation.queries(); + try { + for (int i = 0; i < queries.length; i++) { + ResultSet result = client.executeQueryAndGetResult(queries[i]); + Path resultPath = StorageUtil.concatPath( + currentResultPath, methodName + "." + String.valueOf(i + 1) + ".result"); + if (currentResultFS.exists(resultPath)) { + assertEquals("Result Verification for: " + (i+1) + "th test", + FileUtil.readTextFromStream(currentResultFS.open(resultPath)), resultSetToString(result).trim()); + } else if (!isNull(result)) { + // If there is no result file expected, create gold files for new tests. + FileUtil.writeTextToStream(resultSetToString(result).trim(), currentResultFS.create(resultPath)); + LOG.info("New test output for " + current.getDisplayName() + " is written to " + resultPath); + // should be copied to src directory + } + } + } finally { + for (String tableName : annotation.cleanupTables()) { + try { + client.dropTable(tableName); + } catch (ServiceException e) { + // ignore + } + } + } + } + + private boolean isNull(ResultSet result) throws SQLException { + return result.getMetaData().getColumnCount() == 0; + } + protected String getMethodName() { String methodName = name.getMethodName(); // In the case of parameter execution name's pattern is methodName[0] http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index f7b1382..b54d7ea 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -105,6 +105,22 @@ public class TestSelectQuery extends QueryTestCaseBase { } @Test + @SimpleTest(queries = { + "explain global " + + "select l_orderkey, l_partkey from lineitem", + "explain global " + + "select n1.n_nationkey, n1.n_name, n2.n_name from nation n1 join nation n2 on n1.n_name = upper(n2.n_name) " + + "order by n1.n_nationkey;", + "explain global " + + "select l_linenumber, count(*), count(distinct l_orderkey), sum(distinct l_orderkey) from lineitem " + + "group by l_linenumber having sum(distinct l_orderkey) = 6"}) + public final void testExplainSelectPhysical() throws Exception { + // Enable this option to fix the shape of the generated plans. + testingCluster.getConfiguration().set(ConfVars.$TEST_PLAN_SHAPE_FIX_ENABLED.varname, "true"); + runSimpleTests(); + } + + @Test public final void testSelect() throws Exception { // select l_orderkey, l_partkey from lineitem; ResultSet res = executeQuery(); http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.1.result b/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.1.result new file mode 100644 index 0000000..0069639 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.1.result @@ -0,0 +1,26 @@ +explain +------------------------------- +------------------------------------------------------------------------------- +Execution Block Graph (TERMINAL - eb_0000000000000_0000_000002) +------------------------------------------------------------------------------- +|-eb_0000000000000_0000_000002 + |-eb_0000000000000_0000_000001 +------------------------------------------------------------------------------- +Order of Execution +------------------------------------------------------------------------------- +1: eb_0000000000000_0000_000001 +2: eb_0000000000000_0000_000002 +------------------------------------------------------------------------------- + +======================================================= +Block Id: eb_0000000000000_0000_000001 [ROOT] +======================================================= + +SCAN(0) on default.lineitem + => target list: default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4) + => out schema: {(2) default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4)} + => in schema: {(16) default.lineitem.l_comment (TEXT), default.lineitem.l_commitdate (TEXT), default.lineitem.l_discount (FLOAT8), default.lineitem.l_extendedprice (FLOAT8), default.lineitem.l_linenumber (INT4), default.lineitem.l_linestatus (TEXT), default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4), default.lineitem.l_quantity (FLOAT8), default.lineitem.l_receiptdate (TEXT), default.lineitem.l_returnflag (TEXT), default.lineitem.l_shipdate (TEXT), default.lineitem.l_shipinstruct (TEXT), default.lineitem.l_shipmode (TEXT), default.lineitem.l_suppkey (INT4), default.lineitem.l_tax (FLOAT8)} + +======================================================= +Block Id: eb_0000000000000_0000_000002 [TERMINAL] +======================================================= \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.2.result b/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.2.result new file mode 100644 index 0000000..7946c5b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.2.result @@ -0,0 +1,88 @@ +explain +------------------------------- +------------------------------------------------------------------------------- +Execution Block Graph (TERMINAL - eb_0000000000000_0000_000005) +------------------------------------------------------------------------------- +|-eb_0000000000000_0000_000005 + |-eb_0000000000000_0000_000004 + |-eb_0000000000000_0000_000003 + |-eb_0000000000000_0000_000002 + |-eb_0000000000000_0000_000001 +------------------------------------------------------------------------------- +Order of Execution +------------------------------------------------------------------------------- +1: eb_0000000000000_0000_000001 +2: eb_0000000000000_0000_000002 +3: eb_0000000000000_0000_000003 +4: eb_0000000000000_0000_000004 +5: eb_0000000000000_0000_000005 +------------------------------------------------------------------------------- + +======================================================= +Block Id: eb_0000000000000_0000_000001 [LEAF] +======================================================= + +[Outgoing] +[q_0000000000000_0000] 1 => 3 (type=HASH_SHUFFLE, key=?upper_1 (TEXT), num=32) + +SCAN(0) on default.nation as n2 + => target list: default.n2.n_name (TEXT), upper(default.n2.n_name (TEXT)) as ?upper_1 + => out schema: {(2) default.n2.n_name (TEXT), ?upper_1 (TEXT)} + => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000002 [LEAF] +======================================================= + +[Outgoing] +[q_0000000000000_0000] 2 => 3 (type=HASH_SHUFFLE, key=default.n1.n_name (TEXT), num=32) + +SCAN(1) on default.nation as n1 + => target list: default.n1.n_nationkey (INT4), default.n1.n_name (TEXT) + => out schema: {(2) default.n1.n_nationkey (INT4), default.n1.n_name (TEXT)} + => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000003 [INTERMEDIATE] +======================================================= + +[Incoming] +[q_0000000000000_0000] 1 => 3 (type=HASH_SHUFFLE, key=?upper_1 (TEXT), num=32) +[q_0000000000000_0000] 2 => 3 (type=HASH_SHUFFLE, key=default.n1.n_name (TEXT), num=32) + +[Outgoing] +[q_0000000000000_0000] 3 => 4 (type=RANGE_SHUFFLE, key=default.n1.n_nationkey (INT4), num=32) + +SORT(10) + => Sort Keys: default.n1.n_nationkey (INT4) (asc) + JOIN(6)(INNER) + => Join Cond: default.n1.n_name (TEXT) = ?upper_1 (TEXT) + => target list: default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n2.n_name (TEXT) + => out schema: {(3) default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n2.n_name (TEXT)} + => in schema: {(4) ?upper_1 (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n2.n_name (TEXT)} + SCAN(9) on eb_0000000000000_0000_000002 + => out schema: {(2) default.n1.n_nationkey (INT4), default.n1.n_name (TEXT)} + => in schema: {(2) default.n1.n_nationkey (INT4), default.n1.n_name (TEXT)} + SCAN(8) on eb_0000000000000_0000_000001 + => out schema: {(2) default.n2.n_name (TEXT), ?upper_1 (TEXT)} + => in schema: {(2) default.n2.n_name (TEXT), ?upper_1 (TEXT)} + +======================================================= +Block Id: eb_0000000000000_0000_000004 [ROOT] +======================================================= + +[Incoming] +[q_0000000000000_0000] 3 => 4 (type=RANGE_SHUFFLE, key=default.n1.n_nationkey (INT4), num=32) + +[Enforcers] + 0: sorted input=eb_0000000000000_0000_000003 + +SORT(3) + => Sort Keys: default.n1.n_nationkey (INT4) (asc) + SCAN(11) on eb_0000000000000_0000_000003 + => out schema: {(3) default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n2.n_name (TEXT)} + => in schema: {(3) default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n2.n_name (TEXT)} + +======================================================= +Block Id: eb_0000000000000_0000_000005 [TERMINAL] +======================================================= \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.3.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.3.result b/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.3.result new file mode 100644 index 0000000..c4e8c2c --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testExplainSelectPhysical.3.result @@ -0,0 +1,89 @@ +explain +------------------------------- +------------------------------------------------------------------------------- +Execution Block Graph (TERMINAL - eb_0000000000000_0000_000004) +------------------------------------------------------------------------------- +|-eb_0000000000000_0000_000004 + |-eb_0000000000000_0000_000003 + |-eb_0000000000000_0000_000002 + |-eb_0000000000000_0000_000001 +------------------------------------------------------------------------------- +Order of Execution +------------------------------------------------------------------------------- +1: eb_0000000000000_0000_000001 +2: eb_0000000000000_0000_000002 +3: eb_0000000000000_0000_000003 +4: eb_0000000000000_0000_000004 +------------------------------------------------------------------------------- + +======================================================= +Block Id: eb_0000000000000_0000_000001 [LEAF] +======================================================= + +[Outgoing] +[q_0000000000000_0000] 1 => 2 (type=HASH_SHUFFLE, key=?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), num=32) + +[Enforcers] + 0: type=Distinct,alg=hash + +DISTINCT_GROUP_BY(9)(l_linenumber) + => exprs: (count( distinct default.lineitem.l_orderkey (INT4)),sum( distinct default.lineitem.l_orderkey (INT4)),count()) + => target list: ?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), ?count (INT8) + => out schema:{(4) ?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), ?count (INT8)} + => in schema:{(2) default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4)} + => distinct: true, GROUP_BY(10)(l_orderkey), exprs: (count( distinct default.lineitem.l_orderkey (INT4)),sum( distinct default.lineitem.l_orderkey (INT4))), target list:{default.lineitem.l_orderkey (INT4), ?count_1 (INT8), ?sum_2 (INT8)}, out schema:{(3) default.lineitem.l_orderkey (INT4), ?count_1 (INT8), ?sum_2 (INT8)}, in schema:{(2) default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4)} + => distinct: false, GROUP_BY(11)(), exprs: (count()), target list:{?count (INT8)}, out schema:{(1) ?count (INT8)}, in schema:{(2) default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4)} + SCAN(0) on default.lineitem + => target list: default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4) + => out schema: {(2) default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4)} + => in schema: {(16) default.lineitem.l_comment (TEXT), default.lineitem.l_commitdate (TEXT), default.lineitem.l_discount (FLOAT8), default.lineitem.l_extendedprice (FLOAT8), default.lineitem.l_linenumber (INT4), default.lineitem.l_linestatus (TEXT), default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4), default.lineitem.l_quantity (FLOAT8), default.lineitem.l_receiptdate (TEXT), default.lineitem.l_returnflag (TEXT), default.lineitem.l_shipdate (TEXT), default.lineitem.l_shipinstruct (TEXT), default.lineitem.l_shipmode (TEXT), default.lineitem.l_suppkey (INT4), default.lineitem.l_tax (FLOAT8)} + +======================================================= +Block Id: eb_0000000000000_0000_000002 [INTERMEDIATE] +======================================================= + +[Incoming] +[q_0000000000000_0000] 1 => 2 (type=HASH_SHUFFLE, key=?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), num=32) + +[Outgoing] +[q_0000000000000_0000] 2 => 3 (type=HASH_SHUFFLE, key=default.lineitem.l_linenumber (INT4), num=32) + +[Enforcers] + 0: type=Distinct,alg=hash + +DISTINCT_GROUP_BY(12)(l_linenumber) + => exprs: (count( distinct default.lineitem.l_orderkey (INT4)),sum( distinct default.lineitem.l_orderkey (INT4)),count(?count (INT8))) + => target list: ?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), ?count (INT8) + => out schema:{(4) ?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), ?count (INT8)} + => in schema:{(4) ?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), ?count (INT8)} + => distinct: true, GROUP_BY(13)(l_orderkey), exprs: (count( distinct default.lineitem.l_orderkey (INT4)),sum( distinct default.lineitem.l_orderkey (INT4))), target list:{default.lineitem.l_orderkey (INT4), ?count_1 (INT8), ?sum_2 (INT8)}, out schema:{(3) default.lineitem.l_orderkey (INT4), ?count_1 (INT8), ?sum_2 (INT8)}, in schema:{(2) default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4)} + => distinct: false, GROUP_BY(14)(), exprs: (count(?count (INT8))), target list:{?count (INT8)}, out schema:{(1) ?count (INT8)}, in schema:{(2) default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4)} + SCAN(18) on eb_0000000000000_0000_000001 + => out schema: {(4) ?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), ?count (INT8)} + => in schema: {(4) ?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), ?count (INT8)} + +======================================================= +Block Id: eb_0000000000000_0000_000003 [ROOT] +======================================================= + +[Incoming] +[q_0000000000000_0000] 2 => 3 (type=HASH_SHUFFLE, key=default.lineitem.l_linenumber (INT4), num=32) + +[Enforcers] + 0: type=Distinct,alg=sort,keys=default.lineitem.l_orderkey | + +HAVING(2) (?sum_2 (INT8) = 6) + DISTINCT_GROUP_BY(15)(l_linenumber) + => exprs: (count( distinct default.lineitem.l_orderkey (INT4)),sum( distinct default.lineitem.l_orderkey (INT4)),count(?count (INT8))) + => target list: ?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), ?count (INT8) + => out schema:{(4) default.lineitem.l_linenumber (INT4), ?count (INT8), ?count_1 (INT8), ?sum_2 (INT8)} + => in schema:{(4) ?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), ?count (INT8)} + => distinct: true, GROUP_BY(16)(l_orderkey), exprs: (count( distinct default.lineitem.l_orderkey (INT4)),sum( distinct default.lineitem.l_orderkey (INT4))), target list:{default.lineitem.l_orderkey (INT4), ?count_1 (INT8), ?sum_2 (INT8)}, out schema:{(3) default.lineitem.l_orderkey (INT4), ?count_1 (INT8), ?sum_2 (INT8)}, in schema:{(2) default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4)} + => distinct: false, GROUP_BY(17)(), exprs: (count(?count (INT8))), target list:{?count (INT8)}, out schema:{(1) ?count (INT8)}, in schema:{(2) default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4)} + SCAN(19) on eb_0000000000000_0000_000002 + => out schema: {(4) ?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), ?count (INT8)} + => in schema: {(4) ?distinctseq (INT2), default.lineitem.l_linenumber (INT4), default.lineitem.l_orderkey (INT4), ?count (INT8)} + +======================================================= +Block Id: eb_0000000000000_0000_000004 [TERMINAL] +======================================================= \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index 0425f2e..17f79da 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@ -68,7 +68,12 @@ public class LogicalPlan { /** planning and optimization log */ private List<String> planingHistory = Lists.newArrayList(); - private boolean isExplain; + private static enum ExplainType { + NOT_EXPLAIN, + EXPLAIN_LOGICAL, + EXPLAIN_GLOBAL + } + private ExplainType explainType = ExplainType.NOT_EXPLAIN; public LogicalPlan(LogicalPlanner planner) { } @@ -104,12 +109,16 @@ public class LogicalPlan { } } - public void setExplain() { - isExplain = true; + public void setExplain(boolean isGlobal) { + explainType = isGlobal ? ExplainType.EXPLAIN_GLOBAL : ExplainType.EXPLAIN_LOGICAL; } public boolean isExplain() { - return isExplain; + return explainType != ExplainType.NOT_EXPLAIN; + } + + public boolean isExplainGlobal() { + return explainType == ExplainType.EXPLAIN_GLOBAL; } /** http://git-wip-us.apache.org/repos/asf/tajo/blob/3e9a2dd2/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 8395c3d..ff3d6c2 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -200,7 +200,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex } public LogicalNode visitExplain(PlanContext ctx, Stack<Expr> stack, Explain expr) throws PlanningException { - ctx.plan.setExplain(); + ctx.plan.setExplain(expr.isGlobal()); return visit(ctx, stack, expr.getChild()); }
