Repository: tajo Updated Branches: refs/heads/branch-0.11.0 7a1a8ce3c -> 357b7be81
TAJO-1833: Refine LogicalPlanPreprocessor to add new rules easily. Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/357b7be8 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/357b7be8 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/357b7be8 Branch: refs/heads/branch-0.11.0 Commit: 357b7be815e61bc55b5de6a6161ce1d9282fdf99 Parents: 7a1a8ce Author: Jihoon Son <[email protected]> Authored: Thu Sep 10 22:01:35 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Thu Sep 10 22:02:24 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../tajo/plan/LogicalPlanPreprocessor.java | 551 +---------------- .../org/apache/tajo/plan/LogicalPlanner.java | 6 +- .../BaseLogicalPlanPreprocessEngine.java | 100 ++++ .../BaseLogicalPlanPreprocessPhaseProvider.java | 34 ++ .../rewrite/BaseLogicalPlanRewriteEngine.java | 2 +- .../tajo/plan/rewrite/BaseSchemaBuildPhase.java | 591 +++++++++++++++++++ .../rewrite/LogicalPlanPreprocessEngine.java | 28 + .../rewrite/LogicalPlanPreprocessPhase.java | 64 ++ .../LogicalPlanPreprocessPhaseProvider.java | 30 + .../plan/rewrite/LogicalPlanRewriteEngine.java | 2 +- .../plan/rewrite/LogicalPlanRewriteRule.java | 4 +- 12 files changed, 873 insertions(+), 542 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 49c1d45..e6de3f0 100644 --- a/CHANGES +++ b/CHANGES @@ -526,6 +526,9 @@ Release 0.11.0 - unreleased TASKS + TAJO-1833: Refine LogicalPlanPreprocessor to add new rules easily. + (jihoon) + TAJO-1809: Change default value of several configurations. (jihoon) TAJO-1803: Use in-memory derby as the default catalog for unit tests. http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java index f2c15ac..1daba17 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java @@ -18,551 +18,28 @@ package org.apache.tajo.plan; -import org.apache.tajo.SessionVars; -import org.apache.tajo.algebra.*; -import org.apache.tajo.catalog.*; -import org.apache.tajo.exception.UndefinedColumnException; -import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.algebra.Expr; +import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.exception.TajoException; -import org.apache.tajo.plan.LogicalPlan.QueryBlock; -import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; -import org.apache.tajo.plan.expr.ConstEval; -import org.apache.tajo.plan.expr.EvalNode; -import org.apache.tajo.plan.expr.FieldEval; -import org.apache.tajo.plan.logical.*; -import org.apache.tajo.plan.nameresolver.NameResolver; -import org.apache.tajo.plan.nameresolver.NameResolvingMode; -import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; -import org.apache.tajo.util.TUtil; - -import java.util.*; +import org.apache.tajo.plan.LogicalPlanner.PlanContext; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.rewrite.BaseLogicalPlanPreprocessEngine; +import org.apache.tajo.plan.rewrite.BaseLogicalPlanPreprocessPhaseProvider; /** * It finds all relations for each block and builds base schema information. */ -public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.PlanContext, LogicalNode> { - private TypeDeterminant typeDeterminant; - private ExprAnnotator annotator; - - /** Catalog service */ - private CatalogService catalog; +public class LogicalPlanPreprocessor { + private BaseLogicalPlanPreprocessEngine engine; + private BaseLogicalPlanPreprocessPhaseProvider provider; LogicalPlanPreprocessor(CatalogService catalog, ExprAnnotator annotator) { - this.catalog = catalog; - this.annotator = annotator; - this.typeDeterminant = new TypeDeterminant(catalog); - } - - @Override - public void preHook(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Expr expr) throws TajoException { - ctx.queryBlock.setAlgebraicExpr(expr); - ctx.plan.mapExprToBlock(expr, ctx.queryBlock.getName()); - } - - @Override - public LogicalNode postHook(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Expr expr, LogicalNode result) - throws TajoException { - // If non-from statement, result can be null. It avoids that case. - if (result != null) { - // setNode method registers each node to corresponding block and plan. - ctx.queryBlock.registerNode(result); - // It makes a map between an expr and a logical node. - ctx.queryBlock.registerExprWithNode(expr, result); - } - return result; - } - - /** - * Get all columns of the relations correspondent to the asterisk expression. - * @param ctx - * @param asteriskExpr - * @return array of columns - * @throws TajoException - */ - public static Column[] getColumns(LogicalPlanner.PlanContext ctx, QualifiedAsteriskExpr asteriskExpr) - throws TajoException { - RelationNode relationOp = null; - QueryBlock block = ctx.queryBlock; - Collection<QueryBlock> queryBlocks = ctx.plan.getQueryBlocks(); - if (asteriskExpr.hasQualifier()) { - String qualifier; - - if (CatalogUtil.isFQTableName(asteriskExpr.getQualifier())) { - qualifier = asteriskExpr.getQualifier(); - } else { - qualifier = CatalogUtil.buildFQName( - ctx.queryContext.get(SessionVars.CURRENT_DATABASE), asteriskExpr.getQualifier()); - } - - relationOp = block.getRelation(qualifier); - - // if a column name is outside of this query block - if (relationOp == null) { - // TODO - nested query can only refer outer query block? or not? - for (QueryBlock eachBlock : queryBlocks) { - if (eachBlock.existsRelation(qualifier)) { - relationOp = eachBlock.getRelation(qualifier); - } - } - } - - // If we cannot find any relation against a qualified column name - if (relationOp == null) { - throw new UndefinedColumnException(CatalogUtil.buildFQName(qualifier, "*")); - } - - Schema schema = relationOp.getLogicalSchema(); - Column[] resolvedColumns = new Column[schema.size()]; - return schema.getRootColumns().toArray(resolvedColumns); - } else { // if a column reference is not qualified - // columns of every relation should be resolved. - Iterator<RelationNode> iterator = block.getRelations().iterator(); - Schema schema; - List<Column> resolvedColumns = TUtil.newList(); - - while (iterator.hasNext()) { - relationOp = iterator.next(); - if (relationOp.isNameResolveBase()) { - schema = relationOp.getLogicalSchema(); - resolvedColumns.addAll(schema.getRootColumns()); - } - } - - if (resolvedColumns.size() == 0) { - throw new UndefinedColumnException(asteriskExpr.toString()); - } - - return resolvedColumns.toArray(new Column[resolvedColumns.size()]); - } - } - - /** - * Resolve an asterisk expression to the real column reference expressions. - * @param ctx context - * @param asteriskExpr asterisk expression - * @return a list of NamedExpr each of which has ColumnReferenceExprs as its child - * @throws TajoException - */ - private static List<NamedExpr> resolveAsterisk(LogicalPlanner.PlanContext ctx, QualifiedAsteriskExpr asteriskExpr) - throws TajoException { - Column[] columns = getColumns(ctx, asteriskExpr); - List<NamedExpr> newTargetExprs = new ArrayList<NamedExpr>(columns.length); - int i; - for (i = 0; i < columns.length; i++) { - newTargetExprs.add(new NamedExpr(new ColumnReferenceExpr(columns[i].getQualifier(), columns[i].getSimpleName()))); - } - return newTargetExprs; - } - - private static boolean hasAsterisk(NamedExpr [] namedExprs) { - for (NamedExpr eachTarget : namedExprs) { - if (eachTarget.getExpr().getType() == OpType.Asterisk) { - return true; - } - } - return false; - } - - private static NamedExpr [] voidResolveAsteriskNamedExpr(LogicalPlanner.PlanContext context, - NamedExpr [] namedExprs) throws TajoException { - List<NamedExpr> rewrittenTargets = TUtil.newList(); - for (NamedExpr originTarget : namedExprs) { - if (originTarget.getExpr().getType() == OpType.Asterisk) { - // rewrite targets - rewrittenTargets.addAll(resolveAsterisk(context, (QualifiedAsteriskExpr) originTarget.getExpr())); - } else { - rewrittenTargets.add(originTarget); - } - } - return rewrittenTargets.toArray(new NamedExpr[rewrittenTargets.size()]); - } - - @Override - public LogicalNode visitSetSession(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, SetSession expr) - throws TajoException { - SetSessionNode setSession = ctx.plan.createNode(SetSessionNode.class); - return setSession; - } - - @Override - public LogicalNode visitProjection(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Projection expr) - throws TajoException { - // If Non-from statement, it immediately returns. - if (!expr.hasChild()) { - EvalExprNode exprNode = ctx.plan.createNode(EvalExprNode.class); - exprNode.setTargets(buildTargets(ctx, expr.getNamedExprs())); - return exprNode; - } - - stack.push(expr); // <--- push - LogicalNode child = visit(ctx, stack, expr.getChild()); - - // Resolve the asterisk expression - if (hasAsterisk(expr.getNamedExprs())) { - expr.setNamedExprs(voidResolveAsteriskNamedExpr(ctx, expr.getNamedExprs())); - } - - NamedExpr[] projectTargetExprs = expr.getNamedExprs(); - for (int i = 0; i < expr.getNamedExprs().length; i++) { - NamedExpr namedExpr = projectTargetExprs[i]; - - // 1) Normalize all field names occurred in each expr into full qualified names - NameRefInSelectListNormalizer.normalize(ctx, namedExpr.getExpr()); - - // 2) Register explicit column aliases to block - if (namedExpr.getExpr().getType() == OpType.Column && namedExpr.hasAlias()) { - ctx.queryBlock.addColumnAlias(((ColumnReferenceExpr)namedExpr.getExpr()).getCanonicalName(), - namedExpr.getAlias()); - } else if (OpType.isLiteralType(namedExpr.getExpr().getType()) && namedExpr.hasAlias()) { - Expr constExpr = namedExpr.getExpr(); - ConstEval constEval = (ConstEval) annotator.createEvalNode(ctx, constExpr, NameResolvingMode.RELS_ONLY); - ctx.queryBlock.addConstReference(namedExpr.getAlias(), constExpr, constEval); - } - } - - Target[] targets = buildTargets(ctx, expr.getNamedExprs()); - - stack.pop(); // <--- Pop - - ProjectionNode projectionNode = ctx.plan.createNode(ProjectionNode.class); - projectionNode.setInSchema(child.getOutSchema()); - projectionNode.setOutSchema(PlannerUtil.targetToSchema(targets)); - - ctx.queryBlock.setSchema(projectionNode.getOutSchema()); - return projectionNode; - } - - private Target [] buildTargets(LogicalPlanner.PlanContext context, NamedExpr [] exprs) throws TajoException { - Target [] targets = new Target[exprs.length]; - for (int i = 0; i < exprs.length; i++) { - NamedExpr namedExpr = exprs[i]; - TajoDataTypes.DataType dataType = typeDeterminant.determineDataType(context, namedExpr.getExpr()); - - if (namedExpr.hasAlias()) { - targets[i] = new Target(new FieldEval(new Column(namedExpr.getAlias(), dataType))); - } else { - String generatedName = context.plan.generateUniqueColumnName(namedExpr.getExpr()); - targets[i] = new Target(new FieldEval(new Column(generatedName, dataType))); - } - } - return targets; - } - - @Override - public LogicalNode visitLimit(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Limit expr) - throws TajoException { - stack.push(expr); - LogicalNode child = visit(ctx, stack, expr.getChild()); - stack.pop(); - - LimitNode limitNode = ctx.plan.createNode(LimitNode.class); - limitNode.setInSchema(child.getOutSchema()); - limitNode.setOutSchema(child.getOutSchema()); - return limitNode; - } - - @Override - public LogicalNode visitSort(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Sort expr) throws TajoException { - stack.push(expr); - LogicalNode child = visit(ctx, stack, expr.getChild()); - stack.pop(); - - SortNode sortNode = ctx.plan.createNode(SortNode.class); - sortNode.setInSchema(child.getOutSchema()); - sortNode.setOutSchema(child.getOutSchema()); - return sortNode; - } - - @Override - public LogicalNode visitHaving(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Having expr) - throws TajoException { - stack.push(expr); - LogicalNode child = visit(ctx, stack, expr.getChild()); - stack.pop(); - - HavingNode havingNode = ctx.plan.createNode(HavingNode.class); - havingNode.setInSchema(child.getOutSchema()); - havingNode.setOutSchema(child.getOutSchema()); - return havingNode; - } - - @Override - public LogicalNode visitGroupBy(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Aggregation expr) - throws TajoException { - stack.push(expr); // <--- push - LogicalNode child = visit(ctx, stack, expr.getChild()); - - Projection projection = ctx.queryBlock.getSingletonExpr(OpType.Projection); - int finalTargetNum = projection.getNamedExprs().length; - Target [] targets = new Target[finalTargetNum]; - - if (hasAsterisk(projection.getNamedExprs())) { - projection.setNamedExprs(voidResolveAsteriskNamedExpr(ctx, projection.getNamedExprs())); - } - - for (int i = 0; i < finalTargetNum; i++) { - NamedExpr namedExpr = projection.getNamedExprs()[i]; - EvalNode evalNode = annotator.createEvalNode(ctx, namedExpr.getExpr(), NameResolvingMode.SUBEXPRS_AND_RELS); - - if (namedExpr.hasAlias()) { - targets[i] = new Target(evalNode, namedExpr.getAlias()); - } else { - targets[i] = new Target(evalNode, "?name_" + i); - } - } - stack.pop(); - - GroupbyNode groupByNode = ctx.plan.createNode(GroupbyNode.class); - groupByNode.setInSchema(child.getOutSchema()); - groupByNode.setOutSchema(PlannerUtil.targetToSchema(targets)); - return groupByNode; - } - - @Override - public LogicalNode visitUnion(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, SetOperation expr) - throws TajoException { - LogicalPlan.QueryBlock leftBlock = ctx.plan.newQueryBlock(); - LogicalPlanner.PlanContext leftContext = new LogicalPlanner.PlanContext(ctx, leftBlock); - LogicalNode leftChild = visit(leftContext, new Stack<Expr>(), expr.getLeft()); - leftBlock.setRoot(leftChild); - ctx.queryBlock.registerExprWithNode(expr.getLeft(), leftChild); - - LogicalPlan.QueryBlock rightBlock = ctx.plan.newQueryBlock(); - LogicalPlanner.PlanContext rightContext = new LogicalPlanner.PlanContext(ctx, rightBlock); - LogicalNode rightChild = visit(rightContext, new Stack<Expr>(), expr.getRight()); - rightBlock.setRoot(rightChild); - ctx.queryBlock.registerExprWithNode(expr.getRight(), rightChild); - - UnionNode unionNode = new UnionNode(ctx.plan.newPID()); - unionNode.setLeftChild(leftChild); - unionNode.setRightChild(rightChild); - unionNode.setInSchema(leftChild.getOutSchema()); - unionNode.setOutSchema(leftChild.getOutSchema()); - unionNode.setDistinct(expr.isDistinct()); - - return unionNode; - } - - public LogicalNode visitFilter(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Selection expr) - throws TajoException { - stack.push(expr); - // Since filter push down will be done later, it is guaranteed that in-subqueries are found at only selection. - for (Expr eachQual : PlannerUtil.extractInSubquery(expr.getQual())) { - InPredicate inPredicate = (InPredicate) eachQual; - stack.push(inPredicate); - visit(ctx, stack, inPredicate.getRight()); - stack.pop(); - } - LogicalNode child = visit(ctx, stack, expr.getChild()); - stack.pop(); - - SelectionNode selectionNode = ctx.plan.createNode(SelectionNode.class); - selectionNode.setInSchema(child.getOutSchema()); - selectionNode.setOutSchema(child.getOutSchema()); - return selectionNode; - } - - @Override - public LogicalNode visitJoin(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Join expr) throws TajoException { - stack.push(expr); - LogicalNode left = visit(ctx, stack, expr.getLeft()); - LogicalNode right = visit(ctx, stack, expr.getRight()); - stack.pop(); - JoinNode joinNode = ctx.plan.createNode(JoinNode.class); - joinNode.setJoinType(expr.getJoinType()); - Schema merged = SchemaUtil.merge(left.getOutSchema(), right.getOutSchema()); - joinNode.setInSchema(merged); - joinNode.setOutSchema(merged); - - ctx.queryBlock.addJoinType(expr.getJoinType()); - return joinNode; - } - - @Override - public LogicalNode visitRelation(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Relation expr) - throws TajoException { - Relation relation = expr; - - String actualRelationName; - if (CatalogUtil.isFQTableName(expr.getName())) { - actualRelationName = relation.getName(); - } else { - actualRelationName = - CatalogUtil.buildFQName(ctx.queryContext.get(SessionVars.CURRENT_DATABASE), relation.getName()); - } - - TableDesc desc = catalog.getTableDesc(actualRelationName); - - ScanNode scanNode = ctx.plan.createNode(ScanNode.class); - if (relation.hasAlias()) { - scanNode.init(desc, relation.getAlias()); - } else { - scanNode.init(desc); - } - - TablePropertyUtil.setTableProperty(ctx.getQueryContext(), scanNode); - - ctx.queryBlock.addRelation(scanNode); - return scanNode; - } - - @Override - public LogicalNode visitTableSubQuery(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, TablePrimarySubQuery expr) - throws TajoException { - - LogicalPlanner.PlanContext newContext; - // Note: TableSubQuery always has a table name. - // SELECT .... FROM (SELECT ...) TB_NAME <- - QueryBlock queryBlock = ctx.plan.newQueryBlock(); - newContext = new LogicalPlanner.PlanContext(ctx, queryBlock); - LogicalNode child = super.visitTableSubQuery(newContext, stack, expr); - queryBlock.setRoot(child); - - // a table subquery should be dealt as a relation. - TableSubQueryNode node = ctx.plan.createNode(TableSubQueryNode.class); - node.init(CatalogUtil.buildFQName(ctx.queryContext.get(SessionVars.CURRENT_DATABASE), expr.getName()), child); - ctx.queryBlock.addRelation(node); - - return node; - } - - @Override - public LogicalNode visitSimpleTableSubquery(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, SimpleTableSubquery expr) - throws TajoException { - LogicalPlanner.PlanContext newContext; - // Note: TableSubQuery always has a table name. - // SELECT .... FROM (SELECT ...) TB_NAME <- - QueryBlock queryBlock = ctx.plan.newQueryBlock(); - newContext = new LogicalPlanner.PlanContext(ctx, queryBlock); - LogicalNode child = super.visitSimpleTableSubquery(newContext, stack, expr); - queryBlock.setRoot(child); - - // a table subquery should be dealt as a relation. - TableSubQueryNode node = ctx.plan.createNode(TableSubQueryNode.class); - node.init(CatalogUtil.buildFQName(ctx.queryContext.get(SessionVars.CURRENT_DATABASE), - ctx.generateUniqueSubQueryName()), child); - ctx.queryBlock.addRelation(node); - if (stack.peek().getType() == OpType.InPredicate) { - // In-subquery and scalar subquery cannot be the base for name resolution. - node.setNameResolveBase(false); - } - return node; - } - - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - // Data Definition Language Section - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - - @Override - public LogicalNode visitCreateDatabase(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, CreateDatabase expr) - throws TajoException { - CreateDatabaseNode createDatabaseNode = ctx.plan.createNode(CreateDatabaseNode.class); - return createDatabaseNode; - } - - @Override - public LogicalNode visitDropDatabase(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, DropDatabase expr) - throws TajoException { - DropDatabaseNode dropDatabaseNode = ctx.plan.createNode(DropDatabaseNode.class); - return dropDatabaseNode; - } - - @Override - public LogicalNode visitCreateTable(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, CreateTable expr) - throws TajoException { - - CreateTableNode createTableNode = ctx.plan.createNode(CreateTableNode.class); - - if (expr.hasSubQuery()) { - stack.push(expr); - visit(ctx, stack, expr.getSubQuery()); - stack.pop(); - } - - return createTableNode; - } - - @Override - public LogicalNode visitDropTable(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, DropTable expr) - throws TajoException { - DropTableNode dropTable = ctx.plan.createNode(DropTableNode.class); - return dropTable; - } - - @Override - public LogicalNode visitAlterTablespace(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, AlterTablespace expr) - throws TajoException { - AlterTablespaceNode alterTablespace = ctx.plan.createNode(AlterTablespaceNode.class); - return alterTablespace; - } - - @Override - public LogicalNode visitAlterTable(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, AlterTable expr) - throws TajoException { - AlterTableNode alterTableNode = ctx.plan.createNode(AlterTableNode.class); - return alterTableNode; - } - - @Override - public LogicalNode visitCreateIndex(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, CreateIndex expr) - throws TajoException { - stack.push(expr); - LogicalNode child = visit(ctx, stack, expr.getChild()); - stack.pop(); - - CreateIndexNode createIndex = ctx.plan.createNode(CreateIndexNode.class); - createIndex.setInSchema(child.getOutSchema()); - createIndex.setOutSchema(child.getOutSchema()); - return createIndex; - } - - @Override - public LogicalNode visitDropIndex(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, DropIndex expr) { - return ctx.plan.createNode(DropIndexNode.class); - } - - public LogicalNode visitTruncateTable(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, TruncateTable expr) - throws TajoException { - TruncateTableNode truncateTableNode = ctx.plan.createNode(TruncateTableNode.class); - return truncateTableNode; - } - - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - // Insert or Update Section - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - - public LogicalNode visitInsert(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Insert expr) - throws TajoException { - LogicalNode child = super.visitInsert(ctx, stack, expr); - - InsertNode insertNode = new InsertNode(ctx.plan.newPID()); - insertNode.setInSchema(child.getOutSchema()); - insertNode.setOutSchema(child.getOutSchema()); - return insertNode; + provider = new BaseLogicalPlanPreprocessPhaseProvider(); + engine = new BaseLogicalPlanPreprocessEngine(catalog, annotator); + engine.addProcessPhase(provider.getPhases()); } - static class NameRefInSelectListNormalizer extends SimpleAlgebraVisitor<LogicalPlanner.PlanContext, Object> { - private static final NameRefInSelectListNormalizer instance; - - static { - instance = new NameRefInSelectListNormalizer(); - } - - public static void normalize(LogicalPlanner.PlanContext context, Expr expr) throws TajoException { - NameRefInSelectListNormalizer normalizer = new NameRefInSelectListNormalizer(); - normalizer.visit(context,new Stack<Expr>(), expr); - } - - @Override - public Expr visitColumnReference(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, ColumnReferenceExpr expr) - throws TajoException { - - String normalized = NameResolver.resolve(ctx.plan, ctx.queryBlock, expr, - NameResolvingMode.RELS_ONLY).getQualifiedName(); - expr.setName(normalized); - - return expr; - } + public LogicalNode process(PlanContext context, Expr expr) throws TajoException { + return engine.process(context, expr); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 24dcfd5..4e74862 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -126,6 +126,10 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex return queryBlock; } + public LogicalPlan getPlan() { + return plan; + } + public OverridableConf getQueryContext() { return queryContext; } @@ -160,7 +164,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex QueryBlock rootBlock = plan.newAndGetBlock(LogicalPlan.ROOT_BLOCK); PlanContext context = new PlanContext(queryContext, plan, rootBlock, evalOptimizer, debug); - preprocessor.visit(context, new Stack<Expr>(), expr); + preprocessor.process(context, expr); plan.resetGeneratedId(); LogicalNode topMostNode = this.visit(context, new Stack<Expr>(), expr); http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessEngine.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessEngine.java new file mode 100644 index 0000000..f02faef --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessEngine.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.algebra.Expr; +import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.plan.ExprAnnotator; +import org.apache.tajo.plan.LogicalPlanner.PlanContext; +import org.apache.tajo.plan.logical.LogicalNode; + +import java.lang.reflect.Constructor; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; + +public class BaseLogicalPlanPreprocessEngine implements LogicalPlanPreprocessEngine { + private final CatalogService catalogService; + private final ExprAnnotator exprAnnotator; + + public BaseLogicalPlanPreprocessEngine(CatalogService catalogService, ExprAnnotator exprAnnotator) { + this.catalogService = catalogService; + this.exprAnnotator = exprAnnotator; + } + + /** class logger */ + private Log LOG = LogFactory.getLog(BaseLogicalPlanPreprocessEngine.class); + + /** a map for pre-process phases */ + private Map<String, LogicalPlanPreprocessPhase> preprocessPhases = new LinkedHashMap<>(); + + /** + * Add a pre-process phase to this engine. + * + * @param phases pre-process phase + */ + public void addProcessPhase(Iterable<Class<? extends LogicalPlanPreprocessPhase>> phases) { + for (Class<? extends LogicalPlanPreprocessPhase> clazz : phases) { + try { + Constructor cons = clazz.getConstructor(CatalogService.class, ExprAnnotator.class); + LogicalPlanPreprocessPhase rule = (LogicalPlanPreprocessPhase) cons.newInstance(catalogService, exprAnnotator); + addProcessPhase(rule); + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + } + + /** + * Add a pre-process phase to this engine. + * + * @param phase The pre-process phase to be added to this engine. + */ + public void addProcessPhase(LogicalPlanPreprocessPhase phase) { + if (!preprocessPhases.containsKey(phase.getName())) { + preprocessPhases.put(phase.getName(), phase); + } + } + + /** + * Do every pre-process phase added to this engine. + * + * @param context + * @return The rewritten logical node. + */ + @Override + public LogicalNode process(PlanContext context, Expr expr) throws TajoException { + LogicalPlanPreprocessPhase rule; + LogicalNode node = null; + for (Entry<String, LogicalPlanPreprocessPhase> preprocessPhase : preprocessPhases.entrySet()) { + rule = preprocessPhase.getValue(); + if (rule.isEligible(context, expr)) { + if (LOG.isDebugEnabled()) { + LOG.debug("The rule \"" + rule.getName() + " \" rewrites the query."); + } + node = rule.process(context, expr); + } + } + + return node; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessPhaseProvider.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessPhaseProvider.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessPhaseProvider.java new file mode 100644 index 0000000..92af709 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessPhaseProvider.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite; + +import org.apache.tajo.util.TUtil; + +import java.util.Collection; +import java.util.List; + +public class BaseLogicalPlanPreprocessPhaseProvider extends LogicalPlanPreprocessPhaseProvider { + @Override + public Collection<Class<? extends LogicalPlanPreprocessPhase>> getPhases() { + List phases = TUtil.newList( + BaseSchemaBuildPhase.class + ); + return phases; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java index 9a541d9..35ec85c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java @@ -37,7 +37,7 @@ public class BaseLogicalPlanRewriteEngine implements LogicalPlanRewriteEngine { private Log LOG = LogFactory.getLog(BaseLogicalPlanRewriteEngine.class); /** a map for query rewrite rules */ - private Map<String, LogicalPlanRewriteRule> rewriteRules = new LinkedHashMap<String, LogicalPlanRewriteRule>(); + private Map<String, LogicalPlanRewriteRule> rewriteRules = new LinkedHashMap<>(); /** * Add a query rewrite rule to this engine. http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java new file mode 100644 index 0000000..81e9b2c --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java @@ -0,0 +1,591 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite; + +import org.apache.tajo.SessionVars; +import org.apache.tajo.algebra.*; +import org.apache.tajo.catalog.*; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.UndefinedColumnException; +import org.apache.tajo.plan.*; +import org.apache.tajo.plan.LogicalPlan.QueryBlock; +import org.apache.tajo.plan.LogicalPlanner.PlanContext; +import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; +import org.apache.tajo.plan.expr.ConstEval; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.FieldEval; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.nameresolver.NameResolver; +import org.apache.tajo.plan.nameresolver.NameResolvingMode; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; +import org.apache.tajo.util.TUtil; + +import java.util.*; + +public class BaseSchemaBuildPhase extends LogicalPlanPreprocessPhase { + + private final Processor processor; + + public BaseSchemaBuildPhase(CatalogService catalog, ExprAnnotator annotator) { + super(catalog, annotator); + processor = new Processor(catalog, annotator); + } + + @Override + public String getName() { + return "Base schema build phase"; + } + + @Override + public boolean isEligible(PlanContext context, Expr expr) { + return true; + } + + @Override + public LogicalNode process(PlanContext context, Expr expr) throws TajoException { + return processor.visit(context, new Stack<Expr>(), expr); + } + + static class Processor extends BaseAlgebraVisitor<PlanContext, LogicalNode> { + private TypeDeterminant typeDeterminant; + private ExprAnnotator annotator; + + /** Catalog service */ + private CatalogService catalog; + + Processor(CatalogService catalog, ExprAnnotator annotator) { + this.catalog = catalog; + this.annotator = annotator; + this.typeDeterminant = new TypeDeterminant(catalog); + } + + @Override + public void preHook(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Expr expr) throws TajoException { + ctx.getQueryBlock().setAlgebraicExpr(expr); + ctx.getPlan().mapExprToBlock(expr, ctx.getQueryBlock().getName()); + } + + @Override + public LogicalNode postHook(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Expr expr, LogicalNode result) + throws TajoException { + // If non-from statement, result can be null. It avoids that case. + if (result != null) { + // setNode method registers each node to corresponding block and plan. + ctx.getQueryBlock().registerNode(result); + // It makes a map between an expr and a logical node. + ctx.getQueryBlock().registerExprWithNode(expr, result); + } + return result; + } + + /** + * Get all columns of the relations correspondent to the asterisk expression. + * @param ctx + * @param asteriskExpr + * @return array of columns + * @throws TajoException + */ + public static Column[] getColumns(LogicalPlanner.PlanContext ctx, QualifiedAsteriskExpr asteriskExpr) + throws TajoException { + RelationNode relationOp = null; + QueryBlock block = ctx.getQueryBlock(); + Collection<QueryBlock> queryBlocks = ctx.getPlan().getQueryBlocks(); + if (asteriskExpr.hasQualifier()) { + String qualifier; + + if (CatalogUtil.isFQTableName(asteriskExpr.getQualifier())) { + qualifier = asteriskExpr.getQualifier(); + } else { + qualifier = CatalogUtil.buildFQName( + ctx.getQueryContext().get(SessionVars.CURRENT_DATABASE), asteriskExpr.getQualifier()); + } + + relationOp = block.getRelation(qualifier); + + // if a column name is outside of this query block + if (relationOp == null) { + // TODO - nested query can only refer outer query block? or not? + for (QueryBlock eachBlock : queryBlocks) { + if (eachBlock.existsRelation(qualifier)) { + relationOp = eachBlock.getRelation(qualifier); + } + } + } + + // If we cannot find any relation against a qualified column name + if (relationOp == null) { + throw new UndefinedColumnException(CatalogUtil.buildFQName(qualifier, "*")); + } + + Schema schema = relationOp.getLogicalSchema(); + Column[] resolvedColumns = new Column[schema.size()]; + return schema.getRootColumns().toArray(resolvedColumns); + } else { // if a column reference is not qualified + // columns of every relation should be resolved. + Iterator<RelationNode> iterator = block.getRelations().iterator(); + Schema schema; + List<Column> resolvedColumns = TUtil.newList(); + + while (iterator.hasNext()) { + relationOp = iterator.next(); + if (relationOp.isNameResolveBase()) { + schema = relationOp.getLogicalSchema(); + resolvedColumns.addAll(schema.getRootColumns()); + } + } + + if (resolvedColumns.size() == 0) { + throw new UndefinedColumnException(asteriskExpr.toString()); + } + + return resolvedColumns.toArray(new Column[resolvedColumns.size()]); + } + } + + /** + * Resolve an asterisk expression to the real column reference expressions. + * @param ctx context + * @param asteriskExpr asterisk expression + * @return a list of NamedExpr each of which has ColumnReferenceExprs as its child + * @throws TajoException + */ + private static List<NamedExpr> resolveAsterisk(LogicalPlanner.PlanContext ctx, QualifiedAsteriskExpr asteriskExpr) + throws TajoException { + Column[] columns = getColumns(ctx, asteriskExpr); + List<NamedExpr> newTargetExprs = new ArrayList<NamedExpr>(columns.length); + int i; + for (i = 0; i < columns.length; i++) { + newTargetExprs.add(new NamedExpr(new ColumnReferenceExpr(columns[i].getQualifier(), columns[i].getSimpleName()))); + } + return newTargetExprs; + } + + private static boolean hasAsterisk(NamedExpr [] namedExprs) { + for (NamedExpr eachTarget : namedExprs) { + if (eachTarget.getExpr().getType() == OpType.Asterisk) { + return true; + } + } + return false; + } + + private static NamedExpr [] voidResolveAsteriskNamedExpr(LogicalPlanner.PlanContext context, + NamedExpr [] namedExprs) throws TajoException { + List<NamedExpr> rewrittenTargets = TUtil.newList(); + for (NamedExpr originTarget : namedExprs) { + if (originTarget.getExpr().getType() == OpType.Asterisk) { + // rewrite targets + rewrittenTargets.addAll(resolveAsterisk(context, (QualifiedAsteriskExpr) originTarget.getExpr())); + } else { + rewrittenTargets.add(originTarget); + } + } + return rewrittenTargets.toArray(new NamedExpr[rewrittenTargets.size()]); + } + + @Override + public LogicalNode visitSetSession(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, SetSession expr) + throws TajoException { + SetSessionNode setSession = ctx.getPlan().createNode(SetSessionNode.class); + return setSession; + } + + @Override + public LogicalNode visitProjection(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Projection expr) + throws TajoException { + // If Non-from statement, it immediately returns. + if (!expr.hasChild()) { + EvalExprNode exprNode = ctx.getPlan().createNode(EvalExprNode.class); + exprNode.setTargets(buildTargets(ctx, expr.getNamedExprs())); + return exprNode; + } + + stack.push(expr); // <--- push + LogicalNode child = visit(ctx, stack, expr.getChild()); + + // Resolve the asterisk expression + if (hasAsterisk(expr.getNamedExprs())) { + expr.setNamedExprs(voidResolveAsteriskNamedExpr(ctx, expr.getNamedExprs())); + } + + NamedExpr[] projectTargetExprs = expr.getNamedExprs(); + for (int i = 0; i < expr.getNamedExprs().length; i++) { + NamedExpr namedExpr = projectTargetExprs[i]; + + // 1) Normalize all field names occurred in each expr into full qualified names + NameRefInSelectListNormalizer.normalize(ctx, namedExpr.getExpr()); + + // 2) Register explicit column aliases to block + if (namedExpr.getExpr().getType() == OpType.Column && namedExpr.hasAlias()) { + ctx.getQueryBlock().addColumnAlias(((ColumnReferenceExpr)namedExpr.getExpr()).getCanonicalName(), + namedExpr.getAlias()); + } else if (OpType.isLiteralType(namedExpr.getExpr().getType()) && namedExpr.hasAlias()) { + Expr constExpr = namedExpr.getExpr(); + ConstEval constEval = (ConstEval) annotator.createEvalNode(ctx, constExpr, NameResolvingMode.RELS_ONLY); + ctx.getQueryBlock().addConstReference(namedExpr.getAlias(), constExpr, constEval); + } + } + + Target[] targets = buildTargets(ctx, expr.getNamedExprs()); + + stack.pop(); // <--- Pop + + ProjectionNode projectionNode = ctx.getPlan().createNode(ProjectionNode.class); + projectionNode.setInSchema(child.getOutSchema()); + projectionNode.setOutSchema(PlannerUtil.targetToSchema(targets)); + + ctx.getQueryBlock().setSchema(projectionNode.getOutSchema()); + return projectionNode; + } + + private Target [] buildTargets(LogicalPlanner.PlanContext context, NamedExpr [] exprs) throws TajoException { + Target [] targets = new Target[exprs.length]; + for (int i = 0; i < exprs.length; i++) { + NamedExpr namedExpr = exprs[i]; + TajoDataTypes.DataType dataType = typeDeterminant.determineDataType(context, namedExpr.getExpr()); + + if (namedExpr.hasAlias()) { + targets[i] = new Target(new FieldEval(new Column(namedExpr.getAlias(), dataType))); + } else { + String generatedName = context.getPlan().generateUniqueColumnName(namedExpr.getExpr()); + targets[i] = new Target(new FieldEval(new Column(generatedName, dataType))); + } + } + return targets; + } + + @Override + public LogicalNode visitLimit(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Limit expr) + throws TajoException { + stack.push(expr); + LogicalNode child = visit(ctx, stack, expr.getChild()); + stack.pop(); + + LimitNode limitNode = ctx.getPlan().createNode(LimitNode.class); + limitNode.setInSchema(child.getOutSchema()); + limitNode.setOutSchema(child.getOutSchema()); + return limitNode; + } + + @Override + public LogicalNode visitSort(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Sort expr) throws TajoException { + stack.push(expr); + LogicalNode child = visit(ctx, stack, expr.getChild()); + stack.pop(); + + SortNode sortNode = ctx.getPlan().createNode(SortNode.class); + sortNode.setInSchema(child.getOutSchema()); + sortNode.setOutSchema(child.getOutSchema()); + return sortNode; + } + + @Override + public LogicalNode visitHaving(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Having expr) + throws TajoException { + stack.push(expr); + LogicalNode child = visit(ctx, stack, expr.getChild()); + stack.pop(); + + HavingNode havingNode = ctx.getPlan().createNode(HavingNode.class); + havingNode.setInSchema(child.getOutSchema()); + havingNode.setOutSchema(child.getOutSchema()); + return havingNode; + } + + @Override + public LogicalNode visitGroupBy(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Aggregation expr) + throws TajoException { + stack.push(expr); // <--- push + LogicalNode child = visit(ctx, stack, expr.getChild()); + + Projection projection = ctx.getQueryBlock().getSingletonExpr(OpType.Projection); + int finalTargetNum = projection.getNamedExprs().length; + Target [] targets = new Target[finalTargetNum]; + + if (hasAsterisk(projection.getNamedExprs())) { + projection.setNamedExprs(voidResolveAsteriskNamedExpr(ctx, projection.getNamedExprs())); + } + + for (int i = 0; i < finalTargetNum; i++) { + NamedExpr namedExpr = projection.getNamedExprs()[i]; + EvalNode evalNode = annotator.createEvalNode(ctx, namedExpr.getExpr(), NameResolvingMode.SUBEXPRS_AND_RELS); + + if (namedExpr.hasAlias()) { + targets[i] = new Target(evalNode, namedExpr.getAlias()); + } else { + targets[i] = new Target(evalNode, "?name_" + i); + } + } + stack.pop(); + + GroupbyNode groupByNode = ctx.getPlan().createNode(GroupbyNode.class); + groupByNode.setInSchema(child.getOutSchema()); + groupByNode.setOutSchema(PlannerUtil.targetToSchema(targets)); + return groupByNode; + } + + @Override + public LogicalNode visitUnion(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, SetOperation expr) + throws TajoException { + LogicalPlan.QueryBlock leftBlock = ctx.getPlan().newQueryBlock(); + LogicalPlanner.PlanContext leftContext = new LogicalPlanner.PlanContext(ctx, leftBlock); + LogicalNode leftChild = visit(leftContext, new Stack<Expr>(), expr.getLeft()); + leftBlock.setRoot(leftChild); + ctx.getQueryBlock().registerExprWithNode(expr.getLeft(), leftChild); + + LogicalPlan.QueryBlock rightBlock = ctx.getPlan().newQueryBlock(); + LogicalPlanner.PlanContext rightContext = new LogicalPlanner.PlanContext(ctx, rightBlock); + LogicalNode rightChild = visit(rightContext, new Stack<Expr>(), expr.getRight()); + rightBlock.setRoot(rightChild); + ctx.getQueryBlock().registerExprWithNode(expr.getRight(), rightChild); + + UnionNode unionNode = new UnionNode(ctx.getPlan().newPID()); + unionNode.setLeftChild(leftChild); + unionNode.setRightChild(rightChild); + unionNode.setInSchema(leftChild.getOutSchema()); + unionNode.setOutSchema(leftChild.getOutSchema()); + unionNode.setDistinct(expr.isDistinct()); + + return unionNode; + } + + public LogicalNode visitFilter(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Selection expr) + throws TajoException { + stack.push(expr); + // Since filter push down will be done later, it is guaranteed that in-subqueries are found at only selection. + for (Expr eachQual : PlannerUtil.extractInSubquery(expr.getQual())) { + InPredicate inPredicate = (InPredicate) eachQual; + stack.push(inPredicate); + visit(ctx, stack, inPredicate.getRight()); + stack.pop(); + } + LogicalNode child = visit(ctx, stack, expr.getChild()); + stack.pop(); + + SelectionNode selectionNode = ctx.getPlan().createNode(SelectionNode.class); + selectionNode.setInSchema(child.getOutSchema()); + selectionNode.setOutSchema(child.getOutSchema()); + return selectionNode; + } + + @Override + public LogicalNode visitJoin(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Join expr) throws TajoException { + stack.push(expr); + LogicalNode left = visit(ctx, stack, expr.getLeft()); + LogicalNode right = visit(ctx, stack, expr.getRight()); + stack.pop(); + JoinNode joinNode = ctx.getPlan().createNode(JoinNode.class); + joinNode.setJoinType(expr.getJoinType()); + Schema merged = SchemaUtil.merge(left.getOutSchema(), right.getOutSchema()); + joinNode.setInSchema(merged); + joinNode.setOutSchema(merged); + + ctx.getQueryBlock().addJoinType(expr.getJoinType()); + return joinNode; + } + + @Override + public LogicalNode visitRelation(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Relation relation) + throws TajoException { + + String actualRelationName; + if (CatalogUtil.isFQTableName(relation.getName())) { + actualRelationName = relation.getName(); + } else { + actualRelationName = + CatalogUtil.buildFQName(ctx.getQueryContext().get(SessionVars.CURRENT_DATABASE), relation.getName()); + } + + TableDesc desc = catalog.getTableDesc(actualRelationName); + + ScanNode scanNode = ctx.getPlan().createNode(ScanNode.class); + if (relation.hasAlias()) { + scanNode.init(desc, relation.getAlias()); + } else { + scanNode.init(desc); + } + + TablePropertyUtil.setTableProperty(ctx.getQueryContext(), scanNode); + + ctx.getQueryBlock().addRelation(scanNode); + return scanNode; + } + + @Override + public LogicalNode visitTableSubQuery(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, TablePrimarySubQuery expr) + throws TajoException { + + LogicalPlanner.PlanContext newContext; + // Note: TableSubQuery always has a table name. + // SELECT .... FROM (SELECT ...) TB_NAME <- + QueryBlock queryBlock = ctx.getPlan().newQueryBlock(); + newContext = new LogicalPlanner.PlanContext(ctx, queryBlock); + LogicalNode child = super.visitTableSubQuery(newContext, stack, expr); + queryBlock.setRoot(child); + + // a table subquery should be dealt as a relation. + TableSubQueryNode node = ctx.getPlan().createNode(TableSubQueryNode.class); + node.init(CatalogUtil.buildFQName(ctx.getQueryContext().get(SessionVars.CURRENT_DATABASE), expr.getName()), child); + ctx.getQueryBlock().addRelation(node); + + return node; + } + + @Override + public LogicalNode visitSimpleTableSubquery(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, SimpleTableSubquery expr) + throws TajoException { + LogicalPlanner.PlanContext newContext; + // Note: TableSubQuery always has a table name. + // SELECT .... FROM (SELECT ...) TB_NAME <- + QueryBlock queryBlock = ctx.getPlan().newQueryBlock(); + newContext = new LogicalPlanner.PlanContext(ctx, queryBlock); + LogicalNode child = super.visitSimpleTableSubquery(newContext, stack, expr); + queryBlock.setRoot(child); + + // a table subquery should be dealt as a relation. + TableSubQueryNode node = ctx.getPlan().createNode(TableSubQueryNode.class); + node.init(CatalogUtil.buildFQName(ctx.getQueryContext().get(SessionVars.CURRENT_DATABASE), + ctx.generateUniqueSubQueryName()), child); + ctx.getQueryBlock().addRelation(node); + if (stack.peek().getType() == OpType.InPredicate) { + // In-subquery and scalar subquery cannot be the base for name resolution. + node.setNameResolveBase(false); + } + return node; + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Data Definition Language Section + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public LogicalNode visitCreateDatabase(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, CreateDatabase expr) + throws TajoException { + CreateDatabaseNode createDatabaseNode = ctx.getPlan().createNode(CreateDatabaseNode.class); + return createDatabaseNode; + } + + @Override + public LogicalNode visitDropDatabase(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, DropDatabase expr) + throws TajoException { + DropDatabaseNode dropDatabaseNode = ctx.getPlan().createNode(DropDatabaseNode.class); + return dropDatabaseNode; + } + + @Override + public LogicalNode visitCreateTable(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, CreateTable expr) + throws TajoException { + + CreateTableNode createTableNode = ctx.getPlan().createNode(CreateTableNode.class); + + if (expr.hasSubQuery()) { + stack.push(expr); + visit(ctx, stack, expr.getSubQuery()); + stack.pop(); + } + + return createTableNode; + } + + @Override + public LogicalNode visitDropTable(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, DropTable expr) + throws TajoException { + DropTableNode dropTable = ctx.getPlan().createNode(DropTableNode.class); + return dropTable; + } + + @Override + public LogicalNode visitAlterTablespace(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, AlterTablespace expr) + throws TajoException { + AlterTablespaceNode alterTablespace = ctx.getPlan().createNode(AlterTablespaceNode.class); + return alterTablespace; + } + + @Override + public LogicalNode visitAlterTable(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, AlterTable expr) + throws TajoException { + AlterTableNode alterTableNode = ctx.getPlan().createNode(AlterTableNode.class); + return alterTableNode; + } + + @Override + public LogicalNode visitCreateIndex(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, CreateIndex expr) + throws TajoException { + stack.push(expr); + LogicalNode child = visit(ctx, stack, expr.getChild()); + stack.pop(); + + CreateIndexNode createIndex = ctx.getPlan().createNode(CreateIndexNode.class); + createIndex.setInSchema(child.getOutSchema()); + createIndex.setOutSchema(child.getOutSchema()); + return createIndex; + } + + @Override + public LogicalNode visitDropIndex(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, DropIndex expr) { + return ctx.getPlan().createNode(DropIndexNode.class); + } + + public LogicalNode visitTruncateTable(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, TruncateTable expr) + throws TajoException { + TruncateTableNode truncateTableNode = ctx.getPlan().createNode(TruncateTableNode.class); + return truncateTableNode; + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Insert or Update Section + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + + public LogicalNode visitInsert(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Insert expr) + throws TajoException { + LogicalNode child = super.visitInsert(ctx, stack, expr); + + InsertNode insertNode = new InsertNode(ctx.getPlan().newPID()); + insertNode.setInSchema(child.getOutSchema()); + insertNode.setOutSchema(child.getOutSchema()); + return insertNode; + } + + static class NameRefInSelectListNormalizer extends SimpleAlgebraVisitor<PlanContext, Object> { + private static final NameRefInSelectListNormalizer instance; + + static { + instance = new NameRefInSelectListNormalizer(); + } + + public static void normalize(LogicalPlanner.PlanContext context, Expr expr) throws TajoException { + NameRefInSelectListNormalizer normalizer = new NameRefInSelectListNormalizer(); + normalizer.visit(context,new Stack<Expr>(), expr); + } + + @Override + public Expr visitColumnReference(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, ColumnReferenceExpr expr) + throws TajoException { + + String normalized = NameResolver.resolve(ctx.getPlan(), ctx.getQueryBlock(), expr, + NameResolvingMode.RELS_ONLY).getQualifiedName(); + expr.setName(normalized); + + return expr; + } + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessEngine.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessEngine.java new file mode 100644 index 0000000..f5cf7ab --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessEngine.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite; + +import org.apache.tajo.algebra.Expr; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.plan.LogicalPlanner.PlanContext; +import org.apache.tajo.plan.logical.LogicalNode; + +public interface LogicalPlanPreprocessEngine { + LogicalNode process(PlanContext planContext, Expr expr) throws TajoException; +} http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhase.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhase.java new file mode 100644 index 0000000..21f8b5c --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhase.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite; + +import org.apache.tajo.algebra.Expr; +import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.plan.ExprAnnotator; +import org.apache.tajo.plan.LogicalPlanner.PlanContext; +import org.apache.tajo.plan.logical.LogicalNode; + +public abstract class LogicalPlanPreprocessPhase { + + protected final CatalogService catalog; + protected final ExprAnnotator annotator; + + public LogicalPlanPreprocessPhase(CatalogService catalog, ExprAnnotator annotator) { + this.catalog = catalog; + this.annotator = annotator; + } + /** + * It returns the pre-process phase name. It will be used for debugging and + * building a optimization history. + * + * @return The pre-process phase name + */ + public abstract String getName(); + + /** + * This method checks if this pre-process phase can be applied to the given expression tree. + * + * @param context + * @param expr + * @return + */ + public abstract boolean isEligible(PlanContext context, Expr expr); + + /** + * Do a pre-process phase for an expression tree and returns it. + * It must be guaranteed that the input expression tree is not modified even after rewrite. + * In other words, the rewrite has to modify a copy of the expression tree. + * + * @param context + * @param expr + * @return The rewritten logical plan. + */ + public abstract LogicalNode process(PlanContext context, Expr expr) throws TajoException; +} http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhaseProvider.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhaseProvider.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhaseProvider.java new file mode 100644 index 0000000..a59f1d8 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhaseProvider.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite; + +import java.util.Collection; + +public abstract class LogicalPlanPreprocessPhaseProvider { + /** + * It returns LogicalPlanPreprocessPhase classes. + * + * @return LogicalPlanPreprocessPhase classes + */ + public abstract Collection<Class<? extends LogicalPlanPreprocessPhase>> getPhases(); +} http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java index a621921..f70dfe0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java @@ -25,7 +25,7 @@ public interface LogicalPlanRewriteEngine { /** * Rewrite a logical plan with all query rewrite rules added to this engine. * - * @param plan The plan to be rewritten with all query rewrite rule. + * @param context The context containing query context, logical plan, and catalog. * @return The rewritten plan. */ LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws TajoException; http://git-wip-us.apache.org/repos/asf/tajo/blob/357b7be8/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java index 6643d28..9fbc7af 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java @@ -39,7 +39,7 @@ public interface LogicalPlanRewriteRule { * For example, the selection push down can not be applied to the query plan without any filter. * In such case, it will return false. * - * @param plan The plan to be checked + * @param context rewrite rule context. * @return True if this rule can be applied to a given plan. Otherwise, false. */ boolean isEligible(LogicalPlanRewriteRuleContext context); @@ -49,7 +49,7 @@ public interface LogicalPlanRewriteRule { * It must be guaranteed that the input logical plan is not modified even after rewrite. * In other words, the rewrite has to modify an plan copied from the input plan. * - * @param plan Input logical plan. It will not be modified. + * @param context rewrite rule context. * @return The rewritten logical plan. */ LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws TajoException;
