Repository: tajo Updated Branches: refs/heads/index_support a5d14f808 -> 31d524c51
TAJO-836: create index support. (jihoon) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/31d524c5 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/31d524c5 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/31d524c5 Branch: refs/heads/index_support Commit: 31d524c51cc29da1c90b2431279ca63f24a19172 Parents: a5d14f8 Author: Jihoon Son <[email protected]> Authored: Fri Jul 11 11:47:19 2014 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri Jul 11 11:47:19 2014 +0900 ---------------------------------------------------------------------- .../org/apache/tajo/algebra/CreateDatabase.java | 2 +- .../org/apache/tajo/algebra/CreateIndex.java | 129 ++++++++++++++++++ .../org/apache/tajo/algebra/CreateTable.java | 2 +- .../java/org/apache/tajo/algebra/Insert.java | 2 +- .../java/org/apache/tajo/algebra/OpType.java | 1 + .../java/org/apache/tajo/conf/TajoConf.java | 2 + .../org/apache/tajo/engine/parser/SQLParser.g4 | 4 +- .../apache/tajo/engine/parser/SQLAnalyzer.java | 38 ++++++ .../tajo/engine/planner/AlgebraVisitor.java | 1 + .../tajo/engine/planner/BaseAlgebraVisitor.java | 9 ++ .../engine/planner/BasicLogicalPlanVisitor.java | 14 ++ .../planner/ExplainLogicalPlanVisitor.java | 6 + .../engine/planner/LogicalPlanPreprocessor.java | 12 ++ .../tajo/engine/planner/LogicalPlanVisitor.java | 4 + .../tajo/engine/planner/LogicalPlanner.java | 58 +++++++- .../engine/planner/PhysicalPlannerImpl.java | 7 + .../engine/planner/global/GlobalPlanner.java | 14 ++ .../engine/planner/logical/CreateIndexNode.java | 133 +++++++++++++++++++ .../tajo/engine/planner/logical/NodeType.java | 1 + .../engine/planner/physical/StoreIndexExec.java | 112 ++++++++++++++++ .../planner/rewrite/ProjectionPushDownRule.java | 5 + .../apache/tajo/worker/TaskAttemptContext.java | 10 +- .../java/org/apache/tajo/QueryTestCaseBase.java | 3 + .../tajo/engine/planner/TestLogicalPlanner.java | 26 +++- .../planner/physical/TestPhysicalPlanner.java | 13 +- .../tajo/engine/query/TestCreateIndex.java | 72 ++++++++++ .../queries/TestCreateIndex/testCreateIndex.sql | 1 + .../testCreateIndexOnExpression.sql | 1 + .../testCreateIndexOnMultiAttrs.sql | 1 + .../testCreateIndexWithCondition.sql | 1 + 30 files changed, 668 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateDatabase.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateDatabase.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateDatabase.java index 1144b6e..6e74d3d 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateDatabase.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateDatabase.java @@ -32,7 +32,7 @@ public class CreateDatabase extends Expr { private String tablespaceName; @Expose @SerializedName("IfNotExists") private boolean ifNotExists; - @Expose @SerializedName("DatabaseProperties") + @Expose @SerializedName("Properties") private Map<String, String> params; public CreateDatabase(final String databaseName, final String tablespaceName, final boolean ifNotExists) { http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java new file mode 100644 index 0000000..5d9ffa9 --- /dev/null +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.algebra; + +import com.google.common.base.Objects; +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; +import org.apache.tajo.algebra.Sort.SortSpec; +import org.apache.tajo.util.TUtil; + +import java.util.Map; + +public class CreateIndex extends UnaryOperator { + @Expose @SerializedName("IsUnique") + private boolean unique = false; + @Expose @SerializedName("IndexName") + private String indexName; + @Expose @SerializedName("SortSpecs") + private SortSpec[] sortSpecs; + @Expose @SerializedName("Properties") + private Map<String, String> params; + @Expose @SerializedName("IndexMethodSpec") + private IndexMethodSpec methodSpec; + + public CreateIndex(final String indexName, final SortSpec[] sortSpecs) { + super(OpType.CreateIndex); + this.indexName = indexName; + this.sortSpecs = sortSpecs; + this.methodSpec = new IndexMethodSpec("TWO_LEVEL_BIN_TREE"); + } + + public void setUnique(boolean unique) { + this.unique = unique; + } + + public boolean isUnique() { + return this.unique; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getIndexName() { + return indexName; + } + + public void setSortSpecs(SortSpec[] sortSpecs) { + this.sortSpecs = sortSpecs; + } + + public SortSpec[] getSortSpecs() { + return sortSpecs; + } + + public void setParams(Map<String, String> params) { + this.params = params; + } + + public Map<String, String> getParams() { + return this.params; + } + + public void setMethodSpec(IndexMethodSpec methodSpec) { + this.methodSpec = methodSpec; + } + + public IndexMethodSpec getMethodSpec() { + return this.methodSpec; + } + + @Override + public int hashCode() { + return Objects.hashCode(unique, indexName, sortSpecs, params, methodSpec); + } + + @Override + boolean equalsTo(Expr expr) { + CreateIndex other = (CreateIndex) expr; + return this.unique == other.unique && + this.indexName.equals(other.indexName) && + TUtil.checkEquals(this.sortSpecs, other.sortSpecs) && + TUtil.checkEquals(this.params, other.params) && + this.methodSpec.equals(other.methodSpec); + } + + public static class IndexMethodSpec { + @Expose @SerializedName("IndexMethodName") + private String name; + + public IndexMethodSpec(final String name) { + this.name = name; + } + + public String getName() { + return this.name; + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof IndexMethodSpec) { + IndexMethodSpec other = (IndexMethodSpec) o; + return this.name.equals(other.name); + } + return false; + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java index bd04a91..3121711 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java @@ -43,7 +43,7 @@ public class CreateTable extends Expr { private String location; @Expose @SerializedName("SubPlan") private Expr subquery; - @Expose @SerializedName("TableProperties") + @Expose @SerializedName("Properties") private Map<String, String> params; @Expose @SerializedName("PartitionMethodDesc") private PartitionMethodDescExpr partition; http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java index ce9b703..35d92cd 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java @@ -39,7 +39,7 @@ public class Insert extends Expr { private String location; @Expose @SerializedName("SubPlan") private Expr subquery; - @Expose @SerializedName("InsertParams") + @Expose @SerializedName("Properties") private Map<String, String> params; public Insert() { http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java index 19c4ab5..2ef22f7 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java @@ -50,6 +50,7 @@ public enum OpType { DropTable(DropTable.class), AlterTablespace(AlterTablespace.class), AlterTable(AlterTable.class), + CreateIndex(CreateIndex.class), TruncateTable(TruncateTable.class), // Insert or Update http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 6298d27..1bbb22b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -150,6 +150,8 @@ public class TajoConf extends Configuration { SYSTEM_CONF_PATH("tajo.system-conf.path", EMPTY_VALUE), SYSTEM_CONF_REPLICA_COUNT("tajo.system-conf.replica-count", 20), +// INDEX_DIR("tajo.index.rootdir", "file:///tmp/tajo-${user.name}/index"), + // Tajo Master Service Addresses TAJO_MASTER_UMBILICAL_RPC_ADDRESS("tajo.master.umbilical-rpc.address", "localhost:26001"), TAJO_MASTER_CLIENT_RPC_ADDRESS("tajo.master.client-rpc.address", "localhost:26002"), http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index beba248..706c973 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -68,8 +68,8 @@ schema_statement ; index_statement - : CREATE (u=UNIQUE)? INDEX n=identifier ON t=table_name (m=method_specifier)? - LEFT_PAREN s=sort_specifier_list RIGHT_PAREN p=param_clause? + : CREATE (u=UNIQUE)? INDEX identifier ON table_name (method_specifier)? + LEFT_PAREN sort_specifier_list RIGHT_PAREN param_clause? (where_clause)? ; database_definition http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 580ec61..faf0212 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -26,7 +26,9 @@ import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.tree.TerminalNode; import org.apache.tajo.algebra.*; import org.apache.tajo.algebra.Aggregation.GroupType; +import org.apache.tajo.algebra.CreateIndex.IndexMethodSpec; import org.apache.tajo.algebra.LiteralValue.LiteralType; +import org.apache.tajo.algebra.Sort.SortSpec; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.engine.parser.SQLParser.*; import org.apache.tajo.storage.StorageConstants; @@ -1140,6 +1142,42 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { } @Override + public Expr visitIndex_statement(SQLParser.Index_statementContext ctx) { + String indexName = ctx.identifier().getText(); + String tableName = ctx.table_name().getText(); + Relation relation = new Relation(tableName); + SortSpec[] sortSpecs = buildSortSpecs(ctx.sort_specifier_list()); + NamedExpr[] targets = new NamedExpr[sortSpecs.length]; + Projection projection = new Projection(); + int i = 0; + for (SortSpec sortSpec : sortSpecs) { + targets[i++] = new NamedExpr(sortSpec.getKey()); + } + projection.setNamedExprs(targets); + projection.setChild(relation); + + CreateIndex createIndex = new CreateIndex(indexName, sortSpecs); + if (checkIfExist(ctx.UNIQUE())) { + createIndex.setUnique(true); + } + if (checkIfExist(ctx.method_specifier())) { + String methodName = ctx.method_specifier().identifier().getText(); + createIndex.setMethodSpec(new IndexMethodSpec(methodName)); + } + if (checkIfExist(ctx.param_clause())) { + Map<String, String> params = getParams(ctx.param_clause()); + createIndex.setParams(params); + } + if (checkIfExist(ctx.where_clause())) { + Selection selection = visitWhere_clause(ctx.where_clause()); + selection.setChild(relation); + projection.setChild(selection); + } + createIndex.setChild(projection); + return createIndex; + } + + @Override public Expr visitDatabase_definition(@NotNull SQLParser.Database_definitionContext ctx) { return new CreateDatabase(ctx.identifier().getText(), null, checkIfExist(ctx.if_not_exists())); } http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java index 8c3e606..eb8a21f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java @@ -48,6 +48,7 @@ public interface AlgebraVisitor<CONTEXT, RESULT> { RESULT visitDropTable(CONTEXT ctx, Stack<Expr> stack, DropTable expr) throws PlanningException; RESULT visitAlterTablespace(CONTEXT ctx, Stack<Expr> stack, AlterTablespace expr) throws PlanningException; RESULT visitAlterTable(CONTEXT ctx, Stack<Expr> stack, AlterTable expr) throws PlanningException; + RESULT visitCreateIndex(CONTEXT ctx, Stack<Expr> stack, CreateIndex expr) throws PlanningException; RESULT visitTruncateTable(CONTEXT ctx, Stack<Expr> stack, TruncateTable expr) throws PlanningException; // Insert or Update http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java index 24ff2e4..b93541b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java @@ -118,6 +118,9 @@ public class BaseAlgebraVisitor<CONTEXT, RESULT> implements AlgebraVisitor<CONTE case AlterTable: current = visitAlterTable(ctx, stack, (AlterTable) expr); break; + case CreateIndex: + current = visitCreateIndex(ctx, stack, (CreateIndex) expr); + break; case TruncateTable: current = visitTruncateTable(ctx, stack, (TruncateTable)expr); break; @@ -470,6 +473,12 @@ public class BaseAlgebraVisitor<CONTEXT, RESULT> implements AlgebraVisitor<CONTE return null; } + @Override + public RESULT visitCreateIndex(CONTEXT ctx, Stack<Expr> stack, CreateIndex expr) throws PlanningException { + return null; + } + + @Override public RESULT visitTruncateTable(CONTEXT ctx, Stack<Expr> stack, TruncateTable expr) throws PlanningException { return null; } http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java index a4e90b4..59422f1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner; +import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock; import org.apache.tajo.engine.planner.logical.*; import java.util.Stack; @@ -128,6 +129,9 @@ public class BasicLogicalPlanVisitor<CONTEXT, RESULT> implements LogicalPlanVisi case ALTER_TABLE: current = visitAlterTable(context, plan, block, (AlterTableNode) node, stack); break; + case CREATE_INDEX: + current = visitCreateIndex(context, plan, block, (CreateIndexNode) node, stack); + break; case TRUNCATE_TABLE: current = visitTruncateTable(context, plan, block, (TruncateTableNode) node, stack); break; @@ -344,6 +348,16 @@ public class BasicLogicalPlanVisitor<CONTEXT, RESULT> implements LogicalPlanVisi } @Override + public RESULT visitCreateIndex(CONTEXT context, LogicalPlan plan, QueryBlock block, CreateIndexNode node, + Stack<LogicalNode> stack) throws PlanningException { + RESULT result = null; + stack.push(node); + result = visit(context, plan, block, node.getChild(), stack); + stack.pop(); + return result; + } + + @Override public RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TruncateTableNode node, Stack<LogicalNode> stack) throws PlanningException { return null; http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java index a7e5375..97536c5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java @@ -230,6 +230,12 @@ public class ExplainLogicalPlanVisitor extends BasicLogicalPlanVisitor<ExplainLo return node; } + @Override + public LogicalNode visitCreateIndex(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, + CreateIndexNode node, Stack<LogicalNode> stack) throws PlanningException { + return visitUnaryNode(context, plan, block, node, stack); + } + public static String printDepthString(int maxDepth, DepthString planStr) { StringBuilder output = new StringBuilder(); String pad = new String(new char[planStr.getDepth() * 3]).replace('\0', ' '); http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index 4f1218f..9438d34 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -431,6 +431,18 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanPrepr } @Override + public LogicalNode visitCreateIndex(PreprocessContext ctx, Stack<Expr> stack, CreateIndex expr) + throws PlanningException { + stack.push(expr); + LogicalNode child = visit(ctx, stack, expr.getChild()); + stack.pop(); + + CreateIndexNode createIndex = ctx.plan.createNode(CreateIndexNode.class); + createIndex.setInSchema(child.getOutSchema()); + createIndex.setOutSchema(child.getOutSchema()); + return createIndex; + } + public LogicalNode visitTruncateTable(PreprocessContext ctx, Stack<Expr> stack, TruncateTable expr) throws PlanningException { TruncateTableNode truncateTableNode = ctx.plan.createNode(TruncateTableNode.class); http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java index 0a36610..57f943e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner; +import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock; import org.apache.tajo.engine.planner.logical.*; import java.util.Stack; @@ -93,6 +94,9 @@ public interface LogicalPlanVisitor<CONTEXT, RESULT> { RESULT visitAlterTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, AlterTableNode node, Stack<LogicalNode> stack) throws PlanningException; + RESULT visitCreateIndex(CONTEXT context, LogicalPlan plan, QueryBlock block, CreateIndexNode node, + Stack<LogicalNode> stack) throws PlanningException; + RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TruncateTableNode node, Stack<LogicalNode> stack) throws PlanningException; } http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index 80390d3..0315cb1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -34,6 +34,7 @@ import org.apache.tajo.algebra.WindowSpec; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.engine.eval.*; @@ -788,19 +789,23 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex // Building sort keys + sortNode.setSortSpecs(annotateSortSpecs(block, referNames, sortSpecs)); + return sortNode; + } + + private static SortSpec[] annotateSortSpecs(QueryBlock block, String [] referNames, Sort.SortSpec[] rawSortSpecs) { + int sortKeyNum = rawSortSpecs.length; Column column; SortSpec [] annotatedSortSpecs = new SortSpec[sortKeyNum]; for (int i = 0; i < sortKeyNum; i++) { if (block.namedExprsMgr.isEvaluated(referNames[i])) { column = block.namedExprsMgr.getTarget(referNames[i]).getNamedColumn(); } else { - throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(sortSpecs)); + throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(rawSortSpecs)); } - annotatedSortSpecs[i] = new SortSpec(column, sortSpecs[i].isAscending(), sortSpecs[i].isNullFirst()); + annotatedSortSpecs[i] = new SortSpec(column, rawSortSpecs[i].isAscending(), rawSortSpecs[i].isNullFirst()); } - - sortNode.setSortSpecs(annotatedSortSpecs); - return sortNode; + return annotatedSortSpecs; } /*=============================================================================================== @@ -1829,6 +1834,49 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex } @Override + public LogicalNode visitCreateIndex(PlanContext context, Stack<Expr> stack, CreateIndex createIndex) + throws PlanningException { + stack.push(createIndex); + LogicalNode child = visit(context, stack, createIndex.getChild()); + stack.pop(); + + QueryBlock block = context.queryBlock; + CreateIndexNode createIndexNode = block.getNodeFromExpr(createIndex); + if (CatalogUtil.isFQTableName(createIndex.getIndexName())) { + createIndexNode.setIndexName(createIndex.getIndexName()); + } else { + createIndexNode.setIndexName( + CatalogUtil.buildFQName(context.session.getCurrentDatabase(), createIndex.getIndexName())); + } + createIndexNode.setUnique(createIndex.isUnique()); + Sort.SortSpec[] sortSpecs = createIndex.getSortSpecs(); + int sortKeyNum = sortSpecs.length; + String[] referNames = new String[sortKeyNum]; + + ExprNormalizedResult[] normalizedExprList = new ExprNormalizedResult[sortKeyNum]; + for (int i = 0; i < sortKeyNum; i++) { + normalizedExprList[i] = normalizer.normalize(context, sortSpecs[i].getKey()); + } + for (int i = 0; i < sortKeyNum; i++) { + referNames[i] = block.namedExprsMgr.addExpr(normalizedExprList[i].baseExpr); + block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].aggExprs); + block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].scalarExprs); + } + + createIndexNode.setSortSpecs(annotateSortSpecs(block, referNames, sortSpecs)); + createIndexNode.setIndexType(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase())); + + if (createIndex.getParams() != null) { + KeyValueSet keyValueSet = new KeyValueSet(); + keyValueSet.putAll(createIndex.getParams()); + createIndexNode.setOptions(keyValueSet); + } + + createIndexNode.setChild(child); + return createIndexNode; + } + + @Override public LogicalNode visitTruncateTable(PlanContext context, Stack<Expr> stack, TruncateTable truncateTable) throws PlanningException { TruncateTableNode truncateTableNode = context.queryBlock.getNodeFromExpr(truncateTable); http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index cf02ecd..8f2ede3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -234,6 +234,13 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { leftExec = createIndexScanExec(ctx, indexScanNode); return leftExec; + case CREATE_INDEX: + CreateIndexNode createIndexNode = (CreateIndexNode) logicalNode; + stack.push(createIndexNode); + leftExec = createPlanRecursive(ctx, createIndexNode.getChild(), stack); + stack.pop(); + return new StoreIndexExec(ctx, createIndexNode, leftExec); + default: return null; } http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 2d0dd10..e603e87 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -1586,6 +1586,20 @@ public class GlobalPlanner { return node; } + + @Override + public LogicalNode visitCreateIndex(GlobalPlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock queryBlock, + CreateIndexNode node, Stack<LogicalNode> stack) throws PlanningException { + LogicalNode child = super.visitCreateIndex(context, plan, queryBlock, node, stack); + + // Don't separate execution block. CreateIndex is pushed to the first execution block. + ExecutionBlock childBlock = context.execBlockMap.remove(child.getPID()); + node.setChild(childBlock.getPlan()); + childBlock.setPlan(node); + context.execBlockMap.put(node.getPID(), childBlock); + + return node; + } } @SuppressWarnings("unused") http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java new file mode 100644 index 0000000..1dca608 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.logical; + +import com.google.common.base.Objects; +import com.google.gson.annotations.Expose; +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.TUtil; + +import static org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; + +public class CreateIndexNode extends UnaryNode implements Cloneable { + @Expose private boolean isUnique; + @Expose private String indexName; + @Expose private SortSpec[] sortSpecs; + @Expose private IndexMethod indexType = IndexMethod.TWO_LEVEL_BIN_TREE; + @Expose private KeyValueSet options; + + public CreateIndexNode(int pid) { + super(pid, NodeType.CREATE_INDEX); + } + + public void setUnique(boolean unique) { + this.isUnique = unique; + } + + public boolean isUnique() { + return isUnique; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getIndexName() { + return this.indexName; + } + + public void setSortSpecs(SortSpec[] sortSpecs) { + this.sortSpecs = sortSpecs; + } + + public SortSpec[] getSortSpecs() { + return this.sortSpecs; + } + + public void setIndexType(IndexMethod indexType) { + this.indexType = indexType; + } + + public IndexMethod getIndexType() { + return this.indexType; + } + + public void setOptions(KeyValueSet options) { + this.options = options; + } + + public KeyValueSet getOptions() { + return this.options; + } + + @Override + public int hashCode() { + return Objects.hashCode(isUnique, indexName, sortSpecs, indexType, options); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof CreateIndexNode) { + CreateIndexNode other = (CreateIndexNode) obj; + return this.isUnique == other.isUnique && + TUtil.checkEquals(this.indexName, other.indexName) && + TUtil.checkEquals(this.sortSpecs, other.sortSpecs) && + this.indexType.equals(other.indexType) && + TUtil.checkEquals(this.options, other.options); + } + return false; + } + + @Override + public Object clone() throws CloneNotSupportedException { + CreateIndexNode createIndexNode = (CreateIndexNode) super.clone(); + createIndexNode.isUnique = isUnique; + createIndexNode.indexName = indexName; + createIndexNode.sortSpecs = sortSpecs.clone(); + createIndexNode.indexType = indexType; + createIndexNode.options = (KeyValueSet) (options != null ? options.clone() : null); + return createIndexNode; + } + + private String getSortSpecString() { + StringBuilder sb = new StringBuilder("Column [key= "); + for (int i = 0; i < sortSpecs.length; i++) { + sb.append(sortSpecs[i].getSortKey().getQualifiedName()).append(" ") + .append(sortSpecs[i].isAscending() ? "asc" : "desc"); + if(i < sortSpecs.length - 1) { + sb.append(","); + } + } + sb.append("]"); + return sb.toString(); + } + + @Override + public String toString() { + return "CreateIndex (index=" + indexName + ", type=" + indexType.name() + + ", isUnique=" + isUnique + ", " + getSortSpecString() + ")"; + } + + @Override + public PlanString getPlanString() { + return new PlanString(this); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java index fa1199b..2e9e0da 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java @@ -55,6 +55,7 @@ public enum NodeType { DROP_TABLE(DropTableNode.class), ALTER_TABLESPACE (AlterTablespaceNode.class), ALTER_TABLE (AlterTableNode.class), + CREATE_INDEX(CreateIndexNode.class), TRUNCATE_TABLE (TruncateTableNode.class); private final Class<? extends LogicalNode> baseClass; http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java new file mode 100644 index 0000000..91b5322 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.physical; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.engine.planner.PlannerUtil; +import org.apache.tajo.engine.planner.logical.CreateIndexNode; +import org.apache.tajo.storage.RowStoreUtil; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.TupleComparator; +import org.apache.tajo.storage.VTuple; +import org.apache.tajo.storage.index.bst.BSTIndex; +import org.apache.tajo.storage.index.bst.BSTIndex.BSTIndexWriter; +import org.apache.tajo.worker.TaskAttemptContext; + +import java.io.IOException; + +public class StoreIndexExec extends UnaryPhysicalExec { + private static final Log LOG = LogFactory.getLog(StoreIndexExec.class); + private BSTIndexWriter indexWriter; + private final CreateIndexNode logicalPlan; + private int[] indexKeys = null; + private Schema keySchema; + private TupleComparator comparator; + + public StoreIndexExec(final TaskAttemptContext context, final CreateIndexNode logicalPlan, + final PhysicalExec child) { + super(context, logicalPlan.getInSchema(), logicalPlan.getOutSchema(), child); + this.logicalPlan = logicalPlan; + } + + @Override + public void init() throws IOException { + super.init(); + + SortSpec[] sortSpecs = logicalPlan.getSortSpecs(); + indexKeys = new int[sortSpecs.length]; + keySchema = PlannerUtil.sortSpecsToSchema(sortSpecs); + + Column col; + for (int i = 0 ; i < sortSpecs.length; i++) { + col = sortSpecs[i].getSortKey(); + indexKeys[i] = inSchema.getColumnId(col.getQualifiedName()); + } + + TajoConf conf = context.getConf(); + + String[] splits = logicalPlan.getIndexName().split("\\."); + Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), splits[0] + "/" + splits[1] + "/" + + context.getUniqueKeyFromFragments()); + System.out.println("exec: " + indexPath); + // TODO: Create factory using reflection + BSTIndex bst = new BSTIndex(conf); + this.comparator = new TupleComparator(keySchema, sortSpecs); + this.indexWriter = bst.getIndexWriter(indexPath, BSTIndex.TWO_LEVEL_INDEX, keySchema, comparator); + this.indexWriter.setLoadNum(100); + this.indexWriter.open(); + } + + @Override + public Tuple next() throws IOException { + Tuple tuple; + Tuple keyTuple; + Tuple prevKeyTuple = null; + long offset; + + while((tuple = child.next()) != null) { + offset = tuple.getOffset(); + keyTuple = new VTuple(keySchema.size()); + RowStoreUtil.project(tuple, keyTuple, indexKeys); + if (prevKeyTuple == null || !prevKeyTuple.equals(keyTuple)) { + indexWriter.write(keyTuple, offset); + prevKeyTuple = keyTuple; + } + } + return null; + } + + @Override + public void close() throws IOException { + super.close(); + + indexWriter.flush(); + IOUtils.cleanup(LOG, indexWriter); + + indexWriter = null; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java index 2bc210c..d92883e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java @@ -505,6 +505,11 @@ public class ProjectionPushDownRule extends createTableNode.setChild(child); createTableNode.setInSchema(child.getOutSchema()); break; + case CREATE_INDEX: + CreateIndexNode createIndexNode = (CreateIndexNode) parentNode; + createIndexNode.setChild(child); + createIndexNode.setInSchema(child.getOutSchema()); + break; default: throw new PlanningException("Unexpected Parent Node: " + parentNode.getType()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index e073652..325aceb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -275,7 +275,15 @@ public class TaskAttemptContext { } return fragmentMap.get(id).toArray(new FragmentProto[fragmentMap.get(id).size()]); } - + + public long getUniqueKeyFromFragments() { + List<FragmentProto> totalFragments = new ArrayList<FragmentProto>(); + for (List<FragmentProto> eachFragments : fragmentMap.values()) { + totalFragments.addAll(eachFragments); + } + return Objects.hashCode(totalFragments.toArray(new FragmentProto[totalFragments.size()])); + } + public int hashCode() { return Objects.hashCode(queryId); } http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 70c73f9..bc64bb8 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -533,6 +533,9 @@ public class QueryTestCaseBase { if (isLocalTable) { createdTableGlobalSet.remove(tableName); } + } else if (expr.getType() == OpType.CreateIndex) { + // TODO: index existence check + client.executeQuery(compiled); } else { assertTrue(ddlFilePath + " is not a Create or Drop Table statement", false); } http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 6d0f7e0..2ca85cc 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -140,14 +140,14 @@ public class TestLogicalPlanner { "select name, empid, e.deptname, manager from employee as e, dept as dp", // 1 "select name, empid, e.deptname, manager, score from employee as e, dept, score", // 2 "select p.deptname, sumtest(score) from dept as p, score group by p.deptName having sumtest(score) > 30", // 3 - "select p.deptname, score from dept as p, score order by score asc", // 4 + "select p.deptname, score*200 from dept as p, score order by score*10 asc", // 4 "select name from employee where empId = 100", // 5 "select name, score from employee, score", // 6 "select p.deptName, sumtest(score) from dept as p, score group by p.deptName", // 7 "create table store1 as select p.deptName, sumtest(score) from dept as p, score group by p.deptName", // 8 "select deptName, sumtest(score) from score group by deptName having sumtest(score) > 30", // 9 "select 7 + 8 as res1, 8 * 9 as res2, 10 * 10 as res3", // 10 - "create index idx_employee on employee using bitmap (name null first, empId desc) with ('fillfactor' = 70)", // 11 + "create index idx_employee on employee using bitmap_idx (name null first, empId desc) where empid > 100", // 11 "select name, score from employee, score order by score limit 3", // 12 "select length(name), length(deptname), *, empid+10 from employee where empId > 500", // 13 }; @@ -836,6 +836,28 @@ public class TestLogicalPlanner { } @Test + public final void testCreateIndexNode() throws PlanningException { + Expr expr = sqlAnalyzer.parse(QUERIES[11]); + LogicalPlan rootNode = planner.createPlan(session, expr); + LogicalNode plan = rootNode.getRootBlock().getRoot(); + testJsonSerDerObject(plan); + + LogicalRootNode root = (LogicalRootNode) plan; + assertEquals(NodeType.CREATE_INDEX, root.getChild().getType()); + CreateIndexNode createIndexNode = root.getChild(); + + assertEquals(NodeType.PROJECTION, createIndexNode.getChild().getType()); + ProjectionNode projNode = createIndexNode.getChild(); + + assertEquals(NodeType.SELECTION, projNode.getChild().getType()); + SelectionNode selNode = projNode.getChild(); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName()); + } + + @Test public final void testAsterisk() throws CloneNotSupportedException, PlanningException { Expr expr = sqlAnalyzer.parse(QUERIES[13]); LogicalPlan planNode = planner.createPlan(session, expr); http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index a823d2b..920c932 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -34,6 +34,7 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; @@ -789,17 +790,23 @@ public class TestPhysicalPlanner { } public final String [] createIndexStmt = { - "create index idx_employee on employee using bst (name null first, empId desc)" + "create index idx_employee on employee using TWO_LEVEL_BIN_TREE (name null first, empId desc)" }; - //@Test + @Test public final void testCreateIndex() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); + Path indexPath = StorageUtil.concatPath(conf.getVar(ConfVars.ROOT_DIR), "default/idx_employee"); + if (sm.getFileSystem().exists(indexPath)) { + sm.getFileSystem().delete(indexPath, true); + } + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); + ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(createIndexStmt[0]); LogicalPlan plan = planner.createPlan(session, context); LogicalNode rootNode = optimizer.optimize(plan); @@ -811,7 +818,7 @@ public class TestPhysicalPlanner { } exec.close(); - FileStatus [] list = sm.getFileSystem().listStatus(StorageUtil.concatPath(workDir, "index")); + FileStatus[] list = sm.getFileSystem().listStatus(indexPath); assertEquals(2, list.length); } http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java new file mode 100644 index 0000000..2a66909 --- /dev/null +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.query; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.IntegrationTest; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.TajoConstants; +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; + +import static org.junit.Assert.*; + +@Category(IntegrationTest.class) +public class TestCreateIndex extends QueryTestCaseBase { + + public TestCreateIndex() { + super(TajoConstants.DEFAULT_DATABASE_NAME); + } + + private static void assertIndexExist(String indexName) throws IOException { + Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), "default/" + indexName); + FileSystem fs = indexPath.getFileSystem(conf); + assertTrue(fs.exists(indexPath)); + assertEquals(2, fs.listStatus(indexPath).length); + fs.deleteOnExit(indexPath); + } + + @Test + public final void testCreateIndex() throws Exception { + executeQuery(); + assertIndexExist("l_orderkey_idx"); + } + + @Test + public final void testCreateIndexOnMultiAttrs() throws Exception { + executeQuery(); + assertIndexExist("l_orderkey_partkey_idx"); + } + + @Test + public final void testCreateIndexWithCondition() throws Exception { + executeQuery(); + assertIndexExist("l_orderkey_partkey_lt10_idx"); + } + + @Test + public final void testCreateIndexOnExpression() throws Exception { + executeQuery(); + assertIndexExist("l_orderkey_100_lt10_idx"); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndex.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndex.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndex.sql new file mode 100644 index 0000000..1cb8936 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndex.sql @@ -0,0 +1 @@ +create index l_orderkey_idx on lineitem (l_orderkey asc null first); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnExpression.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnExpression.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnExpression.sql new file mode 100644 index 0000000..0d19cb3 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnExpression.sql @@ -0,0 +1 @@ +create index l_orderkey_100_lt10_idx on lineitem (l_orderkey*100 asc null first) where l_orderkey*100 > 10; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiAttrs.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiAttrs.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiAttrs.sql new file mode 100644 index 0000000..3487d2e --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiAttrs.sql @@ -0,0 +1 @@ +create index l_orderkey_partkey_idx on lineitem (l_orderkey asc, l_partkey desc null last); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/31d524c5/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexWithCondition.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexWithCondition.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexWithCondition.sql new file mode 100644 index 0000000..ad4ee47 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexWithCondition.sql @@ -0,0 +1 @@ +create index l_orderkey_partkey_lt10_idx on lineitem (l_orderkey asc, l_partkey desc) where l_partkey > 10; \ No newline at end of file
