Repository: tajo Updated Branches: refs/heads/index_support 5bb4108c9 -> 63c8e1c00
TAJO-1301: Support CREATE INDEX [index_name] on [table_name] ( [index_specs] ) LOCATION [PATH] statement. (jihoon) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/63c8e1c0 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/63c8e1c0 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/63c8e1c0 Branch: refs/heads/index_support Commit: 63c8e1c00430cd881b4ffe5d3f6741cc6347991d Parents: 5bb4108 Author: Jihoon Son <[email protected]> Authored: Fri Mar 13 12:26:41 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri Mar 13 12:26:41 2015 +0900 ---------------------------------------------------------------------- .../org/apache/tajo/algebra/CreateIndex.java | 21 ++++++++++- .../org/apache/tajo/engine/parser/SQLParser.g4 | 4 +- .../apache/tajo/engine/parser/SQLAnalyzer.java | 5 ++- .../apache/tajo/master/exec/DDLExecutor.java | 39 +++++++++++++++++++- .../apache/tajo/master/exec/QueryExecutor.java | 5 +-- .../tajo/engine/query/TestCreateIndex.java | 15 ++++++++ .../testCreateIndexOnLocation.sql | 1 + .../org/apache/tajo/plan/LogicalPlanner.java | 9 ++++- .../tajo/plan/logical/CreateIndexNode.java | 17 +++++++-- .../rules/LogicalPlanEqualityTester.java | 1 - .../plan/serder/LogicalNodeDeserializer.java | 1 + .../tajo/plan/serder/LogicalNodeSerializer.java | 1 + .../org/apache/tajo/plan/util/PlannerUtil.java | 2 +- tajo-plan/src/main/proto/Plan.proto | 1 + 14 files changed, 105 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java index 5d9ffa9..a9f734d 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java @@ -37,6 +37,9 @@ public class CreateIndex extends UnaryOperator { private Map<String, String> params; @Expose @SerializedName("IndexMethodSpec") private IndexMethodSpec methodSpec; + private boolean external = false; + @Expose @SerializedName("IndexPath") + private String indexPath; public CreateIndex(final String indexName, final SortSpec[] sortSpecs) { super(OpType.CreateIndex); @@ -85,9 +88,22 @@ public class CreateIndex extends UnaryOperator { return this.methodSpec; } + public void setIndexPath(String indexPath) { + this.external = true; + this.indexPath = indexPath; + } + + public boolean isExternal() { + return this.external; + } + + public String getIndexPath() { + return this.indexPath; + } + @Override public int hashCode() { - return Objects.hashCode(unique, indexName, sortSpecs, params, methodSpec); + return Objects.hashCode(unique, indexName, sortSpecs, params, methodSpec, external); } @Override @@ -97,7 +113,8 @@ public class CreateIndex extends UnaryOperator { this.indexName.equals(other.indexName) && TUtil.checkEquals(this.sortSpecs, other.sortSpecs) && TUtil.checkEquals(this.params, other.params) && - this.methodSpec.equals(other.methodSpec); + this.methodSpec.equals(other.methodSpec) && + this.external == other.external; } public static class IndexMethodSpec { http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index f3423c1..9056605 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -82,8 +82,8 @@ index_statement ; create_index_statement - : CREATE (u=UNIQUE)? INDEX identifier ON table_name (method_specifier)? - LEFT_PAREN sort_specifier_list RIGHT_PAREN param_clause? (where_clause)? + : CREATE (u=UNIQUE)? INDEX index_name = identifier ON table_name (method_specifier)? + LEFT_PAREN sort_specifier_list RIGHT_PAREN param_clause? (where_clause)? (LOCATION path=Character_String_Literal)? ; drop_index_statement http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index a349b2f..d6696f5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1216,7 +1216,7 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { @Override public Expr visitCreate_index_statement(SQLParser.Create_index_statementContext ctx) { - String indexName = ctx.identifier().getText(); + String indexName = ctx.index_name.getText(); String tableName = ctx.table_name().getText(); Relation relation = new Relation(tableName); SortSpec[] sortSpecs = buildSortSpecs(ctx.sort_specifier_list()); @@ -1246,6 +1246,9 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { selection.setChild(relation); projection.setChild(selection); } + if (checkIfExist(ctx.LOCATION())) { + createIndex.setIndexPath(stripQuote(ctx.path.getText())); + } createIndex.setChild(projection); return createIndex; } http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 9c023a3..dd5451c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -102,8 +102,8 @@ public class DDLExecutor { return true; case CREATE_INDEX: - // The catalog information for the created index is automatically updated when the query is successfully finished. - // See the Query.CreateIndexHook class. + CreateIndexNode createIndex = (CreateIndexNode) root; + createIndex(queryContext, createIndex); return true; case DROP_INDEX: @@ -116,6 +116,41 @@ public class DDLExecutor { } } + public void createIndex(final QueryContext queryContext, final CreateIndexNode createIndexNode) { + String databaseName, simpleIndexName, qualifiedIndexName; + if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) { + String [] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName()); + databaseName = splits[0]; + simpleIndexName = splits[1]; + qualifiedIndexName = createIndexNode.getIndexName(); + } else { + databaseName = queryContext.getCurrentDatabase(); + simpleIndexName = createIndexNode.getIndexName(); + qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName); + } + + if (catalog.existIndexByName(databaseName, simpleIndexName)) { + throw new AlreadyExistsIndexException(simpleIndexName); + } + + ScanNode scanNode = PlannerUtil.findTopNode(createIndexNode, NodeType.SCAN); + if (scanNode == null) { + throw new InternalError("Cannot find the table of the relation"); + } + + IndexDesc indexDesc = new IndexDesc(databaseName, scanNode.getTableName(), + simpleIndexName, createIndexNode.getIndexPath(), + createIndexNode.getKeySortSpecs(), createIndexNode.getIndexMethod(), + createIndexNode.isUnique(), false, scanNode.getLogicalSchema()); + + if (catalog.createIndex(indexDesc)) { + LOG.info("Index " + qualifiedIndexName + " is created for the table " + scanNode.getTableName() + "."); + } else { + LOG.info("Index creation " + qualifiedIndexName + " is failed."); + throw new CatalogException("Cannot create index \"" + qualifiedIndexName + "\"."); + } + } + public void dropIndex(final QueryContext queryContext, final DropIndexNode dropIndexNode) { String databaseName, simpleIndexName; if (CatalogUtil.isFQTableName(dropIndexNode.getIndexName())) { http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index c9ff554..ae3feaf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -108,10 +108,9 @@ public class QueryExecutor { } else { response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); response.setResult(IPCUtil.buildOkRequestResult()); + ddlExecutor.execute(queryContext, plan); } - ddlExecutor.execute(queryContext, plan); - } else if (plan.isExplain()) { // explain query execExplain(plan, response); @@ -139,7 +138,7 @@ public class QueryExecutor { public void execSetSession(Session session, LogicalPlan plan, SubmitQueryResponse.Builder response) { - SetSessionNode setSessionNode = ((LogicalRootNode)plan.getRootBlock().getRoot()).getChild(); + SetSessionNode setSessionNode = ((LogicalRootNode) plan.getRootBlock().getRoot()).getChild(); final String varName = setSessionNode.getName(); http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java index 83e33b7..1a55870 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java @@ -95,4 +95,19 @@ public class TestCreateIndex extends QueryTestCaseBase { assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_100_l_linenumber_10_lt10_idx")); assertIndexNotExist(getCurrentDatabase(), "l_orderkey_100_l_linenumber_10_lt10_idx"); } + + @Test + public final void testCreateIndexOnLocation() throws Exception { + executeQuery(); + assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); + assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey"})); + catalog.dropIndex(getCurrentDatabase(), "l_orderkey_idx"); + assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); + executeString("create index l_orderkey_idx on lineitem (l_orderkey asc null first) location '/tajo/warehouse/default/l_orderkey_idx';"); + assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); + assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey"})); + executeString("drop index l_orderkey_idx"); + assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); + assertIndexNotExist(getCurrentDatabase(), "l_orderkey_idx"); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnLocation.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnLocation.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnLocation.sql new file mode 100644 index 0000000..1cb8936 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnLocation.sql @@ -0,0 +1 @@ +create index l_orderkey_idx on lineitem (l_orderkey asc null first); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index c849ae5..d495953 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -1976,13 +1976,18 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].scalarExprs); } + createIndexNode.setExternal(createIndex.isExternal()); Collection<RelationNode> relations = block.getRelations(); assert relations.size() == 1; createIndexNode.setKeySortSpecs(relations.iterator().next().getLogicalSchema(), annotateSortSpecs(block, referNames, sortSpecs)); createIndexNode.setIndexMethod(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase())); - createIndexNode.setIndexPath( - getIndexPath(context, context.queryContext.get(SessionVars.CURRENT_DATABASE), createIndex.getIndexName())); + if (createIndex.isExternal()) { + createIndexNode.setIndexPath(new Path(createIndex.getIndexPath()).toUri()); + } else { + createIndexNode.setIndexPath( + getIndexPath(context, context.queryContext.get(SessionVars.CURRENT_DATABASE), createIndex.getIndexName())); + } if (createIndex.getParams() != null) { KeyValueSet keyValueSet = new KeyValueSet(); http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java index fcedf48..df7eb34 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java @@ -18,6 +18,7 @@ package org.apache.tajo.plan.logical; +import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.IndexMeta; import org.apache.tajo.catalog.Schema; @@ -31,6 +32,7 @@ import static org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; public class CreateIndexNode extends UnaryNode implements Cloneable { @Expose private IndexMeta indexMeta; + @Expose private boolean external; public CreateIndexNode(int pid) { super(pid, NodeType.CREATE_INDEX); @@ -101,16 +103,24 @@ public class CreateIndexNode extends UnaryNode implements Cloneable { return indexMeta.isClustered(); } + public void setExternal(boolean external) { + this.external = external; + } + + public boolean isExternal() { + return this.external; + } + @Override public int hashCode() { - return indexMeta.hashCode(); + return Objects.hashCode(indexMeta, external); } @Override public boolean equals(Object obj) { if (obj instanceof CreateIndexNode) { CreateIndexNode other = (CreateIndexNode) obj; - return this.indexMeta.equals(other.indexMeta); + return this.indexMeta.equals(other.indexMeta) && this.external == other.external; } return false; } @@ -119,6 +129,7 @@ public class CreateIndexNode extends UnaryNode implements Cloneable { public Object clone() throws CloneNotSupportedException { CreateIndexNode createIndexNode = (CreateIndexNode) super.clone(); createIndexNode.indexMeta = (IndexMeta) this.indexMeta.clone(); + createIndexNode.external = this.external; return createIndexNode; } @@ -140,7 +151,7 @@ public class CreateIndexNode extends UnaryNode implements Cloneable { public String toString() { return "CreateIndex (indexName=" + indexMeta.getIndexName() + ", indexPath=" + indexMeta.getIndexPath() + ", type=" + indexMeta.getIndexMethod().name() + - ", isUnique=" + indexMeta.isUnique() + ", " + getSortSpecString() + ")"; + ", isUnique=" + indexMeta.isUnique() + ", " + getSortSpecString() + ", isExternal=" + isExternal() + ")"; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java index c8a81ec..35e7a91 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java @@ -18,7 +18,6 @@ package org.apache.tajo.plan.rewrite.rules; -import org.apache.tajo.OverridableConf; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.LogicalNode; http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index 9cdf18b..b70f779 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -652,6 +652,7 @@ public class LogicalNodeDeserializer { createIndex.setChild(nodeMap.get(createIndexProto.getChildSeq())); createIndex.setInSchema(convertSchema(protoNode.getInSchema())); createIndex.setOutSchema(convertSchema(protoNode.getOutSchema())); + createIndex.setExternal(createIndexProto.getIsExternal()); return createIndex; } http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java index 5ad79c0..d91db93 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java @@ -723,6 +723,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe if (node.hasOptions()) { createIndexBuilder.setIndexProperties(node.getOptions().getProto()); } + createIndexBuilder.setIsExternal(node.isExternal()); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); nodeBuilder.setCreateIndex(createIndexBuilder); http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index ebd47de..f8e69e8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -94,7 +94,7 @@ public class PlannerUtil { NodeType type = baseNode.getType(); - return type == NodeType.CREATE_INDEX || + return type == NodeType.CREATE_INDEX && !((CreateIndexNode)baseNode).isExternal() || type == NodeType.CREATE_TABLE && ((CreateTableNode)baseNode).hasSubQuery(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/63c8e1c0/tajo-plan/src/main/proto/Plan.proto ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index dfee969..2f2874c 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -328,6 +328,7 @@ message CreateIndexNode { optional bool isUnique = 7 [default = false]; optional bool isClustered = 8 [default = false]; optional KeyValueSetProto indexProperties = 9; + optional bool isExternal = 10; } message DropIndexNode {
