Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/15749c5a Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/15749c5a Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/15749c5a Branch: refs/heads/index_support Commit: 15749c5a66f8d6e230d2c29b74b17897e1e11527 Parents: 8effa33 a1a9d62 Author: Jihoon Son <[email protected]> Authored: Fri Jul 17 23:27:43 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri Jul 17 23:27:43 2015 +0900 ---------------------------------------------------------------------- .travis.yml | 2 +- CHANGES | 20 ++- .../org/apache/tajo/algebra/AlterTable.java | 27 +-- .../org/apache/tajo/catalog/CatalogUtil.java | 66 ++++++++ ...xistsAssumedPartitionDirectoryException.java | 28 ++++ .../AlreadyExistsPartitionException.java | 6 +- .../exception/NoPartitionedTableException.java | 2 +- .../exception/NoSuchPartitionException.java | 12 +- .../exception/NoSuchPartitionKeyException.java | 29 ++++ .../tajo/catalog/partition/PartitionDesc.java | 50 ++---- .../tajo/catalog/partition/PartitionKey.java | 147 ----------------- .../catalog/store/TestHiveCatalogStore.java | 10 +- .../org/apache/tajo/catalog/store/MemStore.java | 6 +- .../org/apache/tajo/catalog/TestCatalog.java | 14 +- .../util/graph/TestSimpleDirectedGraph.java | 7 +- .../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +- .../apache/tajo/engine/parser/SQLAnalyzer.java | 1 + .../apache/tajo/master/exec/DDLExecutor.java | 111 ++++++++++++- .../tajo/ws/rs/resources/QueryResource.java | 25 ++- .../tajo/ws/rs/resources/TablesResource.java | 109 ++---------- .../ws/rs/responses/GetSubmitQueryResponse.java | 54 ++++++ .../org/apache/tajo/cli/tsql/TestTajoCli.java | 30 ++-- .../org/apache/tajo/client/TestTajoClient.java | 62 +++---- .../tajo/engine/parser/TestSQLAnalyzer.java | 4 + .../tajo/engine/planner/TestLogicalPlanner.java | 104 +++++++++++- .../planner/physical/TestExternalSortExec.java | 3 - .../physical/TestProgressExternalSortExec.java | 2 - .../tajo/engine/query/TestAlterTable.java | 47 ++++++ .../apache/tajo/engine/query/TestSortQuery.java | 2 - .../tajo/engine/query/TestUnionQuery.java | 165 +++++++++++++++++++ .../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 13 +- .../tajo/querymaster/TestTaskStatusUpdate.java | 3 - .../org/apache/tajo/storage/TestRowFile.java | 8 +- .../tajo/util/metrics/TestSystemMetrics.java | 4 +- .../tajo/worker/TestNodeResourceManager.java | 3 - .../tajo/ws/rs/resources/TestQueryResource.java | 50 +++--- .../rs/resources/TestQueryResultResource.java | 14 +- .../ws/rs/resources/TestTablesResource.java | 126 +++++++------- .../alter_table_add_partition1.sql | 1 + .../alter_table_drop_partition1.sql | 1 + .../TestAlterTable/create_partitioned_table.sql | 1 + .../testDistinctAggregationWithUnion1.sql | 2 +- .../TestJoinQuery/testComplexJoinCondition6.sql | 2 +- .../TestJoinQuery/testComplexJoinCondition7.sql | 2 +- .../testLeftOuterJoinWithEmptyTable4.sql | 2 +- .../queries/TestTablePartitions/case2.sql | 2 +- .../queries/TestUnionQuery/testUnion10.sql | 8 +- .../queries/TestUnionQuery/testUnion11.sql | 2 +- .../queries/TestUnionQuery/testUnion12.sql | 3 +- .../queries/TestUnionQuery/testUnion13.sql | 3 +- .../queries/TestUnionQuery/testUnion14.sql | 2 +- .../queries/TestUnionQuery/testUnion15.sql | 2 +- .../queries/TestUnionQuery/testUnion16.sql | 2 +- .../queries/TestUnionQuery/testUnion6.sql | 2 +- .../queries/TestUnionQuery/testUnion8.sql | 2 +- .../queries/TestUnionQuery/testUnion9.sql | 2 +- .../queries/TestUnionQuery/testUnionAll1.sql | 19 +++ .../queries/TestUnionQuery/testUnionAll10.sql | 158 ++++++++++++++++++ .../queries/TestUnionQuery/testUnionAll11.sql | 8 + .../queries/TestUnionQuery/testUnionAll12.sql | 13 ++ .../queries/TestUnionQuery/testUnionAll13.sql | 14 ++ .../queries/TestUnionQuery/testUnionAll14.sql | 8 + .../queries/TestUnionQuery/testUnionAll15.sql | 15 ++ .../queries/TestUnionQuery/testUnionAll16.sql | 15 ++ .../queries/TestUnionQuery/testUnionAll2.sql | 18 ++ .../queries/TestUnionQuery/testUnionAll3.sql | 19 +++ .../queries/TestUnionQuery/testUnionAll4.sql | 16 ++ .../queries/TestUnionQuery/testUnionAll5.sql | 20 +++ .../queries/TestUnionQuery/testUnionAll6.sql | 15 ++ .../queries/TestUnionQuery/testUnionAll7.sql | 18 ++ .../queries/TestUnionQuery/testUnionAll8.sql | 24 +++ .../queries/TestUnionQuery/testUnionAll9.sql | 29 ++++ .../testUnionAllWithDifferentAlias.sql | 29 ++++ ...estUnionAllWithDifferentAliasAndFunction.sql | 29 ++++ .../testUnionAllWithSameAliasNames.sql | 25 +++ .../testUnionWithDifferentAlias.sql | 2 +- .../testUnionWithDifferentAliasAndFunction.sql | 2 +- .../default/alter_table_drop_partition_3.sql | 2 +- .../testAlterTableAddDropPartition.result | 9 + .../testAlterTableAddPartition.result | 2 - .../testAlterTableDropPartition.result | 2 - .../results/TestUnionQuery/testUnion1.result | 3 - .../results/TestUnionQuery/testUnion10.result | 15 -- .../results/TestUnionQuery/testUnion2.result | 7 - .../results/TestUnionQuery/testUnion6.result | 2 +- .../results/TestUnionQuery/testUnion7.result | 7 - .../results/TestUnionQuery/testUnionAll1.result | 10 ++ .../TestUnionQuery/testUnionAll10.result | 22 +++ .../TestUnionQuery/testUnionAll11.result | 3 + .../TestUnionQuery/testUnionAll12.result | 6 + .../TestUnionQuery/testUnionAll13.result | 6 + .../TestUnionQuery/testUnionAll14.result | 9 + .../TestUnionQuery/testUnionAll15.result | 5 + .../TestUnionQuery/testUnionAll16.result | 5 + .../results/TestUnionQuery/testUnionAll2.result | 12 ++ .../results/TestUnionQuery/testUnionAll3.result | 4 + .../results/TestUnionQuery/testUnionAll4.result | 3 + .../results/TestUnionQuery/testUnionAll5.result | 3 + .../results/TestUnionQuery/testUnionAll6.result | 3 + .../results/TestUnionQuery/testUnionAll7.result | 12 ++ .../results/TestUnionQuery/testUnionAll8.result | 3 + .../results/TestUnionQuery/testUnionAll9.result | 7 + .../testUnionAllWithDifferentAlias.result | 4 + ...UnionAllWithDifferentAliasAndFunction.result | 7 + .../testUnionAllWithSameAliasNames.result | 12 ++ .../testUnionWithSameAliasNames.result | 7 - tajo-docs/pom.xml | 4 +- tajo-docs/src/main/sphinx/sql_language.rst | 3 +- .../main/sphinx/sql_language/alter_table.rst | 100 +++++++++++ .../java/org/apache/tajo/plan/LogicalPlan.java | 9 + .../tajo/plan/LogicalPlanPreprocessor.java | 1 + .../org/apache/tajo/plan/LogicalPlanner.java | 138 +++++++++++++++- .../tajo/plan/logical/AlterTableNode.java | 53 ++++-- .../tajo/plan/logical/SetOperationNode.java | 36 ++++ .../org/apache/tajo/plan/logical/UnionNode.java | 6 +- .../plan/serder/LogicalNodeDeserializer.java | 19 +++ .../tajo/plan/serder/LogicalNodeSerializer.java | 29 ++++ .../plan/verifier/PreLogicalPlanVerifier.java | 6 - tajo-plan/src/main/proto/Plan.proto | 10 ++ tajo-project/pom.xml | 17 +- tajo-project/src/site/apt/index.apt | 6 + tajo-project/src/site/markdown/downloads.md | 14 +- .../markdown/releases/0.10.1/announcement.md | 42 +++++ .../resources/releases/0.10.1/relnotes.html | 152 +++++++++++++++++ tajo-project/src/site/site.xml | 19 ++- .../ws/rs/netty/NettyRestHandlerContainer.java | 4 + .../ws/rs/netty/NettyRestServerListener.java | 18 +- .../tajo/storage/TestDelimitedTextFile.java | 5 +- 128 files changed, 2197 insertions(+), 591 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/CHANGES ---------------------------------------------------------------------- diff --cc CHANGES index 7bea70c,9d3fbe2..7ad6d51 --- a/CHANGES +++ b/CHANGES @@@ -166,12 -172,8 +172,15 @@@ Release 0.11.0 - unrelease BUG FIXES + TAJO-1608: Fix test failure in index_support branch. (jihoon) + + TAJO-1594: Catalog schema is invalid for some databases. (jihoon) + + TAJO-1657: Tajo Rest API /database/{database-name]/tables should return table + names only without invalid external table info. ++ + TAJO-1657: Tajo Rest API /database/{database-name]/tables should return + table names only without invalid external table info. (Contributed by DaeMyung Kang, Committed by jihoon) TAJO-1644: When inserting empty data into a partitioned table, http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 29eb216,ecf8890..1247482 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@@ -26,8 -27,8 +27,9 @@@ import org.apache.tajo.catalog.partitio import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; + import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidOperationException; http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --cc tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 884b97d,2d52e9c..aa1f1a0 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@@ -19,21 -19,23 +19,18 @@@ package org.apache.tajo.catalog; import com.google.common.collect.Sets; -- import org.apache.hadoop.fs.Path; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.dictionary.InfoSchemaMetadataDictionary; import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.catalog.partition.PartitionDesc; - import org.apache.tajo.catalog.partition.PartitionKey; --import org.apache.tajo.catalog.store.PostgreSQLStore; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; - import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; + import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; -import org.apache.tajo.catalog.store.DerbyStore; -import org.apache.tajo.catalog.store.MySQLStore; -import org.apache.tajo.catalog.store.MariaDBStore; -import org.apache.tajo.catalog.store.OracleStore; +import org.apache.tajo.catalog.store.*; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java index 481d156,e4ee4ea..2063c61 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java @@@ -260,21 -262,29 +262,30 @@@ public class QueryResource return ResourcesUtil.createBadRequestResponse(LOG, "Provided session id (" + sessionId + ") is invalid."); } -- SubmitQueryResponse response = - masterContext.getGlobalEngine().executeQuery(session, request.getQuery(), false); - if (response.hasResultCode() && ClientProtos.ResultCode.ERROR.equals(response.getResultCode())) { - return ResourcesUtil.createExceptionResponse(LOG, response.getErrorMessage()); ++ SubmitQueryResponse response = + masterContext.getGlobalEngine().executeQuery(session, request.getQuery(), false); + if (response.getResult().hasResultCode() && + ClientProtos.ResultCode.ERROR.equals(response.getResult().getResultCode())) { + return ResourcesUtil.createExceptionResponse(LOG, response.getResult().getErrorMessage()); } else { JerseyResourceDelegateContextKey<UriInfo> uriInfoKey = - JerseyResourceDelegateContextKey.valueOf(JerseyResourceDelegateUtil.UriInfoKey, UriInfo.class); + JerseyResourceDelegateContextKey.valueOf(JerseyResourceDelegateUtil.UriInfoKey, UriInfo.class); UriInfo uriInfo = context.get(uriInfoKey); - + + QueryId queryId = new QueryId(response.getQueryId()); URI queryURI = uriInfo.getBaseUriBuilder() - .path(QueryResource.class) - .path(QueryResource.class, "getQuery") - .build(new QueryId(response.getQueryId()).toString()); - return Response.created(queryURI).build(); + .path(QueryResource.class) + .path(QueryResource.class, "getQuery") + .build(queryId.toString()); + + GetSubmitQueryResponse queryResponse = new GetSubmitQueryResponse(); + if (queryId.isNull() == false) { + queryResponse.setUri(queryURI); + } + - queryResponse.setResultCode(response.getResultCode()); ++ queryResponse.setResultCode(response.getResult().getResultCode()); + queryResponse.setQuery(request.getQuery()); + return Response.status(Status.OK).entity(queryResponse).build(); } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index 16cc287,a6a3dbe..61b047a --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@@ -157,10 -157,15 +159,17 @@@ public class LogicalPlan } /** + * It generates a unique table subquery name + */ + public String generateUniqueSubQueryName() { + return NONAME_SUBQUERY_PREFIX + noNameSubqueryId++; + } + + /** * It generates an unique column name from Expr. It is usually used for an expression or predicate without * a specified name (i.e., alias). + * Here, some expressions require to be identified with their names in the future. + * For example, expressions must be identifiable with their names when getting targets in {@link LogicalPlanner#visitCreateIndex}. */ public String generateUniqueColumnName(Expr expr) { String generatedName; http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/15749c5a/tajo-plan/src/main/proto/Plan.proto ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/proto/Plan.proto index 811a0f1,7855acd..6344f73 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@@ -318,25 -310,9 +327,26 @@@ message AlterTableNode optional RenameColumn renameColumn = 4; optional AddColumn addColumn = 5; optional KeyValueSetProto properties = 6; + optional AlterPartition alterPartition = 7; } +message CreateIndexNode { + required int32 childSeq = 1; + required string indexName = 2; + required IndexMethod indexMethod = 3; + required string indexPath = 4; + repeated SortSpecProto keySortSpecs = 5; + required SchemaProto targetRelationSchema = 6; + optional bool isUnique = 7 [default = false]; + optional bool isClustered = 8 [default = false]; + optional KeyValueSetProto indexProperties = 9; + optional bool isExternal = 10; +} + +message DropIndexNode { + required string indexName = 1; +} + enum EvalType { NOT = 0; AND = 1;
