Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/49367117 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/49367117 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/49367117 Branch: refs/heads/index_support Commit: 49367117bba1f98f96c4a9c16e2d227ee3de884b Parents: 3eac157 6d85208 Author: Jihoon Son <[email protected]> Authored: Wed Jul 22 22:00:08 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Wed Jul 22 22:00:08 2015 +0900 ---------------------------------------------------------------------- CHANGES | 20 + .../org/apache/tajo/catalog/CatalogUtil.java | 25 +- .../org/apache/tajo/catalog/DDLBuilder.java | 2 +- .../exception/AmbiguousTableException.java | 30 + .../src/main/proto/CatalogProtos.proto | 9 +- .../org/apache/tajo/catalog/TestTableDesc.java | 4 +- .../org/apache/tajo/catalog/TestTableMeta.java | 18 +- .../tajo/catalog/store/HiveCatalogStore.java | 3 +- .../tajo/catalog/store/HiveCatalogUtil.java | 4 +- .../catalog/store/TestHiveCatalogStore.java | 12 +- .../org/apache/tajo/catalog/TestCatalog.java | 24 +- .../org/apache/tajo/storage/RowStoreUtil.java | 1 - .../java/org/apache/tajo/BuiltinStorages.java | 1 - .../java/org/apache/tajo/conf/TajoConf.java | 4 +- .../exception/AmbiguousColumnException.java | 32 + .../apache/tajo/exception/ErrorMessages.java | 3 + .../tajo/exception/TajoInternalError.java | 4 + .../tajo/exception/TajoRuntimeException.java | 5 + .../java/org/apache/tajo/storage/NullTuple.java | 4 + .../apache/tajo/storage/StorageConstants.java | 9 +- .../java/org/apache/tajo/storage/Tuple.java | 2 + .../java/org/apache/tajo/storage/VTuple.java | 7 + .../java/org/apache/tajo/benchmark/TPCH.java | 2 +- .../engine/codegen/ExecutorPreCompiler.java | 30 +- .../apache/tajo/engine/parser/SQLAnalyzer.java | 6 +- .../tajo/engine/planner/KeyProjector.java | 44 ++ .../tajo/engine/planner/PhysicalPlanner.java | 4 +- .../apache/tajo/engine/planner/Projector.java | 29 +- .../engine/planner/global/GlobalPlanner.java | 77 ++- .../global/builder/DistinctGroupbyBuilder.java | 11 +- .../rewriter/GlobalPlanRewriteEngine.java | 4 +- .../global/rewriter/GlobalPlanRewriteRule.java | 3 +- .../rewriter/rules/BroadcastJoinRule.java | 3 +- .../planner/physical/AggregationExec.java | 11 - .../engine/planner/physical/BNLJoinExec.java | 31 +- .../planner/physical/BSTIndexScanExec.java | 28 +- .../planner/physical/CommonHashJoinExec.java | 70 +-- .../engine/planner/physical/CommonJoinExec.java | 13 +- .../planner/physical/ComparableVector.java | 8 - .../DistinctGroupbyFirstAggregationExec.java | 179 +++--- .../DistinctGroupbyHashAggregationExec.java | 177 +++--- .../DistinctGroupbySecondAggregationExec.java | 108 ++-- .../DistinctGroupbySortAggregationExec.java | 27 +- .../DistinctGroupbyThirdAggregationExec.java | 54 +- .../planner/physical/ExternalSortExec.java | 34 +- .../planner/physical/HashAggregateExec.java | 21 +- .../planner/physical/HashFullOuterJoinExec.java | 35 +- .../engine/planner/physical/HashJoinExec.java | 19 +- .../planner/physical/HashLeftAntiJoinExec.java | 11 +- .../planner/physical/HashLeftOuterJoinExec.java | 11 +- .../planner/physical/HashLeftSemiJoinExec.java | 11 +- .../physical/HashShuffleFileWriteExec.java | 25 +- .../tajo/engine/planner/physical/KeyTuple.java | 85 +++ .../engine/planner/physical/MemSortExec.java | 11 +- .../physical/MergeFullOuterJoinExec.java | 77 ++- .../engine/planner/physical/MergeJoinExec.java | 72 +-- .../engine/planner/physical/NLJoinExec.java | 12 +- .../planner/physical/PhysicalPlanUtil.java | 4 +- .../engine/planner/physical/ProjectionExec.java | 5 +- .../physical/RangeShuffleFileWriteExec.java | 42 +- .../physical/RightOuterMergeJoinExec.java | 102 ++-- .../engine/planner/physical/SeqScanExec.java | 3 +- .../planner/physical/SortAggregateExec.java | 40 +- .../SortBasedColPartitionStoreExec.java | 3 +- .../tajo/engine/planner/physical/SortExec.java | 3 +- .../engine/planner/physical/StoreTableExec.java | 2 +- .../tajo/engine/planner/physical/TupleList.java | 44 ++ .../tajo/engine/planner/physical/TupleMap.java | 74 +++ .../tajo/engine/planner/physical/TupleSet.java | 34 ++ .../engine/planner/physical/TupleSorter.java | 7 +- .../planner/physical/VectorizedSorter.java | 3 +- .../engine/planner/physical/WindowAggExec.java | 63 +- .../apache/tajo/engine/query/QueryContext.java | 7 +- .../apache/tajo/engine/utils/CacheHolder.java | 13 +- .../org/apache/tajo/engine/utils/TupleUtil.java | 21 - .../utils/test/ErrorInjectionRewriter.java | 5 +- .../exec/ExplainPlanPreprocessorForTest.java | 13 +- .../NonForwardQueryResultSystemScanner.java | 14 +- .../apache/tajo/master/exec/QueryExecutor.java | 9 +- .../java/org/apache/tajo/querymaster/Stage.java | 2 +- .../resource/DefaultResourceCalculator.java | 6 +- .../worker/ExecutionBlockSharedResource.java | 4 +- .../main/java/org/apache/tajo/worker/Task.java | 3 +- .../org/apache/tajo/worker/TaskContainer.java | 8 +- .../org/apache/tajo/worker/TaskExecutor.java | 29 +- .../java/org/apache/tajo/worker/TaskImpl.java | 7 +- .../org/apache/tajo/BackendTestingUtil.java | 2 +- .../java/org/apache/tajo/QueryTestCaseBase.java | 44 +- .../org/apache/tajo/TajoTestingCluster.java | 3 +- .../apache/tajo/cli/tools/TestDDLBuilder.java | 2 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 5 +- .../apache/tajo/engine/eval/ExprTestBase.java | 13 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 38 +- .../engine/planner/TestLogicalOptimizer.java | 56 +- .../tajo/engine/planner/TestLogicalPlanner.java | 96 +-- .../tajo/engine/planner/TestPlannerUtil.java | 10 +- .../engine/planner/TestQueryValidation.java | 8 +- .../planner/physical/TestBNLJoinExec.java | 17 +- .../planner/physical/TestExternalSortExec.java | 11 +- .../physical/TestFullOuterHashJoinExec.java | 30 +- .../physical/TestFullOuterMergeJoinExec.java | 29 +- .../planner/physical/TestHashAntiJoinExec.java | 8 +- .../planner/physical/TestHashJoinExec.java | 17 +- .../planner/physical/TestHashSemiJoinExec.java | 8 +- .../physical/TestLeftOuterHashJoinExec.java | 27 +- .../planner/physical/TestMergeJoinExec.java | 8 +- .../engine/planner/physical/TestNLJoinExec.java | 14 +- .../planner/physical/TestPhysicalPlanner.java | 58 +- .../physical/TestRightOuterHashJoinExec.java | 14 +- .../physical/TestRightOuterMergeJoinExec.java | 36 +- .../engine/planner/physical/TestSortExec.java | 6 +- .../planner/physical/TestSortIntersectExec.java | 6 +- .../planner/physical/TestTupleSorter.java | 11 +- .../apache/tajo/engine/query/TestCTASQuery.java | 2 +- .../tajo/engine/query/TestCreateTable.java | 4 +- .../tajo/engine/query/TestGroupByQuery.java | 28 +- .../tajo/engine/query/TestHBaseTable.java | 36 +- .../tajo/engine/query/TestInsertQuery.java | 2 +- .../query/TestJoinOnPartitionedTables.java | 9 +- .../tajo/engine/query/TestTablePartitions.java | 32 +- .../org/apache/tajo/jdbc/TestResultSet.java | 2 +- .../tajo/master/TestExecutionBlockCursor.java | 2 +- .../apache/tajo/querymaster/TestKillQuery.java | 2 +- .../apache/tajo/worker/MockTaskExecutor.java | 3 +- .../apache/tajo/worker/TestTaskExecutor.java | 3 +- .../queries/TestAlterTable/table1_ddl.sql | 2 +- .../queries/TestAlterTable/table2_ddl.sql | 2 +- .../queries/TestCTASQuery/CtasWithOptions.sql | 2 +- .../quoted_identifier_mixed_chars_ddl_1.sql | 2 +- .../quoted_identifier_mixed_chars_ddl_2.sql | 2 +- .../quoted_identifier_non_ascii_ddl.sql | 2 +- .../queries/TestCreateTable/table1_ddl.sql | 2 +- .../TestGroupByQuery/testGroupbyWithJson.json | 101 ++-- .../TestGroupByQuery/testGroupbyWithLimit3.sql | 2 +- ...stInsertOverwriteLocationWithCompression.sql | 2 +- .../testInsertOverwriteWithCompression_ddl.sql | 4 +- .../TestJoinQuery/create_customer_large_ddl.sql | 2 +- .../TestJoinQuery/create_lineitem_large_ddl.sql | 2 +- .../TestJoinQuery/create_orders_large_ddl.sql | 2 +- .../queries/TestJoinQuery/oj_table1_ddl.sql | 4 +- .../queries/TestJoinQuery/oj_table2_ddl.sql | 4 +- .../queries/TestJoinQuery/table1_int4_ddl.sql | 4 +- .../queries/TestJoinQuery/table1_int8_ddl.sql | 4 +- .../queries/TestNetTypes/table1_ddl.sql | 4 +- .../queries/TestNetTypes/table2_ddl.sql | 4 +- .../testNestedFieldAsGroupbyKey1.sql | 2 + .../multibytes_delimiter_table1_ddl.sql | 4 +- .../multibytes_delimiter_table2_ddl.sql | 4 +- .../create_table_with_asc_desc_keys.sql | 2 +- .../create_table_with_date_ddl.sql | 4 +- .../TestTablePartitions/lineitemspecial_ddl.sql | 2 +- .../TestTajoJdbc/create_table_with_date_ddl.sql | 4 +- .../queries/TestTruncateTable/table2_ddl.sql | 2 +- .../queries/default/create_table_12.sql | 2 +- .../TestGroupByQuery/testGroupbyWithJson.result | 4 +- .../testGroupbyWithLimit3.result | 2 +- .../testGroupbyWithPythonFunc.result | 2 +- .../testGroupbyWithPythonFunc2.result | 4 +- .../testHavingWithAggFunction.result | 2 +- .../TestGroupByQuery/testPythonUdaf3.result | 2 +- .../testNestedFieldAsGroupbyKey1.result | 4 +- .../results/TestTajoCli/testDescTable.result | 4 +- .../testDescTableForNestedSchema.result | 4 +- .../results/TestTajoDump/testDump1.result | 4 +- .../results/TestTajoDump/testDump2.result | 4 +- .../testWindowWithAggregation4.result | 4 +- .../testWindowWithAggregation6.result | 4 +- .../testBuildDDLForBaseTable.result | 4 +- .../testBuildDDLForExternalTable.result | 4 +- .../testBuildDDLQuotedTableName1.result | 4 +- .../testBuildDDLQuotedTableName2.result | 4 +- .../main/sphinx/backup_and_restore/catalog.rst | 2 +- tajo-docs/src/main/sphinx/getting_started.rst | 2 +- tajo-docs/src/main/sphinx/index/how_to_use.rst | 2 +- tajo-docs/src/main/sphinx/sql_language.rst | 7 +- .../main/sphinx/sql_language/alter_table.rst | 2 +- .../src/main/sphinx/sql_language/joins.rst | 160 +++++ .../src/main/sphinx/sql_language/queries.rst | 2 + .../src/main/sphinx/table_management/text.rst | 27 +- tajo-docs/src/main/sphinx/tsql/meta_command.rst | 2 +- tajo-docs/src/main/sphinx/tsql/variables.rst | 4 - .../org/apache/tajo/jdbc/MetaDataTuple.java | 5 + .../org/apache/tajo/plan/ExprAnnotator.java | 138 ++--- .../org/apache/tajo/plan/ExprNormalizer.java | 21 +- .../org/apache/tajo/plan/LogicalOptimizer.java | 37 +- .../java/org/apache/tajo/plan/LogicalPlan.java | 18 +- .../tajo/plan/LogicalPlanPreprocessor.java | 62 +- .../org/apache/tajo/plan/LogicalPlanner.java | 172 +++--- .../org/apache/tajo/plan/NamedExprsManager.java | 11 +- .../org/apache/tajo/plan/TablePropertyUtil.java | 4 +- .../org/apache/tajo/plan/TypeDeterminant.java | 40 +- .../tajo/plan/algebra/AlgebraVisitor.java | 138 ++--- .../plan/algebra/AmbiguousFieldException.java | 32 - .../tajo/plan/algebra/BaseAlgebraVisitor.java | 148 ++--- .../apache/tajo/plan/expr/AlgebraicUtil.java | 15 +- .../GreedyHeuristicJoinOrderAlgorithm.java | 10 +- .../apache/tajo/plan/joinorder/JoinGraph.java | 2 +- .../tajo/plan/joinorder/JoinOrderAlgorithm.java | 3 +- .../tajo/plan/logical/PersistentStoreNode.java | 2 +- .../tajo/plan/nameresolver/NameResolver.java | 46 +- .../plan/nameresolver/ResolverByLegacy.java | 8 +- .../tajo/plan/nameresolver/ResolverByRels.java | 3 +- .../nameresolver/ResolverByRelsAndSubExprs.java | 3 +- .../nameresolver/ResolverBySubExprsAndRels.java | 3 +- .../rewrite/BaseLogicalPlanRewriteEngine.java | 5 +- .../plan/rewrite/LogicalPlanRewriteEngine.java | 4 +- .../plan/rewrite/LogicalPlanRewriteRule.java | 4 +- .../plan/rewrite/rules/AccessPathRewriter.java | 6 +- .../plan/rewrite/rules/FilterPushDownRule.java | 64 +- .../rules/LogicalPlanEqualityTester.java | 4 +- .../rewrite/rules/PartitionedTableRewriter.java | 17 +- .../rewrite/rules/ProjectionPushDownRule.java | 61 +- .../tajo/plan/serder/LogicalNodeSerializer.java | 57 +- .../org/apache/tajo/plan/util/ExprFinder.java | 8 +- .../org/apache/tajo/plan/util/PlannerUtil.java | 46 +- .../tajo/plan/verifier/ExprsVerifier.java | 3 +- .../tajo/plan/verifier/LogicalPlanVerifier.java | 44 +- .../plan/verifier/PreLogicalPlanVerifier.java | 34 +- .../plan/visitor/BasicLogicalPlanVisitor.java | 68 +-- .../plan/visitor/ExplainLogicalPlanVisitor.java | 50 +- .../tajo/plan/visitor/LogicalPlanVisitor.java | 62 +- .../tajo/plan/visitor/SimpleAlgebraVisitor.java | 58 +- .../org/apache/tajo/plan/TestLogicalNode.java | 6 +- .../org/apache/tajo/storage/FrameTuple.java | 5 + .../java/org/apache/tajo/storage/LazyTuple.java | 6 + .../org/apache/tajo/storage/RowStoreUtil.java | 2 +- .../org/apache/tajo/storage/Tablespace.java | 6 +- .../apache/tajo/tuple/offheap/HeapTuple.java | 4 + .../apache/tajo/tuple/offheap/UnSafeTuple.java | 4 + .../src/main/resources/storage-default.xml | 18 +- .../src/test/resources/storage-default.xml | 63 +- .../tajo/storage/hbase/HBaseFragment.java | 7 +- .../apache/tajo/storage/hbase/HBaseScanner.java | 9 +- .../tajo/storage/hbase/HBaseTablespace.java | 4 +- .../storage/hbase/SortedInsertRewriter.java | 9 +- .../java/org/apache/tajo/storage/CSVFile.java | 589 ------------------- .../tajo/storage/CompressedSplitLineReader.java | 182 ------ .../org/apache/tajo/storage/LineReader.java | 559 ------------------ .../java/org/apache/tajo/storage/RawFile.java | 42 +- .../java/org/apache/tajo/storage/RowFile.java | 3 +- .../apache/tajo/storage/SplitLineReader.java | 39 -- .../apache/tajo/storage/avro/AvroScanner.java | 26 +- .../tajo/storage/fragment/FileFragment.java | 7 +- .../apache/tajo/storage/index/bst/BSTIndex.java | 20 +- .../org/apache/tajo/storage/rcfile/RCFile.java | 11 +- .../sequencefile/SequenceFileScanner.java | 50 +- .../tajo/storage/text/ByteBufLineReader.java | 2 +- .../tajo/storage/text/CSVLineSerializer.java | 1 + .../tajo/storage/text/DelimitedTextFile.java | 39 +- .../apache/tajo/storage/text/TextLineSerDe.java | 14 +- .../tajo/storage/TestCompressionStorages.java | 17 +- .../tajo/storage/TestDelimitedTextFile.java | 53 ++ .../apache/tajo/storage/TestFileSystems.java | 2 +- .../apache/tajo/storage/TestFileTablespace.java | 6 +- .../apache/tajo/storage/TestMergeScanner.java | 7 +- .../org/apache/tajo/storage/TestStorages.java | 9 +- .../apache/tajo/storage/index/TestBSTIndex.java | 1 - .../index/TestSingleCSVFileBSTIndex.java | 18 +- .../TestDelimitedTextFile/testNormal.json | 6 + .../dataset/TestDelimitedTextFile/testSkip.txt | 7 + .../src/test/resources/storage-default.xml | 18 +- 261 files changed, 2958 insertions(+), 3716 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/CHANGES ---------------------------------------------------------------------- diff --cc CHANGES index 4f61eb2,21799d2..8afa627 --- a/CHANGES +++ b/CHANGES @@@ -177,13 -188,8 +188,15 @@@ Release 0.11.0 - unrelease BUG FIXES + TAJO-1608: Fix test failure in index_support branch. (jihoon) + + TAJO-1594: Catalog schema is invalid for some databases. (jihoon) + + TAJO-1657: Tajo Rest API /database/{database-name]/tables should return table + names only without invalid external table info. + + TAJO-1697: RCFile progress causes NPE occasionally. (jinho) + TAJO-1689: Metrics file reporter prints histogram metric without group name. (hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index f2e7c59,b7244b0..7ba779e --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@@ -26,11 -27,10 +27,11 @@@ import org.apache.tajo.TajoConstants import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; + import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableIdentifierProto; - import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidOperationException; http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --cc tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 6872b59,40b731e..1206bfa --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@@ -545,9 -561,9 +545,9 @@@ public class TestCatalog String tableName = "indexed"; - TableMeta meta = CatalogUtil.newTableMeta("CSV"); + TableMeta meta = CatalogUtil.newTableMeta("TEXT"); return new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), schema, meta, + CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), relationSchema, meta, new Path(CommonTestingUtil.getTestDir(), "indexed").toUri()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java index d04d082,87d33de..fbd88ad --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java @@@ -214,11 -214,4 +214,11 @@@ public class ExecutorPreCompiler extend return node; } + + @Override + public LogicalNode visitIndexScan(CompilationContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, - IndexScanNode node, Stack<LogicalNode> stack) throws PlanningException { ++ IndexScanNode node, Stack<LogicalNode> stack) throws TajoException { + visitScan(context, plan, block, node, stack); + return node; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 1c430ce,d9712c9..67f782a --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@@ -1365,18 -1362,9 +1362,18 @@@ public class GlobalPlanner } @Override + public LogicalNode visitIndexScan(GlobalPlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, - IndexScanNode node, Stack<LogicalNode> stack) throws PlanningException { ++ IndexScanNode node, Stack<LogicalNode> stack) throws TajoException { + ExecutionBlock newBlock = context.plan.newExecutionBlock(); + newBlock.setPlan(node); + context.execBlockMap.put(node.getPID(), newBlock); + return node; + } + + @Override public LogicalNode visitPartitionedTableScan(GlobalPlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, PartitionedTableScanNode node, - Stack<LogicalNode> stack)throws PlanningException { + Stack<LogicalNode> stack)throws TajoException { ExecutionBlock newExecBlock = context.plan.newExecutionBlock(); newExecBlock.setPlan(node); context.execBlockMap.put(node.getPID(), newExecBlock); @@@ -1419,19 -1407,5 +1416,19 @@@ return node; } + + @Override + public LogicalNode visitCreateIndex(GlobalPlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock queryBlock, - CreateIndexNode node, Stack<LogicalNode> stack) throws PlanningException { ++ CreateIndexNode node, Stack<LogicalNode> stack) throws TajoException { + LogicalNode child = super.visitCreateIndex(context, plan, queryBlock, node, stack); + + // Don't separate execution block. CreateIndex is pushed to the first execution block. + ExecutionBlock childBlock = context.execBlockMap.remove(child.getPID()); + node.setChild(childBlock.getPlan()); + childBlock.setPlan(node); + context.execBlockMap.put(node.getPID(), childBlock); + + return node; + } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index 19da87b,28622d7..a59960f --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@@ -18,37 -18,23 +18,32 @@@ package org.apache.tajo.engine.planner.physical; - import org.apache.commons.logging.Log; - import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; +import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; - import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.datum.Datum; import org.apache.tajo.engine.planner.Projector; +import org.apache.tajo.plan.Target; import org.apache.tajo.plan.expr.EvalNode; -import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.plan.expr.EvalTreeUtil; +import org.apache.tajo.plan.logical.IndexScanNode; +import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate; - import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.*; -import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.index.bst.BSTIndex; +import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; +import java.net.URI; +import java.util.HashSet; +import java.util.Set; public class BSTIndexScanExec extends PhysicalExec { - private final static Log LOG = LogFactory.getLog(BSTIndexScanExec.class); - private ScanNode scanNode; + private IndexScanNode plan; private SeekableScanner fileScanner; private EvalNode qual; @@@ -62,37 -46,24 +55,39 @@@ private float progress; + private Tuple indexLookupKey; + - public BSTIndexScanExec(TaskAttemptContext context, ScanNode scanNode , - FileFragment fragment, Path fileName , Schema keySchema, - TupleComparator comparator , Datum[] datum) throws IOException { - super(context, scanNode.getInSchema(), scanNode.getOutSchema()); - this.scanNode = scanNode; - this.qual = scanNode.getQual(); - indexLookupKey = new VTuple(datum); - - this.fileScanner = OldStorageManager.getSeekableScanner(context.getConf(), - scanNode.getTableDesc().getMeta(), scanNode.getInSchema(), fragment, outSchema); - this.fileScanner.init(); - this.projector = new Projector(context, inSchema, outSchema, scanNode.getTargets()); - - FileSystem fs = fileName.getFileSystem(context.getConf()); - this.reader = new BSTIndex(fs.getConf()). - getIndexReader(fileName, keySchema, comparator); + private TableStats inputStats; + + private CatalogProtos.FragmentProto fragment; + + private Schema keySchema; + + public BSTIndexScanExec(TaskAttemptContext context, IndexScanNode plan, + CatalogProtos.FragmentProto fragment, URI indexPrefix , Schema keySchema, + SimplePredicate [] predicates) throws IOException { + super(context, plan.getInSchema(), plan.getOutSchema()); + this.plan = plan; + this.qual = plan.getQual(); + this.fragment = fragment; + this.keySchema = keySchema; + + SortSpec[] keySortSpecs = new SortSpec[predicates.length]; - values = new Datum[predicates.length]; ++ Datum[] values = new Datum[predicates.length]; + for (int i = 0; i < predicates.length; i++) { + keySortSpecs[i] = predicates[i].getKeySortSpec(); + values[i] = predicates[i].getValue(); + } ++ indexLookupKey = new VTuple(values); + + TupleComparator comparator = new BaseTupleComparator(keySchema, + keySortSpecs); + - + this.projector = new Projector(context, inSchema, outSchema, plan.getTargets()); + + Path indexPath = new Path(indexPrefix.toString(), context.getUniqueKeyFromFragments()); + this.reader = new BSTIndex(context.getConf()). + getIndexReader(indexPath, keySchema, comparator); this.reader.open(); } @@@ -209,28 -102,21 +202,25 @@@ fileScanner.seek(offset); } } + Tuple tuple; - Tuple outTuple = new VTuple(outColumnNum); - if (!scanNode.hasQual()) { + if (!plan.hasQual()) { if ((tuple = fileScanner.next()) != null) { - projector.eval(tuple, outTuple); - return outTuple; + return projector.eval(tuple); } else { return null; } } else { while(reader.isCurInMemory() && (tuple = fileScanner.next()) != null) { if (qual.eval(tuple).isTrue()) { - projector.eval(tuple, outTuple); - return outTuple; + return projector.eval(tuple); } else { long offset = reader.next(); - if (offset == -1) return null; + if (offset == -1) { + return null; + } else fileScanner.seek(offset); + return null; } } } @@@ -245,18 -131,12 +235,22 @@@ @Override public void close() throws IOException { IOUtils.cleanup(null, reader, fileScanner); + if (fileScanner != null) { + try { + TableStats stats = fileScanner.getInputStats(); + if (stats != null) { + inputStats = (TableStats) stats.clone(); + } + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + } reader = null; fileScanner = null; - scanNode = null; ++ plan = null; + qual = null; + projector = null; + indexLookupKey = null; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java index 9594b58,8a79005..488c3ac --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java @@@ -57,9 -54,7 +54,9 @@@ public class ProjectionExec extends Una return null; } - projector.eval(tuple, outTuple); - return projector.eval(tuple); ++ Tuple outTuple = projector.eval(tuple); + outTuple.setOffset(tuple.getOffset()); + return outTuple; } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java index 709aa81,6582513..54b6c5e --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java @@@ -18,11 -18,11 +18,10 @@@ package org.apache.tajo.engine.utils.test; --import org.apache.tajo.OverridableConf; + import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.LogicalPlan; --import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; @SuppressWarnings("unused") public class ErrorInjectionRewriter implements LogicalPlanRewriteRule { @@@ -37,7 -37,7 +36,7 @@@ } @Override - public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException { - public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws TajoException { ++ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws TajoException { throw new NullPointerException(); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index a330016,b6e5806..e3f2fa4 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@@ -24,12 -24,11 +24,9 @@@ import org.apache.commons.logging.LogFa import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; - import org.apache.tajo.QueryId; - import org.apache.tajo.QueryIdFactory; - import org.apache.tajo.SessionVars; - import org.apache.tajo.TajoConstants; + import org.apache.tajo.*; -import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.exception.DuplicateIndexException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes; @@@ -69,8 -68,8 +66,8 @@@ import java.net.URI import java.util.ArrayList; import java.util.List; --import static org.apache.tajo.exception.ReturnStateUtil.errUndefinedDatabase; import static org.apache.tajo.exception.ReturnStateUtil.OK; ++import static org.apache.tajo.exception.ReturnStateUtil.errUndefinedDatabase; public class QueryExecutor { private static final Log LOG = LogFactory.getLog(QueryExecutor.class); http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java ---------------------------------------------------------------------- diff --cc tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index f468ab7,7629711..abd0973 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@@ -35,7 -35,8 +35,8 @@@ import org.apache.tajo.engine.codegen.T import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.engine.query.QueryContext; + import org.apache.tajo.exception.TajoException; -import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.function.FunctionSignature; import org.apache.tajo.master.exec.QueryExecutor; import org.apache.tajo.plan.*; @@@ -43,7 -44,9 +44,6 @@@ import org.apache.tajo.plan.expr.EvalCo import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.serder.EvalNodeDeserializer; import org.apache.tajo.plan.serder.EvalNodeSerializer; -import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.catalog.SchemaUtil; --import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.plan.verifier.LogicalPlanVerifier; import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; import org.apache.tajo.plan.verifier.VerificationState; @@@ -57,6 -60,6 +57,7 @@@ import org.apache.tajo.util.KeyValueSet import org.apache.tajo.util.datetime.DateTimeUtil; import org.junit.AfterClass; import org.junit.BeforeClass; ++import org.apache.tajo.plan.serder.PlanProto; import java.io.IOException; import java.util.List; http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java ---------------------------------------------------------------------- diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 401d5d8,097e232..02e921a --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@@ -40,8 -40,12 +40,11 @@@ import org.apache.tajo.engine.function. import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.query.QueryContext; - import org.apache.tajo.session.Session; --import org.apache.tajo.plan.*; + import org.apache.tajo.exception.TajoException; + import org.apache.tajo.plan.LogicalOptimizer; + import org.apache.tajo.plan.LogicalPlan; + import org.apache.tajo.plan.LogicalPlanner; + import org.apache.tajo.plan.Target; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; @@@ -892,30 -896,7 +895,30 @@@ public class TestLogicalPlanner } @Test - public final void testCreateIndexNode() throws PlanningException { ++ public final void testCreateIndexNode() throws TajoException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(QUERIES[11]); + LogicalPlan rootNode = planner.createPlan(qc, expr); + LogicalNode plan = rootNode.getRootBlock().getRoot(); + testJsonSerDerObject(plan); + + LogicalRootNode root = (LogicalRootNode) plan; + assertEquals(NodeType.CREATE_INDEX, root.getChild().getType()); + CreateIndexNode createIndexNode = root.getChild(); + + assertEquals(NodeType.PROJECTION, createIndexNode.getChild().getType()); + ProjectionNode projNode = createIndexNode.getChild(); + + assertEquals(NodeType.SELECTION, projNode.getChild().getType()); + SelectionNode selNode = projNode.getChild(); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName()); + } + + @Test - public final void testAsterisk() throws CloneNotSupportedException, PlanningException { + public final void testAsterisk() throws CloneNotSupportedException, TajoException { QueryContext qc = createQueryContext(); Expr expr = sqlAnalyzer.parse(QUERIES[13]); http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 081eadc,dae2351..0d4e6a4 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@@ -970,11 -970,11 +970,11 @@@ public class TestPhysicalPlanner } public final String [] createIndexStmt = { - "create index idx_employee on employee using bst (name null first, empId desc)" + "create index idx_employee on employee using TWO_LEVEL_BIN_TREE (name null first, empId desc)" }; - //@Test + @Test - public final void testCreateIndex() throws IOException, PlanningException { + public final void testCreateIndex() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testCreateIndex"); http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java ---------------------------------------------------------------------- diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index a19ac34,743c51d..4df383e --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@@ -117,10 -115,10 +117,10 @@@ public class TestSortExec "select managerId, empId, deptName from employee order by managerId, empId desc" }; @Test - public final void testNext() throws IOException, PlanningException { + public final void testNext() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestSortExec"); - TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(queryContext, LocalTajoTestingUtility .newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-docs/src/main/sphinx/index/how_to_use.rst ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index 77a12e8,d06c8e2..fc52d86 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@@ -26,9 -26,9 +26,10 @@@ import org.apache.tajo.ConfigKey import org.apache.tajo.OverridableConf; import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; +import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; + import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.expr.AlgebraicUtil; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.expr.EvalTreeUtil; @@@ -80,8 -76,8 +81,8 @@@ public class LogicalOptimizer return optimize(conf, plan); } - public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws PlanningException { + public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws TajoException { - rulesBeforeJoinOpt.rewrite(context, plan); + rulesBeforeJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog)); DirectedGraphCursor<String, BlockEdge> blockCursor = new DirectedGraphCursor<String, BlockEdge>(plan.getQueryBlockGraph(), plan.getRootBlock().getName()); http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index 61b047a,9cc1e78..eab939d --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@@ -25,10 -25,8 +25,9 @@@ import org.apache.tajo.algebra.* import org.apache.tajo.annotation.NotThreadSafe; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; + import org.apache.tajo.exception.TajoException; + import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.plan.expr.AlgebraicUtil.IdentifiableNameBuilder; - import org.apache.tajo.plan.rewrite.rules.AccessPathInfo; - import org.apache.tajo.util.graph.DirectedGraphCursor; - import org.apache.tajo.util.graph.SimpleDirectedGraph; import org.apache.tajo.plan.expr.ConstEval; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; @@@ -37,8 -35,10 +36,11 @@@ import org.apache.tajo.plan.logical.Nod import org.apache.tajo.plan.logical.RelationNode; import org.apache.tajo.plan.nameresolver.NameResolver; import org.apache.tajo.plan.nameresolver.NameResolvingMode; ++import org.apache.tajo.plan.rewrite.rules.AccessPathInfo; import org.apache.tajo.plan.visitor.ExplainLogicalPlanVisitor; import org.apache.tajo.util.TUtil; + import org.apache.tajo.util.graph.DirectedGraphCursor; + import org.apache.tajo.util.graph.SimpleDirectedGraph; import java.lang.reflect.Constructor; import java.util.*; http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java index f697a2a,0a8d1c4..f3743ee --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java @@@ -473,25 -473,8 +473,25 @@@ public class LogicalPlanPreprocessor ex } @Override + public LogicalNode visitCreateIndex(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, CreateIndex expr) - throws PlanningException { ++ throws TajoException { + stack.push(expr); + LogicalNode child = visit(ctx, stack, expr.getChild()); + stack.pop(); + + CreateIndexNode createIndex = ctx.plan.createNode(CreateIndexNode.class); + createIndex.setInSchema(child.getOutSchema()); + createIndex.setOutSchema(child.getOutSchema()); + return createIndex; + } + + @Override + public LogicalNode visitDropIndex(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, DropIndex expr) { + return ctx.plan.createNode(DropIndexNode.class); + } + public LogicalNode visitTruncateTable(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, TruncateTable expr) - throws PlanningException { + throws TajoException { TruncateTableNode truncateTableNode = ctx.plan.createNode(TruncateTableNode.class); return truncateTableNode; } http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index d10950a,47ab9b1..b61edd9 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@@ -36,13 -36,15 +36,17 @@@ import org.apache.tajo.algebra.* import org.apache.tajo.algebra.WindowSpec; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.UndefinedColumnException; + import org.apache.tajo.catalog.exception.UndefinedTableException; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.exception.ExceptionUtil; + import org.apache.tajo.exception.TajoException; + import org.apache.tajo.exception.TajoInternalError; + import org.apache.tajo.exception.UnimplementedException; import org.apache.tajo.plan.LogicalPlan.QueryBlock; import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; import org.apache.tajo.plan.expr.*; @@@ -2145,76 -2143,9 +2151,76 @@@ public class LogicalPlanner extends Bas return alterTableNode; } + private static URI getIndexPath(PlanContext context, String databaseName, String indexName) { + return new Path(TajoConf.getWarehouseDir(context.queryContext.getConf()), + databaseName + "/" + indexName + "/").toUri(); + } + + @Override + public LogicalNode visitCreateIndex(PlanContext context, Stack<Expr> stack, CreateIndex createIndex) - throws PlanningException { ++ throws TajoException { + stack.push(createIndex); + LogicalNode child = visit(context, stack, createIndex.getChild()); + stack.pop(); + + QueryBlock block = context.queryBlock; + CreateIndexNode createIndexNode = block.getNodeFromExpr(createIndex); + if (CatalogUtil.isFQTableName(createIndex.getIndexName())) { + createIndexNode.setIndexName(createIndex.getIndexName()); + } else { + createIndexNode.setIndexName( + CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), createIndex.getIndexName())); + } + createIndexNode.setUnique(createIndex.isUnique()); + Sort.SortSpec[] sortSpecs = createIndex.getSortSpecs(); + int sortKeyNum = sortSpecs.length; + String[] referNames = new String[sortKeyNum]; + + ExprNormalizedResult[] normalizedExprList = new ExprNormalizedResult[sortKeyNum]; + for (int i = 0; i < sortKeyNum; i++) { + normalizedExprList[i] = normalizer.normalize(context, sortSpecs[i].getKey()); + } + for (int i = 0; i < sortKeyNum; i++) { + // even if base expressions don't have their name, + // reference names should be identifiable for the later sort spec creation. + referNames[i] = block.namedExprsMgr.addExpr(normalizedExprList[i].baseExpr); + block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].aggExprs); + block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].scalarExprs); + } + + createIndexNode.setExternal(createIndex.isExternal()); + Collection<RelationNode> relations = block.getRelations(); + assert relations.size() == 1; + createIndexNode.setKeySortSpecs(relations.iterator().next().getLogicalSchema(), + annotateSortSpecs(block, referNames, sortSpecs)); + createIndexNode.setIndexMethod(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase())); + if (createIndex.isExternal()) { + createIndexNode.setIndexPath(new Path(createIndex.getIndexPath()).toUri()); + } else { + createIndexNode.setIndexPath( + getIndexPath(context, context.queryContext.get(SessionVars.CURRENT_DATABASE), createIndex.getIndexName())); + } + + if (createIndex.getParams() != null) { + KeyValueSet keyValueSet = new KeyValueSet(); + keyValueSet.putAll(createIndex.getParams()); + createIndexNode.setOptions(keyValueSet); + } + + createIndexNode.setChild(child); + return createIndexNode; + } + + @Override + public LogicalNode visitDropIndex(PlanContext context, Stack<Expr> stack, DropIndex dropIndex) { + DropIndexNode dropIndexNode = context.queryBlock.getNodeFromExpr(dropIndex); + dropIndexNode.setIndexName(dropIndex.getIndexName()); + return dropIndexNode; + } + @Override public LogicalNode visitTruncateTable(PlanContext context, Stack<Expr> stack, TruncateTable truncateTable) - throws PlanningException { + throws TajoException { TruncateTableNode truncateTableNode = context.queryBlock.getNodeFromExpr(truncateTable); truncateTableNode.setTableNames(truncateTable.getTableNames()); return truncateTableNode; http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java index 63c10ae,08a9bd1..3ca2fe5 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java @@@ -141,9 -141,8 +141,9 @@@ public class NamedExprsManager /** * Adds an expression and returns a reference name. + * @param expr added expression */ - public String addExpr(Expr expr) throws PlanningException { + public String addExpr(Expr expr) { if (idToExprBiMap.inverse().containsKey(expr)) { int refId = idToExprBiMap.inverse().get(expr); return idToNamesMap.get(refId).get(0); http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java index d818249,942dbb9..6149080 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java @@@ -25,95 -25,93 +25,95 @@@ import java.util.Stack public interface AlgebraVisitor<CONTEXT, RESULT> { // Relational Operators - RESULT visitSetSession(CONTEXT ctx, Stack<Expr> stack, SetSession expr) throws PlanningException; + RESULT visitSetSession(CONTEXT ctx, Stack<Expr> stack, SetSession expr) throws TajoException; // Relational Operators - RESULT visitProjection(CONTEXT ctx, Stack<Expr> stack, Projection expr) throws PlanningException; - RESULT visitLimit(CONTEXT ctx, Stack<Expr> stack, Limit expr) throws PlanningException; - RESULT visitSort(CONTEXT ctx, Stack<Expr> stack, Sort expr) throws PlanningException; - RESULT visitHaving(CONTEXT ctx, Stack<Expr> stack, Having expr) throws PlanningException; - RESULT visitGroupBy(CONTEXT ctx, Stack<Expr> stack, Aggregation expr) throws PlanningException; - RESULT visitJoin(CONTEXT ctx, Stack<Expr> stack, Join expr) throws PlanningException; - RESULT visitFilter(CONTEXT ctx, Stack<Expr> stack, Selection expr) throws PlanningException; - RESULT visitUnion(CONTEXT ctx, Stack<Expr> stack, SetOperation expr) throws PlanningException; - RESULT visitExcept(CONTEXT ctx, Stack<Expr> stack, SetOperation expr) throws PlanningException; - RESULT visitIntersect(CONTEXT ctx, Stack<Expr> stack, SetOperation expr) throws PlanningException; - RESULT visitSimpleTableSubQuery(CONTEXT ctx, Stack<Expr> stack, SimpleTableSubQuery expr) throws PlanningException; - RESULT visitTableSubQuery(CONTEXT ctx, Stack<Expr> stack, TablePrimarySubQuery expr) throws PlanningException; - RESULT visitRelationList(CONTEXT ctx, Stack<Expr> stack, RelationList expr) throws PlanningException; - RESULT visitRelation(CONTEXT ctx, Stack<Expr> stack, Relation expr) throws PlanningException; - RESULT visitScalarSubQuery(CONTEXT ctx, Stack<Expr> stack, ScalarSubQuery expr) throws PlanningException; - RESULT visitExplain(CONTEXT ctx, Stack<Expr> stack, Explain expr) throws PlanningException; + RESULT visitProjection(CONTEXT ctx, Stack<Expr> stack, Projection expr) throws TajoException; + RESULT visitLimit(CONTEXT ctx, Stack<Expr> stack, Limit expr) throws TajoException; + RESULT visitSort(CONTEXT ctx, Stack<Expr> stack, Sort expr) throws TajoException; + RESULT visitHaving(CONTEXT ctx, Stack<Expr> stack, Having expr) throws TajoException; + RESULT visitGroupBy(CONTEXT ctx, Stack<Expr> stack, Aggregation expr) throws TajoException; + RESULT visitJoin(CONTEXT ctx, Stack<Expr> stack, Join expr) throws TajoException; + RESULT visitFilter(CONTEXT ctx, Stack<Expr> stack, Selection expr) throws TajoException; + RESULT visitUnion(CONTEXT ctx, Stack<Expr> stack, SetOperation expr) throws TajoException; + RESULT visitExcept(CONTEXT ctx, Stack<Expr> stack, SetOperation expr) throws TajoException; + RESULT visitIntersect(CONTEXT ctx, Stack<Expr> stack, SetOperation expr) throws TajoException; + RESULT visitSimpleTableSubQuery(CONTEXT ctx, Stack<Expr> stack, SimpleTableSubQuery expr) throws TajoException; + RESULT visitTableSubQuery(CONTEXT ctx, Stack<Expr> stack, TablePrimarySubQuery expr) throws TajoException; + RESULT visitRelationList(CONTEXT ctx, Stack<Expr> stack, RelationList expr) throws TajoException; + RESULT visitRelation(CONTEXT ctx, Stack<Expr> stack, Relation expr) throws TajoException; + RESULT visitScalarSubQuery(CONTEXT ctx, Stack<Expr> stack, ScalarSubQuery expr) throws TajoException; + RESULT visitExplain(CONTEXT ctx, Stack<Expr> stack, Explain expr) throws TajoException; // Data definition language - RESULT visitCreateDatabase(CONTEXT ctx, Stack<Expr> stack, CreateDatabase expr) throws PlanningException; - RESULT visitDropDatabase(CONTEXT ctx, Stack<Expr> stack, DropDatabase expr) throws PlanningException; - RESULT visitCreateTable(CONTEXT ctx, Stack<Expr> stack, CreateTable expr) throws PlanningException; - RESULT visitDropTable(CONTEXT ctx, Stack<Expr> stack, DropTable expr) throws PlanningException; - RESULT visitAlterTablespace(CONTEXT ctx, Stack<Expr> stack, AlterTablespace expr) throws PlanningException; - RESULT visitAlterTable(CONTEXT ctx, Stack<Expr> stack, AlterTable expr) throws PlanningException; - RESULT visitCreateIndex(CONTEXT ctx, Stack<Expr> stack, CreateIndex expr) throws PlanningException; - RESULT visitDropIndex(CONTEXT ctx, Stack<Expr> stack, DropIndex expr) throws PlanningException; - RESULT visitTruncateTable(CONTEXT ctx, Stack<Expr> stack, TruncateTable expr) throws PlanningException; + RESULT visitCreateDatabase(CONTEXT ctx, Stack<Expr> stack, CreateDatabase expr) throws TajoException; + RESULT visitDropDatabase(CONTEXT ctx, Stack<Expr> stack, DropDatabase expr) throws TajoException; + RESULT visitCreateTable(CONTEXT ctx, Stack<Expr> stack, CreateTable expr) throws TajoException; + RESULT visitDropTable(CONTEXT ctx, Stack<Expr> stack, DropTable expr) throws TajoException; + RESULT visitAlterTablespace(CONTEXT ctx, Stack<Expr> stack, AlterTablespace expr) throws TajoException; + RESULT visitAlterTable(CONTEXT ctx, Stack<Expr> stack, AlterTable expr) throws TajoException; + RESULT visitTruncateTable(CONTEXT ctx, Stack<Expr> stack, TruncateTable expr) throws TajoException; ++ RESULT visitCreateIndex(CONTEXT ctx, Stack<Expr> stack, CreateIndex expr) throws TajoException; ++ RESULT visitDropIndex(CONTEXT ctx, Stack<Expr> stack, DropIndex expr) throws TajoException; // Insert or Update - RESULT visitInsert(CONTEXT ctx, Stack<Expr> stack, Insert expr) throws PlanningException; + RESULT visitInsert(CONTEXT ctx, Stack<Expr> stack, Insert expr) throws TajoException; // Logical operators - RESULT visitAnd(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; - RESULT visitOr(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; - RESULT visitNot(CONTEXT ctx, Stack<Expr> stack, NotExpr expr) throws PlanningException; + RESULT visitAnd(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; + RESULT visitOr(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; + RESULT visitNot(CONTEXT ctx, Stack<Expr> stack, NotExpr expr) throws TajoException; // comparison predicates - RESULT visitEquals(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; - RESULT visitNotEquals(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; - RESULT visitLessThan(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; - RESULT visitLessThanOrEquals(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; - RESULT visitGreaterThan(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; - RESULT visitGreaterThanOrEquals(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; + RESULT visitEquals(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; + RESULT visitNotEquals(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; + RESULT visitLessThan(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; + RESULT visitLessThanOrEquals(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; + RESULT visitGreaterThan(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; + RESULT visitGreaterThanOrEquals(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; // Other Predicates - RESULT visitBetween(CONTEXT ctx, Stack<Expr> stack, BetweenPredicate expr) throws PlanningException; - RESULT visitCaseWhen(CONTEXT ctx, Stack<Expr> stack, CaseWhenPredicate expr) throws PlanningException; - RESULT visitIsNullPredicate(CONTEXT ctx, Stack<Expr> stack, IsNullPredicate expr) throws PlanningException; - RESULT visitInPredicate(CONTEXT ctx, Stack<Expr> stack, InPredicate expr) throws PlanningException; - RESULT visitValueListExpr(CONTEXT ctx, Stack<Expr> stack, ValueListExpr expr) throws PlanningException; - RESULT visitExistsPredicate(CONTEXT ctx, Stack<Expr> stack, ExistsPredicate expr) throws PlanningException; + RESULT visitBetween(CONTEXT ctx, Stack<Expr> stack, BetweenPredicate expr) throws TajoException; + RESULT visitCaseWhen(CONTEXT ctx, Stack<Expr> stack, CaseWhenPredicate expr) throws TajoException; + RESULT visitIsNullPredicate(CONTEXT ctx, Stack<Expr> stack, IsNullPredicate expr) throws TajoException; + RESULT visitInPredicate(CONTEXT ctx, Stack<Expr> stack, InPredicate expr) throws TajoException; + RESULT visitValueListExpr(CONTEXT ctx, Stack<Expr> stack, ValueListExpr expr) throws TajoException; + RESULT visitExistsPredicate(CONTEXT ctx, Stack<Expr> stack, ExistsPredicate expr) throws TajoException; // String Operator or Pattern Matching Predicates - RESULT visitLikePredicate(CONTEXT ctx, Stack<Expr> stack, PatternMatchPredicate expr) throws PlanningException; - RESULT visitSimilarToPredicate(CONTEXT ctx, Stack<Expr> stack, PatternMatchPredicate expr) throws PlanningException; - RESULT visitRegexpPredicate(CONTEXT ctx, Stack<Expr> stack, PatternMatchPredicate expr) throws PlanningException; - RESULT visitConcatenate(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; + RESULT visitLikePredicate(CONTEXT ctx, Stack<Expr> stack, PatternMatchPredicate expr) throws TajoException; + RESULT visitSimilarToPredicate(CONTEXT ctx, Stack<Expr> stack, PatternMatchPredicate expr) throws TajoException; + RESULT visitRegexpPredicate(CONTEXT ctx, Stack<Expr> stack, PatternMatchPredicate expr) throws TajoException; + RESULT visitConcatenate(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; // arithmetic operators - RESULT visitPlus(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; - RESULT visitMinus(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; - RESULT visitMultiply(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; - RESULT visitDivide(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; - RESULT visitModular(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException; + RESULT visitPlus(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; + RESULT visitMinus(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; + RESULT visitMultiply(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; + RESULT visitDivide(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; + RESULT visitModular(CONTEXT ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException; // other expressions - RESULT visitSign(CONTEXT ctx, Stack<Expr> stack, SignedExpr expr) throws PlanningException; - RESULT visitColumnReference(CONTEXT ctx, Stack<Expr> stack, ColumnReferenceExpr expr) throws PlanningException; - RESULT visitTargetExpr(CONTEXT ctx, Stack<Expr> stack, NamedExpr expr) throws PlanningException; - RESULT visitQualifiedAsterisk(CONTEXT ctx, Stack<Expr> stack, QualifiedAsteriskExpr expr) throws PlanningException; + RESULT visitSign(CONTEXT ctx, Stack<Expr> stack, SignedExpr expr) throws TajoException; + RESULT visitColumnReference(CONTEXT ctx, Stack<Expr> stack, ColumnReferenceExpr expr) throws TajoException; + RESULT visitTargetExpr(CONTEXT ctx, Stack<Expr> stack, NamedExpr expr) throws TajoException; + RESULT visitQualifiedAsterisk(CONTEXT ctx, Stack<Expr> stack, QualifiedAsteriskExpr expr) throws TajoException; // functions - RESULT visitFunction(CONTEXT ctx, Stack<Expr> stack, FunctionExpr expr) throws PlanningException; + RESULT visitFunction(CONTEXT ctx, Stack<Expr> stack, FunctionExpr expr) throws TajoException; RESULT visitGeneralSetFunction(CONTEXT ctx, Stack<Expr> stack, GeneralSetFunctionExpr expr) - throws PlanningException; - RESULT visitCountRowsFunction(CONTEXT ctx, Stack<Expr> stack, CountRowsFunctionExpr expr) throws PlanningException; - RESULT visitWindowFunction(CONTEXT ctx, Stack<Expr> stack, WindowFunctionExpr expr) throws PlanningException; + throws TajoException; + RESULT visitCountRowsFunction(CONTEXT ctx, Stack<Expr> stack, CountRowsFunctionExpr expr) throws TajoException; + RESULT visitWindowFunction(CONTEXT ctx, Stack<Expr> stack, WindowFunctionExpr expr) throws TajoException; // Literal - RESULT visitCastExpr(CONTEXT ctx, Stack<Expr> stack, CastExpr expr) throws PlanningException; - - RESULT visitDataType(CONTEXT ctx, Stack<Expr> stack, DataTypeExpr expr) throws PlanningException; - RESULT visitLiteral(CONTEXT ctx, Stack<Expr> stack, LiteralValue expr) throws PlanningException; - RESULT visitNullLiteral(CONTEXT ctx, Stack<Expr> stack, NullLiteral expr) throws PlanningException; - RESULT visitTimestampLiteral(CONTEXT ctx, Stack<Expr> stack, TimestampLiteral expr) throws PlanningException; - RESULT visitIntervalLiteral(CONTEXT ctx, Stack<Expr> stack, IntervalLiteral expr) throws PlanningException; - RESULT visitTimeLiteral(CONTEXT ctx, Stack<Expr> stack, TimeLiteral expr) throws PlanningException; - RESULT visitDateLiteral(CONTEXT ctx, Stack<Expr> stack, DateLiteral expr) throws PlanningException; + RESULT visitCastExpr(CONTEXT ctx, Stack<Expr> stack, CastExpr expr) throws TajoException; + + RESULT visitDataType(CONTEXT ctx, Stack<Expr> stack, DataTypeExpr expr) throws TajoException; + RESULT visitLiteral(CONTEXT ctx, Stack<Expr> stack, LiteralValue expr) throws TajoException; + RESULT visitNullLiteral(CONTEXT ctx, Stack<Expr> stack, NullLiteral expr) throws TajoException; + RESULT visitTimestampLiteral(CONTEXT ctx, Stack<Expr> stack, TimestampLiteral expr) throws TajoException; + RESULT visitIntervalLiteral(CONTEXT ctx, Stack<Expr> stack, IntervalLiteral expr) throws TajoException; + RESULT visitTimeLiteral(CONTEXT ctx, Stack<Expr> stack, TimeLiteral expr) throws TajoException; + RESULT visitDateLiteral(CONTEXT ctx, Stack<Expr> stack, DateLiteral expr) throws TajoException; } http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java index dfdbf5b,e32bd85..2b4fb30 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java @@@ -487,18 -481,7 +487,18 @@@ public class BaseAlgebraVisitor<CONTEXT return null; } + @Override - public RESULT visitCreateIndex(CONTEXT ctx, Stack<Expr> stack, CreateIndex expr) throws PlanningException { ++ public RESULT visitCreateIndex(CONTEXT ctx, Stack<Expr> stack, CreateIndex expr) throws TajoException { + return null; + } + + @Override + public RESULT visitDropIndex(CONTEXT ctx, Stack<Expr> stack, DropIndex expr) { + return null; + } + + @Override - public RESULT visitTruncateTable(CONTEXT ctx, Stack<Expr> stack, TruncateTable expr) throws PlanningException { + public RESULT visitTruncateTable(CONTEXT ctx, Stack<Expr> stack, TruncateTable expr) throws TajoException { return null; } /////////////////////////////////////////////////////////////////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java index 7517e3a,c6b7354..cba41c1 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java @@@ -18,10 -18,7 +18,11 @@@ package org.apache.tajo.plan.expr; +import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.Column; ++import org.apache.tajo.exception.TajoException; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; import java.util.ArrayList; import java.util.Collection; @@@ -427,70 -424,4 +428,70 @@@ public class AlgebraicUtil found.add(node); } } + + public static class IdentifiableNameBuilder extends SimpleAlgebraVisitor<Object, Object> { + private Expr expr; + private StringBuilder nameBuilder = new StringBuilder(); + + public IdentifiableNameBuilder(Expr expr) { + this.expr = expr; + } + + public String build() { + Stack<Expr> stack = new Stack<Expr>(); + stack.push(expr); + try { + this.visit(null, stack, expr); - } catch (PlanningException e) { ++ } catch (TajoException e) { + + } + return nameBuilder.deleteCharAt(nameBuilder.length()-1).toString(); + } + + @Override - public Object visitBinaryOperator(Object ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException { ++ public Object visitBinaryOperator(Object ctx, Stack<Expr> stack, BinaryOperator expr) throws TajoException { + addIntermExpr(expr); + return super.visitBinaryOperator(ctx, stack, expr); + } + + private void append(String str) { + nameBuilder.append(str).append("_"); + } + + private void addIntermExpr(Expr expr) { + this.append(expr.getType().name()); + } + + @Override + public Object visitColumnReference(Object ctx, Stack<Expr> stack, ColumnReferenceExpr expr) - throws PlanningException { ++ throws TajoException { + this.append(expr.getName()); + return super.visitColumnReference(ctx, stack, expr); + } + + @Override - public Object visitLiteral(Object ctx, Stack<Expr> stack, LiteralValue expr) throws PlanningException { ++ public Object visitLiteral(Object ctx, Stack<Expr> stack, LiteralValue expr) throws TajoException { + this.append(expr.getValue()); + return super.visitLiteral(ctx, stack, expr); + } + + @Override - public Object visitNullLiteral(Object ctx, Stack<Expr> stack, NullLiteral expr) throws PlanningException { ++ public Object visitNullLiteral(Object ctx, Stack<Expr> stack, NullLiteral expr) throws TajoException { + this.append("null"); + return super.visitNullLiteral(ctx, stack, expr); + } + + @Override - public Object visitTimestampLiteral(Object ctx, Stack<Expr> stack, TimestampLiteral expr) throws PlanningException { ++ public Object visitTimestampLiteral(Object ctx, Stack<Expr> stack, TimestampLiteral expr) throws TajoException { + this.append(expr.getDate().toString()); + this.append(expr.getTime().toString()); + return super.visitTimestampLiteral(ctx, stack, expr); + } + + @Override - public Object visitTimeLiteral(Object ctx, Stack<Expr> stack, TimeLiteral expr) throws PlanningException { ++ public Object visitTimeLiteral(Object ctx, Stack<Expr> stack, TimeLiteral expr) throws TajoException { + this.append(expr.getTime().toString()); + return super.visitTimeLiteral(ctx, stack, expr); + } + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java index 6b7e32c,f6f1e63..9a541d9 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java @@@ -68,15 -69,15 +69,15 @@@ public class BaseLogicalPlanRewriteEngi /** * Rewrite a logical plan with all query rewrite rules added to this engine. * - * @param plan The plan to be rewritten with all query rewrite rule. + * @param context * @return The rewritten plan. */ - public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException { - public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws TajoException { ++ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws TajoException { LogicalPlanRewriteRule rule; + LogicalPlan plan = null; for (Entry<String, LogicalPlanRewriteRule> rewriteRule : rewriteRules.entrySet()) { rule = rewriteRule.getValue(); - if (rule.isEligible(queryContext, plan)) { - plan = rule.rewrite(queryContext, plan); + if (rule.isEligible(context)) { if (LOG.isDebugEnabled()) { LOG.debug("The rule \"" + rule.getName() + " \" rewrites the query."); } http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java index f264977,99d8979..a621921 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java @@@ -18,8 -18,9 +18,8 @@@ package org.apache.tajo.plan.rewrite; -import org.apache.tajo.OverridableConf; + import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.LogicalPlan; - import org.apache.tajo.plan.PlanningException; public interface LogicalPlanRewriteEngine { /** @@@ -28,5 -29,5 +28,5 @@@ * @param plan The plan to be rewritten with all query rewrite rule. * @return The rewritten plan. */ - LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException; - LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws TajoException; ++ LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws TajoException; } http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java index 1569a07,d5d02f2..6643d28 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java @@@ -18,8 -18,10 +18,8 @@@ package org.apache.tajo.plan.rewrite; -import org.apache.tajo.OverridableConf; + import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.LogicalPlan; --import org.apache.tajo.plan.PlanningException; /** * An interface for a rewrite rule. @@@ -52,5 -54,5 +52,5 @@@ public interface LogicalPlanRewriteRul * @param plan Input logical plan. It will not be modified. * @return The rewritten logical plan. */ - LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException; - LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws TajoException; ++ LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws TajoException; } http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java index bfa8d04,0000000..afabe7a mode 100644,000000..100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java @@@ -1,129 -1,0 +1,129 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite.rules; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; ++import org.apache.tajo.exception.TajoException; +import org.apache.tajo.plan.LogicalPlan; - import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.IndexScanNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.RelationNode; +import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; + +import java.util.List; +import java.util.Stack; + +public class AccessPathRewriter implements LogicalPlanRewriteRule { + private static final Log LOG = LogFactory.getLog(AccessPathRewriter.class); + + private static final String NAME = "Access Path Rewriter"; + private Rewriter rewriter = new Rewriter(); + + @Override + public String getName() { + return NAME; + } + + @Override + public boolean isEligible(LogicalPlanRewriteRuleContext context) { + if (context.getQueryContext().getBool(SessionVars.INDEX_ENABLED)) { + for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) { + for (RelationNode relationNode : block.getRelations()) { + List<AccessPathInfo> accessPathInfos = block.getAccessInfos(relationNode); + // If there are any alternative access paths + if (accessPathInfos.size() > 1) { + for (AccessPathInfo accessPathInfo : accessPathInfos) { + if (accessPathInfo.getScanType() == AccessPathInfo.ScanTypeControl.INDEX_SCAN) { + return true; + } + } + } + } + } + } + return false; + } + + @Override - public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException { ++ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws TajoException { + LogicalPlan plan = context.getPlan(); + LogicalPlan.QueryBlock rootBlock = plan.getRootBlock(); + rewriter.init(context.getQueryContext()); + rewriter.visit(rootBlock, plan, rootBlock, rootBlock.getRoot(), new Stack<LogicalNode>()); + return plan; + } + + private final class Rewriter extends BasicLogicalPlanVisitor<Object, Object> { + + private OverridableConf conf; + + public void init(OverridableConf conf) { + this.conf = conf; + } + + @Override + public Object visitScan(Object object, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scanNode, - Stack<LogicalNode> stack) throws PlanningException { ++ Stack<LogicalNode> stack) throws TajoException { + List<AccessPathInfo> accessPaths = block.getAccessInfos(scanNode); + AccessPathInfo optimalPath = null; + // initialize + for (AccessPathInfo accessPath : accessPaths) { + if (accessPath.getScanType() == AccessPathInfo.ScanTypeControl.SEQ_SCAN) { + optimalPath = accessPath; + break; + } + } + // find the optimal path + for (AccessPathInfo accessPath : accessPaths) { + if (accessPath.getScanType() == AccessPathInfo.ScanTypeControl.INDEX_SCAN) { + // estimation selectivity and choose the better path + // TODO: improve the selectivity estimation + double estimateSelectivity = 0.001; + double selectivityThreshold = conf.getFloat(SessionVars.INDEX_SELECTIVITY_THRESHOLD); + LOG.info("Selectivity threshold: " + selectivityThreshold); + LOG.info("Estimated selectivity: " + estimateSelectivity); + if (estimateSelectivity < selectivityThreshold) { + // if the estimated selectivity is greater than threshold, use the index scan + optimalPath = accessPath; + } + } + } + + if (optimalPath != null && optimalPath.getScanType() == AccessPathInfo.ScanTypeControl.INDEX_SCAN) { + IndexScanInfo indexScanInfo = (IndexScanInfo) optimalPath; + plan.addHistory("AccessPathRewriter chooses the index scan for " + scanNode.getTableName()); + IndexScanNode indexScanNode = new IndexScanNode(plan.newPID(), scanNode, indexScanInfo.getKeySchema(), + indexScanInfo.getPredicates(), indexScanInfo.getIndexPath()); + if (stack.empty() || block.getRoot().equals(scanNode)) { + block.setRoot(indexScanNode); + } else { + PlannerUtil.replaceNode(plan, stack.peek(), scanNode, indexScanNode); + } + } + return null; + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/49367117/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index c6eb479,503f371..ebc35a5 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@@ -26,8 -25,12 +26,14 @@@ import org.apache.commons.logging.Log import org.apache.commons.logging.LogFactory; import org.apache.tajo.OverridableConf; import org.apache.tajo.algebra.JoinType; +import org.apache.tajo.catalog.*; +import org.apache.tajo.datum.Datum; + import org.apache.tajo.catalog.CatalogUtil; + import org.apache.tajo.catalog.Column; + import org.apache.tajo.catalog.Schema; + import org.apache.tajo.catalog.TableDesc; + import org.apache.tajo.exception.TajoException; + import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.plan.*; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; @@@ -100,7 -98,7 +106,7 @@@ public class FilterPushDownRule extend } @Override - public LogicalPlan rewrite(LogicalPlanRewriteRuleContext rewriteRuleContext) throws PlanningException { - public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws TajoException { ++ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext rewriteRuleContext) throws TajoException { /* FilterPushDown rule: processing when visits each node - If a target which is corresponding on a filter EvalNode's column is not FieldEval, do not PushDown. @@@ -840,8 -836,8 +846,8 @@@ @Override public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, - LogicalPlan.QueryBlock block, ScanNode scanNode, + LogicalPlan.QueryBlock block, final ScanNode scanNode, - Stack<LogicalNode> stack) throws PlanningException { + Stack<LogicalNode> stack) throws TajoException { List<EvalNode> matched = TUtil.newList(); // find partition column and check matching @@@ -958,51 -920,8 +964,51 @@@ return scanNode; } + private static class Predicate { + Column column; + Datum value; + EvalType evalType; + + public Predicate(EvalType evalType, Column column, Datum value) { + this.evalType = evalType; + this.column = column; + this.value = value; + } + } + + private static SimplePredicate[] getSimplePredicates(IndexDesc desc, Set<Predicate> predicates) { + SimplePredicate[] simplePredicates = new SimplePredicate[predicates.size()]; + Map<Column, Datum> colToValue = TUtil.newHashMap(); + for (Predicate predicate : predicates) { + colToValue.put(predicate.column, predicate.value); + } + SortSpec [] keySortSpecs = desc.getKeySortSpecs(); + for (int i = 0; i < keySortSpecs.length; i++) { + simplePredicates[i] = new SimplePredicate(keySortSpecs[i], + colToValue.get(keySortSpecs[i].getSortKey())); + } + return simplePredicates; + } + + private static Datum[] extractPredicateValues(List<Predicate> predicates) { + Datum[] values = new Datum[predicates.size()]; + for (int i = 0; i < values.length; i++) { + values[i] = predicates.get(i).value; + } + return values; + } + + private static Column[] extractColumns(Set<Predicate> predicates) { + Column[] columns = new Column[predicates.size()]; + int i = 0; + for (Predicate p : predicates) { + columns[i++] = p.column; + } + return columns; + } + private void errorFilterPushDown(LogicalPlan plan, LogicalNode node, - FilterPushDownContext context) throws PlanningException { + FilterPushDownContext context) { String prefix = ""; StringBuilder notMatchedNodeStrBuilder = new StringBuilder(); for (EvalNode notMatchedNode: context.pushingDownFilters) {
