TAJO-1591: Change StoreType represented as Enum to String type. (hyunsik) Closes #564
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/53ed1c37 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/53ed1c37 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/53ed1c37 Branch: refs/heads/master Commit: 53ed1c376f4969b5a2c54e30ddbd8113847a8768 Parents: 70df3a8 0d1bf41 Author: Hyunsik Choi <[email protected]> Authored: Mon May 11 15:29:17 2015 -0700 Committer: Hyunsik Choi <[email protected]> Committed: Mon May 11 15:29:17 2015 -0700 ---------------------------------------------------------------------- CHANGES | 11 +- .../org/apache/tajo/catalog/CatalogUtil.java | 12 +- .../org/apache/tajo/catalog/NestedPathUtil.java | 109 ++++++ .../java/org/apache/tajo/catalog/Schema.java | 109 +++++- .../org/apache/tajo/catalog/SchemaUtil.java | 33 +- .../tajo/catalog/store/HiveCatalogStore.java | 4 +- .../cli/tsql/commands/DescTableCommand.java | 4 +- .../org/apache/tajo/storage/RowStoreUtil.java | 2 +- .../java/org/apache/tajo/storage/Tuple.java | 2 +- .../java/org/apache/tajo/util/StringUtils.java | 70 ++++ .../main/java/org/apache/tajo/util/TUtil.java | 32 -- .../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +- .../apache/tajo/engine/parser/SQLAnalyzer.java | 17 +- .../engine/planner/PhysicalPlannerImpl.java | 17 +- .../tajo/engine/planner/global/DataChannel.java | 4 +- .../engine/planner/global/GlobalPlanner.java | 6 +- .../global/builder/DistinctGroupbyBuilder.java | 6 +- .../DistinctGroupbyThirdAggregationExec.java | 2 +- .../planner/physical/PhysicalPlanUtil.java | 2 +- .../engine/planner/physical/SeqScanExec.java | 27 +- .../tajo/engine/utils/TupleCacheScanner.java | 2 +- .../org/apache/tajo/engine/utils/TupleUtil.java | 8 +- .../exec/NonForwardQueryResultFileScanner.java | 2 +- .../NonForwardQueryResultSystemScanner.java | 20 +- .../tajo/worker/ExecutionBlockContext.java | 15 +- .../java/org/apache/tajo/worker/TaskRunner.java | 1 + .../resources/webapps/admin/catalogview.jsp | 5 +- .../engine/function/TestFunctionLoader.java | 5 +- .../tajo/engine/planner/TestLogicalPlanner.java | 8 +- .../apache/tajo/engine/query/TestCTASQuery.java | 4 +- .../tajo/engine/query/TestJoinBroadcast.java | 2 +- .../engine/query/TestSelectNestedRecord.java | 71 ++++ .../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 2 +- .../rs/resources/TestQueryResultResource.java | 6 +- .../TestSelectNestedRecord/sample1/table.json | 3 + .../TestSelectNestedRecord/tweets/sample1.json | 4 + .../TestSelectNestedRecord/sample1_ddl.sql | 7 + .../TestSelectNestedRecord/sample2_ddl.sql | 19 ++ .../testNestedFieldAsGroupbyKey1.sql | 7 + .../testNestedFieldAsJoinKey1.sql | 7 + .../TestSelectNestedRecord/testSelect1.sql | 1 + .../TestSelectNestedRecord/testSelect2.sql | 61 ++++ .../TestSelectNestedRecord/tweets_ddl.sql | 74 +++++ .../testSelectWithParentheses2.sql | 2 +- .../testNestedFieldAsGroupbyKey1.result | 6 + .../testNestedFieldAsJoinKey1.result | 6 + .../TestSelectNestedRecord/testSelect1.result | 5 + .../TestSelectNestedRecord/testSelect2.result | 6 + .../apache/tajo/jdbc/TajoDatabaseMetaData.java | 2 +- .../tajo/plan/LogicalPlanPreprocessor.java | 4 +- .../org/apache/tajo/plan/LogicalPlanner.java | 32 +- .../apache/tajo/plan/expr/RowConstantEval.java | 3 +- .../tajo/plan/expr/WindowFunctionEval.java | 3 +- .../GreedyHeuristicJoinOrderAlgorithm.java | 8 +- .../apache/tajo/plan/joinorder/JoinEdge.java | 4 +- .../apache/tajo/plan/joinorder/JoinGraph.java | 6 +- .../tajo/plan/logical/DistinctGroupbyNode.java | 4 +- .../apache/tajo/plan/logical/EvalExprNode.java | 3 +- .../apache/tajo/plan/logical/GroupbyNode.java | 10 +- .../tajo/plan/logical/ProjectionNode.java | 3 +- .../apache/tajo/plan/logical/RelationNode.java | 12 + .../tajo/plan/logical/ShuffleFileWriteNode.java | 3 +- .../tajo/plan/logical/TruncateTableNode.java | 4 +- .../apache/tajo/plan/logical/WindowAggNode.java | 7 +- .../tajo/plan/nameresolver/NameResolver.java | 194 ++++++++--- .../plan/nameresolver/ResolverByLegacy.java | 8 +- .../plan/rewrite/rules/FilterPushDownRule.java | 4 +- .../rewrite/rules/PartitionedTableRewriter.java | 4 +- .../org/apache/tajo/plan/util/PlannerUtil.java | 2 +- .../tajo/plan/verifier/LogicalPlanVerifier.java | 24 +- .../plan/verifier/PreLogicalPlanVerifier.java | 6 +- .../org/apache/tajo/rpc/AsyncRpcServer.java | 6 +- .../org/apache/tajo/rpc/BlockingRpcServer.java | 6 +- .../org/apache/tajo/storage/MergeScanner.java | 2 +- .../org/apache/tajo/storage/RowStoreUtil.java | 2 +- .../java/org/apache/tajo/storage/Scanner.java | 9 + .../org/apache/tajo/storage/StorageManager.java | 4 +- .../storage/hbase/AbstractHBaseAppender.java | 2 - .../tajo/storage/hbase/ColumnMapping.java | 2 +- .../HBaseBinarySerializerDeserializer.java | 10 +- .../apache/tajo/storage/hbase/HBaseScanner.java | 22 +- .../tajo/storage/hbase/HBaseStorageManager.java | 6 +- .../java/org/apache/tajo/storage/CSVFile.java | 2 +- .../org/apache/tajo/storage/FileScanner.java | 2 +- .../apache/tajo/storage/FileStorageManager.java | 2 +- .../apache/tajo/storage/avro/AvroScanner.java | 25 +- .../tajo/storage/json/JsonLineDeserializer.java | 331 +++++++++++-------- .../apache/tajo/storage/json/JsonLineSerDe.java | 5 +- .../tajo/storage/json/JsonLineSerializer.java | 2 - .../storage/parquet/TajoRecordConverter.java | 7 +- .../org/apache/tajo/storage/rcfile/RCFile.java | 10 +- .../sequencefile/SequenceFileScanner.java | 16 +- .../tajo/storage/text/CSVLineDeserializer.java | 26 +- .../apache/tajo/storage/text/CSVLineSerDe.java | 5 +- .../tajo/storage/text/DelimitedTextFile.java | 11 +- .../tajo/storage/text/TextLineDeserializer.java | 4 +- .../apache/tajo/storage/text/TextLineSerDe.java | 3 +- .../apache/tajo/storage/TestMergeScanner.java | 16 +- .../org/apache/tajo/storage/TestStorages.java | 36 +- 99 files changed, 1292 insertions(+), 462 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/CHANGES ---------------------------------------------------------------------- diff --cc CHANGES index 9788307,84be66a..73fd2f3 --- a/CHANGES +++ b/CHANGES @@@ -24,6 -24,6 +24,8 @@@ Release 0.11.0 - unrelease IMPROVEMENT ++ TAJO-1591: Change StoreType represented as Enum to String type. (hyunsik) ++ TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun, Committed by hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java ---------------------------------------------------------------------- diff --cc tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java index e2cfe8f,322818d..2c856e1 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java @@@ -175,7 -175,19 +175,19 @@@ public class TestMergeScanner Tuple tuple; while ((tuple = scanner.next()) != null) { totalCounts++; - if (isProjectableStorage(meta.getStoreType())) { + - if (storeType == StoreType.RAW) { ++ if (storeType.equalsIgnoreCase("RAW")) { + assertEquals(4, tuple.size()); + assertNotNull(tuple.get(0)); + assertNotNull(tuple.get(1)); + assertNotNull(tuple.get(2)); + assertNotNull(tuple.get(3)); + } else if (scanner.isProjectable()) { + assertEquals(2, tuple.size()); + assertNotNull(tuple.get(0)); + assertNotNull(tuple.get(1)); + } else { + assertEquals(4, tuple.size()); assertNotNull(tuple.get(0)); assertNull(tuple.get(1)); assertNotNull(tuple.get(2)); @@@ -187,15 -199,15 +199,13 @@@ assertEquals(tupleNum * 2, totalCounts); } - private static boolean isProjectableStorage(StoreType type) { - switch (type) { - case CSV: - case SEQUENCEFILE: - case RAW: - case ROWFILE: - return false; - default: + private static boolean isProjectableStorage(String type) { + if (type.equalsIgnoreCase("RCFILE") || + type.equalsIgnoreCase("PARQUET") || - type.equalsIgnoreCase("SEQUENCEFILE") || - type.equalsIgnoreCase("CSV") || + type.equalsIgnoreCase("AVRO")) { return true; + } else { + return false; } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java ---------------------------------------------------------------------- diff --cc tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java index 7c75973,a735307..949e2e8 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java @@@ -313,9 -305,23 +305,23 @@@ public class TestStorages assertEquals(tupleNum, tupleCnt); } + private void verifyProjectedFields(boolean projectable, Tuple tuple, int tupleCnt) { + if (projectable) { + assertTrue(tupleCnt + 2 == tuple.get(0).asInt8()); + assertTrue(tupleCnt + 3 == tuple.get(1).asFloat4()); + } else { + // RAW and ROW always project all fields. - if (storeType != StoreType.RAW && storeType != StoreType.ROWFILE) { ++ if (!storeType.equalsIgnoreCase("RAW") && !storeType.equalsIgnoreCase("ROWFILE")) { + assertTrue(tuple.get(0) == null); + } + assertTrue(tupleCnt + 2 == tuple.get(1).asInt8()); + assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4()); + } + } + @Test public void testVariousTypes() throws IOException { - boolean handleProtobuf = storeType != StoreType.JSON; + boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON"); Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@@ -956,7 -962,7 +962,8 @@@ @Test public void testLessThanSchemaSize() throws IOException { /* RAW is internal storage. It must be same with schema size */ - if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO")){ - if (storeType == StoreType.RAW || storeType == StoreType.AVRO || storeType == StoreType.PARQUET) { ++ if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO") ++ || storeType.equalsIgnoreCase("PARQUET")){ return; }
