Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fd2f8281 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fd2f8281 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fd2f8281 Branch: refs/heads/index_support Commit: fd2f8281dd0efcf07322ae94086a1a72ad8b5f87 Parents: 63c8e1c 8d0146b Author: Jihoon Son <[email protected]> Authored: Mon Mar 23 11:55:25 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Mon Mar 23 11:55:25 2015 +0900 ---------------------------------------------------------------------- CHANGES | 39 +++ tajo-catalog/tajo-catalog-client/pom.xml | 2 +- .../tajo-catalog-drivers/tajo-hcatalog/pom.xml | 2 +- tajo-catalog/tajo-catalog-server/pom.xml | 2 +- tajo-cli/pom.xml | 2 +- .../org/apache/tajo/cli/tools/TajoAdmin.java | 1 - .../org/apache/tajo/cli/tools/TajoGetConf.java | 5 - .../org/apache/tajo/cli/tools/TajoHAAdmin.java | 5 - .../cli/tsql/DefaultTajoCliOutputFormatter.java | 2 +- .../org/apache/tajo/cli/tsql/SimpleParser.java | 3 +- .../java/org/apache/tajo/cli/tsql/TajoCli.java | 10 +- tajo-client/pom.xml | 2 +- .../apache/tajo/client/SessionConnection.java | 20 +- .../java/org/apache/tajo/conf/TajoConf.java | 2 + .../java/org/apache/tajo/util/BytesUtils.java | 159 +++++---- .../java/org/apache/tajo/util/StringUtils.java | 6 +- .../org/apache/tajo/util/TestStringUtil.java | 4 +- tajo-core/pom.xml | 2 +- .../engine/planner/global/GlobalPlanner.java | 39 ++- .../engine/planner/physical/HashJoinExec.java | 51 ++- .../planner/physical/HashLeftOuterJoinExec.java | 74 +++- .../physical/PartitionMergeScanExec.java | 17 +- .../tajo/engine/planner/physical/ScanExec.java | 72 ++++ .../engine/planner/physical/SeqScanExec.java | 105 +----- .../apache/tajo/engine/utils/CacheHolder.java | 97 ++++++ .../apache/tajo/engine/utils/TableCache.java | 84 +++++ .../apache/tajo/engine/utils/TableCacheKey.java | 57 +++ .../apache/tajo/engine/utils/TupleCache.java | 122 ------- .../apache/tajo/engine/utils/TupleCacheKey.java | 57 --- .../org/apache/tajo/master/QueryInProgress.java | 2 +- .../org/apache/tajo/master/QueryManager.java | 2 +- .../exec/NonForwardQueryResultFileScanner.java | 37 +- .../apache/tajo/master/exec/QueryExecutor.java | 5 + .../java/org/apache/tajo/querymaster/Stage.java | 3 +- .../tajo/worker/ExecutionBlockContext.java | 43 ++- .../worker/ExecutionBlockSharedResource.java | 26 ++ .../main/java/org/apache/tajo/worker/Task.java | 75 ++-- .../apache/tajo/worker/TaskAttemptContext.java | 2 +- .../java/org/apache/tajo/worker/TaskRunner.java | 9 +- .../apache/tajo/worker/TaskRunnerManager.java | 4 +- .../apache/tajo/engine/eval/ExprTestBase.java | 5 +- .../apache/tajo/engine/query/TestCTASQuery.java | 28 ++ .../tajo/engine/query/TestSelectQuery.java | 48 +++ .../tajo/engine/query/TestUnionQuery.java | 64 +++- .../apache/tajo/engine/util/TestTableCache.java | 109 ++++++ .../apache/tajo/engine/util/TestTupleCache.java | 89 ----- .../apache/tajo/master/TestGlobalPlanner.java | 3 +- .../multibytes_delimiter1/table1.tbl | 5 + .../multibytes_delimiter2/table2.tbl | 5 + .../TestCTASQuery/CtasWithMultipleUnions.sql | 12 + .../testCtasWithMultipleUnions.sql | 1 + .../multibytes_delimiter_table1_ddl.sql | 3 + .../multibytes_delimiter_table2_ddl.sql | 3 + .../multibytes_delimiter_table3_ddl.sql | 3 + .../multibytes_delimiter_table4_ddl.sql | 3 + .../testMultiBytesDelimiter1.sql | 1 + .../testMultiBytesDelimiter2.sql | 1 + .../testMultiBytesDelimiter3.sql | 1 + .../testMultiBytesDelimiter4.sql | 1 + .../testMultiBytesDelimiter1.result | 7 + .../testMultiBytesDelimiter2.result | 7 + .../testMultiBytesDelimiter3.result | 7 + .../testMultiBytesDelimiter4.result | 7 + tajo-dist/pom.xml | 7 +- tajo-dist/src/main/conf/tajo-env.sh | 2 +- .../org/apache/tajo/plan/ExprAnnotator.java | 2 +- .../java/org/apache/tajo/plan/LogicalPlan.java | 2 - .../plan/expr/AggregationFunctionCallEval.java | 14 + .../apache/tajo/plan/expr/AlgebraicUtil.java | 5 + .../tajo/plan/expr/BetweenPredicateEval.java | 14 + .../org/apache/tajo/plan/expr/CaseWhenEval.java | 25 +- .../org/apache/tajo/plan/expr/CastEval.java | 10 + .../org/apache/tajo/plan/expr/EvalTreeUtil.java | 66 +++- .../java/org/apache/tajo/plan/expr/InEval.java | 10 + .../org/apache/tajo/plan/expr/IsNullEval.java | 9 + .../tajo/plan/expr/PartialBinaryExpr.java | 5 +- .../tajo/plan/expr/RegexPredicateEval.java | 25 ++ .../apache/tajo/plan/expr/RowConstantEval.java | 11 + .../tajo/plan/expr/WindowFunctionEval.java | 14 + .../GreedyHeuristicJoinOrderAlgorithm.java | 4 +- .../tajo/plan/logical/AlterTableNode.java | 14 + .../tajo/plan/logical/DistinctGroupbyNode.java | 17 +- .../apache/tajo/plan/logical/EvalExprNode.java | 18 +- .../apache/tajo/plan/logical/GroupbyNode.java | 16 +- .../apache/tajo/plan/logical/HavingNode.java | 9 + .../apache/tajo/plan/logical/IndexScanNode.java | 13 + .../apache/tajo/plan/logical/InsertNode.java | 13 + .../org/apache/tajo/plan/logical/JoinNode.java | 13 + .../org/apache/tajo/plan/logical/LimitNode.java | 11 +- .../apache/tajo/plan/logical/LogicalNode.java | 27 +- .../tajo/plan/logical/PersistentStoreNode.java | 10 + .../tajo/plan/logical/ProjectionNode.java | 28 +- .../apache/tajo/plan/logical/SelectionNode.java | 9 + .../tajo/plan/logical/ShuffleFileWriteNode.java | 13 + .../org/apache/tajo/plan/logical/SortNode.java | 14 +- .../tajo/plan/logical/StoreTableNode.java | 10 + .../tajo/plan/nameresolver/NameResolver.java | 4 +- .../plan/rewrite/rules/FilterPushDownRule.java | 10 +- .../tajo/plan/serder/EvalNodeDeserializer.java | 17 +- .../plan/serder/LogicalNodeDeserializer.java | 3 + .../tajo/plan/serder/LogicalNodeSerializer.java | 2 + .../org/apache/tajo/plan/util/PlannerUtil.java | 26 +- .../tajo/plan/verifier/LogicalPlanVerifier.java | 7 - .../plan/verifier/PreLogicalPlanVerifier.java | 19 +- .../plan/visitor/BasicLogicalPlanVisitor.java | 6 +- tajo-plan/src/main/proto/Plan.proto | 1 + tajo-project/pom.xml | 31 ++ tajo-project/src/site/site.xml | 2 +- tajo-pullserver/pom.xml | 2 +- tajo-rpc/pom.xml | 180 ++-------- .../org/apache/tajo/rpc/AsyncRpcClient.java | 267 -------------- .../org/apache/tajo/rpc/AsyncRpcServer.java | 148 -------- .../org/apache/tajo/rpc/BlockingRpcClient.java | 315 ----------------- .../org/apache/tajo/rpc/BlockingRpcServer.java | 147 -------- .../java/org/apache/tajo/rpc/CallFuture.java | 84 ----- .../apache/tajo/rpc/DefaultRpcController.java | 65 ---- .../org/apache/tajo/rpc/NettyClientBase.java | 173 --------- .../org/apache/tajo/rpc/NettyRpcController.java | 63 ---- .../org/apache/tajo/rpc/NettyServerBase.java | 205 ----------- .../java/org/apache/tajo/rpc/NullCallback.java | 38 -- .../tajo/rpc/ProtoChannelInitializer.java | 50 --- .../apache/tajo/rpc/RemoteCallException.java | 69 ---- .../org/apache/tajo/rpc/RemoteException.java | 37 -- .../tajo/rpc/RetriesExhaustedException.java | 104 ------ .../org/apache/tajo/rpc/RpcChannelFactory.java | 182 ---------- .../org/apache/tajo/rpc/RpcConnectionPool.java | 194 ----------- .../main/java/org/apache/tajo/rpc/RpcUtils.java | 68 ---- .../org/apache/tajo/rpc/ServerCallable.java | 162 --------- .../apache/tajo/rpc/TajoServiceException.java | 58 --- tajo-rpc/src/main/proto/DummyProtos.proto | 47 --- tajo-rpc/src/main/proto/RpcProtos.proto | 32 -- tajo-rpc/src/main/proto/TestProtocol.proto | 31 -- tajo-rpc/src/main/proto/TestProtos.proto | 35 -- tajo-rpc/src/test/java/log4j.properties | 25 -- .../java/org/apache/tajo/rpc/TestAsyncRpc.java | 339 ------------------ .../org/apache/tajo/rpc/TestBlockingRpc.java | 341 ------------------ .../rpc/test/impl/DummyProtocolAsyncImpl.java | 86 ----- .../test/impl/DummyProtocolBlockingImpl.java | 83 ----- tajo-rpc/tajo-rpc-common/pom.xml | 216 ++++++++++++ .../org/apache/tajo/rpc/NettyServerBase.java | 243 +++++++++++++ .../org/apache/tajo/rpc/RemoteException.java | 37 ++ .../tajo/rpc/RetriesExhaustedException.java | 104 ++++++ .../org/apache/tajo/rpc/RpcChannelFactory.java | 182 ++++++++++ .../org/apache/tajo/rpc/RpcEventListener.java | 62 ++++ .../main/java/org/apache/tajo/rpc/RpcUtils.java | 122 +++++++ tajo-rpc/tajo-rpc-protobuf/pom.xml | 274 +++++++++++++++ .../org/apache/tajo/rpc/AsyncRpcClient.java | 227 ++++++++++++ .../org/apache/tajo/rpc/AsyncRpcServer.java | 148 ++++++++ .../org/apache/tajo/rpc/BlockingRpcClient.java | 273 +++++++++++++++ .../org/apache/tajo/rpc/BlockingRpcServer.java | 147 ++++++++ .../java/org/apache/tajo/rpc/CallFuture.java | 84 +++++ .../apache/tajo/rpc/DefaultRpcController.java | 65 ++++ .../org/apache/tajo/rpc/NettyClientBase.java | 221 ++++++++++++ .../org/apache/tajo/rpc/NettyRpcController.java | 63 ++++ .../java/org/apache/tajo/rpc/NullCallback.java | 38 ++ .../tajo/rpc/ProtoChannelInitializer.java | 50 +++ .../apache/tajo/rpc/RemoteCallException.java | 69 ++++ .../tajo/rpc/RetriesExhaustedException.java | 104 ++++++ .../org/apache/tajo/rpc/RpcConnectionPool.java | 190 ++++++++++ .../org/apache/tajo/rpc/ServerCallable.java | 162 +++++++++ .../apache/tajo/rpc/TajoServiceException.java | 58 +++ .../src/main/proto/DummyProtos.proto | 47 +++ .../src/main/proto/RpcProtos.proto | 32 ++ .../src/main/proto/TestProtocol.proto | 31 ++ .../src/main/proto/TestProtos.proto | 35 ++ .../src/test/java/log4j.properties | 25 ++ .../java/org/apache/tajo/rpc/TestAsyncRpc.java | 345 ++++++++++++++++++ .../org/apache/tajo/rpc/TestBlockingRpc.java | 349 +++++++++++++++++++ .../rpc/test/impl/DummyProtocolAsyncImpl.java | 86 +++++ .../test/impl/DummyProtocolBlockingImpl.java | 83 +++++ tajo-rpc/tajo-ws-rs/pom.xml | 218 ++++++++++++ .../rs/netty/NettyRestChannelInitializer.java | 50 +++ .../ws/rs/netty/NettyRestHandlerContainer.java | 319 +++++++++++++++++ .../NettyRestHandlerContainerProvider.java | 42 +++ .../tajo/ws/rs/netty/NettyRestServer.java | 67 ++++ .../ws/rs/netty/NettyRestServerFactory.java | 89 +++++ .../ws/rs/netty/NettyRestServerListener.java | 72 ++++ .../tajo/ws/rs/netty/gson/GsonFeature.java | 34 ++ .../tajo/ws/rs/netty/gson/GsonReader.java | 52 +++ .../apache/tajo/ws/rs/netty/gson/GsonUtil.java | 32 ++ .../tajo/ws/rs/netty/gson/GsonWriter.java | 59 ++++ .../NettyRestHandlerContainerProviderTest.java | 66 ++++ .../tajo/ws/rs/netty/NettyRestServerTest.java | 137 ++++++++ .../ws/rs/netty/testapp1/TestApplication1.java | 38 ++ .../ws/rs/netty/testapp1/TestResource1.java | 36 ++ .../ws/rs/netty/testapp2/DirectoriesDao.java | 39 +++ .../rs/netty/testapp2/DirectoriesResource.java | 85 +++++ .../tajo/ws/rs/netty/testapp2/Directory.java | 52 +++ .../testapp2/FileManagementApplication.java | 35 ++ .../org/apache/tajo/storage/TestLazyTuple.java | 4 +- .../tajo/storage/hbase/ColumnMapping.java | 6 +- .../apache/tajo/storage/hbase/HBaseScanner.java | 3 +- .../tajo/storage/hbase/HBaseStorageManager.java | 3 +- .../java/org/apache/tajo/storage/CSVFile.java | 21 +- .../apache/tajo/storage/FileStorageManager.java | 27 +- .../sequencefile/SequenceFileScanner.java | 3 +- .../tajo/storage/text/CSVLineDeserializer.java | 14 +- .../apache/tajo/storage/text/CSVLineSerDe.java | 5 +- .../tajo/storage/text/CSVLineSerializer.java | 8 +- .../tajo/storage/text/DelimitedTextFile.java | 2 +- .../tajo/storage/text/FieldSplitProcessor.java | 8 +- .../text/MultiBytesFieldSplitProcessor.java | 45 +++ .../tajo/storage/text/TextLineDeserializer.java | 6 +- .../apache/tajo/storage/text/TextLineSerDe.java | 3 +- .../apache/tajo/storage/TestSplitProcessor.java | 38 +- 205 files changed, 7516 insertions(+), 4256 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/main/java/org/apache/tajo/worker/Task.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java index a36e982,8b73756..e320ce9 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java @@@ -18,28 -18,24 +18,31 @@@ package org.apache.tajo.plan.logical; + import java.util.Arrays; + import com.google.gson.Gson; import com.google.gson.annotations.Expose; + import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.datum.Datum; +import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate; import org.apache.tajo.plan.serder.PlanGsonHelper; +import org.apache.tajo.util.TUtil; + +import java.net.URI; public class IndexScanNode extends ScanNode { - @Expose private SortSpec [] sortKeys; @Expose private Schema keySchema = null; - @Expose private Datum[] datum = null; + @Expose private URI indexPath = null; + @Expose private SimplePredicate[] predicates = null; + + public IndexScanNode(int pid) { + super(pid); + setType(NodeType.INDEX_SCAN); + } public IndexScanNode(int pid, ScanNode scanNode , - Schema keySchema , Datum[] datum, SortSpec[] sortKeys ) { - super(pid); + Schema keySchema , SimplePredicate[] predicates, URI indexPath) { + this(pid); init(scanNode.getTableDesc()); setQual(scanNode.getQual()); setInSchema(scanNode.getInSchema()); http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index 1ce7019,4cd008a..a257296 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@@ -957,58 -918,17 +957,60 @@@ public class FilterPushDownRule extend return scanNode; } + private static class Predicate { + Column column; + Datum value; + EvalType evalType; + + public Predicate(EvalType evalType, Column column, Datum value) { + this.evalType = evalType; + this.column = column; + this.value = value; + } + } + + private static SimplePredicate[] getSimplePredicates(IndexDesc desc, Set<Predicate> predicates) { + SimplePredicate[] simplePredicates = new SimplePredicate[predicates.size()]; + Map<Column, Datum> colToValue = TUtil.newHashMap(); + for (Predicate predicate : predicates) { + colToValue.put(predicate.column, predicate.value); + } + SortSpec [] keySortSpecs = desc.getKeySortSpecs(); + for (int i = 0; i < keySortSpecs.length; i++) { + simplePredicates[i] = new SimplePredicate(keySortSpecs[i], + colToValue.get(keySortSpecs[i].getSortKey())); + } + return simplePredicates; + } + + private static Datum[] extractPredicateValues(List<Predicate> predicates) { + Datum[] values = new Datum[predicates.size()]; + for (int i = 0; i < values.length; i++) { + values[i] = predicates.get(i).value; + } + return values; + } + + private static Column[] extractColumns(Set<Predicate> predicates) { + Column[] columns = new Column[predicates.size()]; + int i = 0; + for (Predicate p : predicates) { + columns[i++] = p.column; + } + return columns; + } + private void errorFilterPushDown(LogicalPlan plan, LogicalNode node, FilterPushDownContext context) throws PlanningException { - String notMatchedNodeStr = ""; String prefix = ""; + StringBuilder notMatchedNodeStrBuilder = new StringBuilder(); for (EvalNode notMatchedNode: context.pushingDownFilters) { - notMatchedNodeStr += prefix + notMatchedNode; - prefix = ", "; + notMatchedNodeStrBuilder.append(prefix).append(notMatchedNode.toString()); + if (prefix.isEmpty()) { + prefix = ", "; + } } - throw new PlanningException("FilterPushDown failed cause some filters not matched: " + notMatchedNodeStr + "\n" + + throw new PlanningException("FilterPushDown failed cause some filters not matched: " + notMatchedNodeStrBuilder.toString() + "\n" + "Error node: " + node.getPlanString() + "\n" + plan.toString()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd2f8281/tajo-plan/src/main/proto/Plan.proto ----------------------------------------------------------------------
