Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into
index_support
Conflicts:
CHANGES
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/1bf1536d
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/1bf1536d
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/1bf1536d
Branch: refs/heads/index_support
Commit: 1bf1536db45b50f80872fe01ac8f1b4a2400f0d2
Parents: 5a8fed4 bcec5eb
Author: Jihoon Son <[email protected]>
Authored: Fri Jul 24 12:38:56 2015 +0900
Committer: Jihoon Son <[email protected]>
Committed: Fri Jul 24 12:38:56 2015 +0900
----------------------------------------------------------------------
CHANGES | 10 ++
.../apache/tajo/client/SessionConnection.java | 17 ++-
.../main/proto/TajoMasterClientProtocol.proto | 2 +-
.../planner/physical/ColPartitionStoreExec.java | 17 +--
.../HashBasedColPartitionStoreExec.java | 59 ++++-----
.../SortBasedColPartitionStoreExec.java | 60 ++++-----
.../tajo/master/TajoMasterClientService.java | 105 +++++++++++----
.../tajo/engine/query/TestInnerJoinQuery.java | 1 -
.../engine/query/TestInnerJoinWithSubQuery.java | 1 -
.../engine/query/TestMultipleJoinTypes.java | 1 -
.../tajo/engine/query/TestOuterJoinQuery.java | 2 -
...sJoinWithThetaJoinConditionInWhere.Hash.plan | 8 +-
...taJoinConditionInWhere.Hash_NoBroadcast.plan | 12 +-
...sJoinWithThetaJoinConditionInWhere.Sort.plan | 8 +-
...taJoinConditionInWhere.Sort_NoBroadcast.plan | 12 +-
...rJoinWithThetaJoinConditionInWhere.Hash.plan | 8 +-
...taJoinConditionInWhere.Hash_NoBroadcast.plan | 12 +-
...rJoinWithThetaJoinConditionInWhere.Sort.plan | 8 +-
...taJoinConditionInWhere.Sort_NoBroadcast.plan | 12 +-
.../testBroadcastSubquery2.Hash.plan | 28 ++--
...testBroadcastSubquery2.Hash_NoBroadcast.plan | 44 +++----
.../testBroadcastSubquery2.Sort.plan | 28 ++--
...testBroadcastSubquery2.Sort_NoBroadcast.plan | 44 +++----
.../testJoinWithMultipleJoinQual4.Hash.plan | 16 +--
...nWithMultipleJoinQual4.Hash_NoBroadcast.plan | 28 ++--
.../testJoinWithMultipleJoinQual4.Sort.plan | 16 +--
...nWithMultipleJoinQual4.Sort_NoBroadcast.plan | 28 ++--
.../testThetaJoinKeyPairs.Hash.plan | 128 ++++++++++---------
.../testThetaJoinKeyPairs.Hash_NoBroadcast.plan | 114 +++++++++--------
.../testThetaJoinKeyPairs.Sort.plan | 128 ++++++++++---------
.../testThetaJoinKeyPairs.Sort_NoBroadcast.plan | 114 +++++++++--------
.../testThetaJoinKeyPairs.result | 6 -
...rJoinWithThetaJoinConditionInWhere.Hash.plan | 8 +-
...taJoinConditionInWhere.Hash_NoBroadcast.plan | 12 +-
...rJoinWithThetaJoinConditionInWhere.Sort.plan | 8 +-
...taJoinConditionInWhere.Sort_NoBroadcast.plan | 12 +-
...rJoinWithThetaJoinConditionInWhere.Hash.plan | 8 +-
...taJoinConditionInWhere.Hash_NoBroadcast.plan | 12 +-
...rJoinWithThetaJoinConditionInWhere.Sort.plan | 8 +-
...taJoinConditionInWhere.Sort_NoBroadcast.plan | 12 +-
.../org/apache/tajo/plan/LogicalOptimizer.java | 13 +-
.../org/apache/tajo/plan/expr/EvalTreeUtil.java | 2 +-
.../plan/rewrite/rules/FilterPushDownRule.java | 55 +++++++-
43 files changed, 669 insertions(+), 558 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index 9fb7eb3,420aa4b..2c32817
--- a/CHANGES
+++ b/CHANGES
@@@ -188,13 -191,8 +191,15 @@@ Release 0.11.0 - unrelease
BUG FIXES
+ TAJO-1608: Fix test failure in index_support branch. (jihoon)
+
+ TAJO-1594: Catalog schema is invalid for some databases. (jihoon)
+
+ TAJO-1657: Tajo Rest API /database/{database-name]/tables should return
table
+ names only without invalid external table info.
+
+ TAJO-1597: Problem of ignoring theta join condition. (jihoon)
+
TAJO-1697: RCFile progress causes NPE occasionally. (jinho)
TAJO-1689: Metrics file reporter prints histogram metric without group
name.
@@@ -406,7 -404,8 +411,12 @@@
SUB TASKS
++<<<<<<< HEAD
+ TAJO-1302: Support index metadata backup and restore. (jihoon)
++=======
+ TAJO-1484 Apply on ColPartitionStoreExec. (Contributed by Navis,
+ committed by hyunsik)
++>>>>>>> bcec5eb7e247dc983184b37a3dbb4143afd19ac8
TAJO-1464: Add ORCFileScanner to read ORCFile table. (Contributed by
Jongyoung Park, Committed by jihoon)
http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --cc
tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index c1afbc9,baf1320..0e9b6e6
---
a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++
b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@@ -61,14 -65,15 +64,18 @@@ import org.apache.tajo.util.ProtoUtil
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
+ import static org.apache.tajo.exception.ExceptionUtil.printStackTraceIfError;
import static org.apache.tajo.exception.ReturnStateUtil.*;
+ /**
+ * It provides Client Remote API service for TajoMaster.
+ */
public class TajoMasterClientService extends AbstractService {
private final static Log LOG =
LogFactory.getLog(TajoMasterClientService.class);
private final MasterContext context;
@@@ -899,203 -953,12 +954,207 @@@
.build();
} catch (Throwable t) {
- return
FunctionListResponse.newBuilder().setState(returnError(t)).build();
+ printStackTraceIfError(LOG, t);
+
+ return FunctionListResponse.newBuilder().
+ setState(returnError(t))
+ .build();
}
}
+
+ @Override
+ public IndexResponse getIndexWithName(RpcController controller,
SessionedStringProto request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session =
context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String indexName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted =
CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ indexName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ indexName = request.getValue();
+ }
+ IndexDescProto indexProto = catalog.getIndexByName(databaseName,
indexName).getProto();
+ return IndexResponse.newBuilder()
+ .setState(OK)
+ .setIndexDesc(indexProto)
+ .build();
+
+ } catch (Throwable t) {
+ return IndexResponse.newBuilder()
+ .setState(returnError(t))
+ .build();
+ }
+ }
+
+ @Override
+ public ReturnState existIndexWithName(RpcController controller,
SessionedStringProto request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session =
context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String indexName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted =
CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ indexName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ indexName = request.getValue();
+ }
+
+ if (catalog.existIndexByName(databaseName, indexName)) {
+ return OK;
+ } else {
+ return errUndefinedIndexName(indexName);
+ }
+ } catch (Throwable t) {
+ return returnError(t);
+ }
+ }
+
+ @Override
+ public IndexListResponse getIndexesForTable(RpcController controller,
SessionedStringProto request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session =
context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String tableName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted =
CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ tableName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ tableName = request.getValue();
+ }
+
+ IndexListResponse.Builder builder =
IndexListResponse.newBuilder().setState(OK);
+ for (IndexDesc index : catalog.getAllIndexesByTable(databaseName,
tableName)) {
+ builder.addIndexDesc(index.getProto());
+ }
+ return builder.build();
+ } catch (Throwable t) {
+ return IndexListResponse.newBuilder()
+ .setState(returnError(t))
+ .build();
+ }
+ }
+
+ @Override
+ public ReturnState existIndexesForTable(RpcController controller,
SessionedStringProto request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session =
context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String tableName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted =
CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ tableName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ tableName = request.getValue();
+ }
+ if (catalog.existIndexesByTable(databaseName, tableName)) {
+ return OK;
+ } else {
+ return errUndefinedIndex(tableName);
+ }
+ } catch (Throwable t) {
+ return returnError(t);
+ }
+ }
+
+ @Override
+ public IndexResponse getIndexWithColumns(RpcController controller,
GetIndexWithColumnsRequest request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session =
context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String tableName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getTableName())) {
+ String [] splitted =
CatalogUtil.splitFQTableName(request.getTableName());
+ databaseName = splitted[0];
+ tableName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ tableName = request.getTableName();
+ }
+ String[] columnNames = new String[request.getColumnNamesCount()];
+ columnNames = request.getColumnNamesList().toArray(columnNames);
+
+ return IndexResponse.newBuilder()
+ .setState(OK)
+ .setIndexDesc(catalog.getIndexByColumnNames(databaseName,
tableName, columnNames).getProto())
+ .build();
+
+ } catch (Throwable t) {
+ return IndexResponse.newBuilder()
+ .setState(returnError(t))
+ .build();
+ }
+ }
+
+ @Override
+ public ReturnState existIndexWithColumns(RpcController controller,
GetIndexWithColumnsRequest request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session =
context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String tableName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getTableName())) {
+ String [] splitted =
CatalogUtil.splitFQTableName(request.getTableName());
+ databaseName = splitted[0];
+ tableName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ tableName = request.getTableName();
+ }
+ String[] columnNames = new String[request.getColumnNamesCount()];
+ columnNames = request.getColumnNamesList().toArray(columnNames);
+ if (catalog.existIndexByColumnNames(databaseName, tableName,
columnNames)) {
+ return OK;
+ } else {
+ return errUndefinedIndex(tableName, request.getColumnNamesList());
+ }
+ } catch (Throwable t) {
+ return returnError(t);
+ }
+ }
+
+ @Override
+ public ReturnState dropIndex(RpcController controller,
SessionedStringProto request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session =
context.getSessionManager().getSession(request.getSessionId().getId());
+ QueryContext queryContext = new QueryContext(conf, session);
+
+ String indexName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted =
CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ indexName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ indexName = request.getValue();
+ }
+ catalog.dropIndex(databaseName, indexName);
+
+ return OK;
+ } catch (Throwable t) {
+ return returnError(t);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/1bf1536d/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
----------------------------------------------------------------------
diff --cc
tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
index ebc35a5,98e674e..4fb8aac
---
a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
+++
b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
@@@ -35,12 -32,10 +35,13 @@@ import org.apache.tajo.catalog.TableDes
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.exception.TajoInternalError;
import org.apache.tajo.plan.*;
+ import org.apache.tajo.plan.LogicalPlan.QueryBlock;
import org.apache.tajo.plan.expr.*;
import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
import
org.apache.tajo.plan.rewrite.rules.FilterPushDownRule.FilterPushDownContext;
+import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate;
+import org.apache.tajo.plan.util.IndexUtil;
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
