Repository: tajo Updated Branches: refs/heads/branch-0.8.0 8c2703c2d -> c7e361676
TAJO-706: In the case of very quick query, client can't get query status. (hyoungjunkim via hyunsik) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/c7e36167 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/c7e36167 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/c7e36167 Branch: refs/heads/branch-0.8.0 Commit: c7e361676470e823b2e2718f98bfd05ab35ea992 Parents: 8c2703c Author: Hyunsik Choi <[email protected]> Authored: Sun Mar 30 22:16:28 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Sun Mar 30 23:53:22 2014 +0900 ---------------------------------------------------------------------- .../java/org/apache/tajo/client/TajoDump.java | 20 ++++--- .../tajo/master/TajoMasterClientService.java | 26 +++++++-- .../master/querymaster/QueryJobManager.java | 6 +++ .../org/apache/tajo/client/TestTajoClient.java | 56 ++++++++++++++++++++ 4 files changed, 97 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/c7e36167/tajo-client/src/main/java/org/apache/tajo/client/TajoDump.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoDump.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoDump.java index 224de45..924a65d 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/TajoDump.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoDump.java @@ -150,14 +150,20 @@ public class TajoDump { List<String> tableNames = TUtil.newList(client.getTableList(databaseName)); Collections.sort(tableNames); for (String tableName : tableNames) { - TableDesc table = - client.getTableDesc(CatalogUtil.denormalizeIdentifier(CatalogUtil.buildFQName(databaseName,tableName))); - if (table.isExternal()) { - writer.write(DDLBuilder.buildDDLForExternalTable(table)); - } else { - writer.write(DDLBuilder.buildDDLForBaseTable(table)); + try { + TableDesc table = + client.getTableDesc(CatalogUtil.denormalizeIdentifier(CatalogUtil.buildFQName(databaseName,tableName))); + if (table.isExternal()) { + writer.write(DDLBuilder.buildDDLForExternalTable(table)); + } else { + writer.write(DDLBuilder.buildDDLForBaseTable(table)); + } + writer.write("\n\n"); + } catch (Exception e) { + // dump for each table can throw any exception. We need to skip the exception case. + // here, the error message prints out via stderr. + System.err.println("ERROR:" + tableName + "," + e.getMessage()); } - writer.write("\n\n"); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/c7e36167/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index 924be95..42aba7c 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -328,9 +328,24 @@ public class TajoMasterClientService extends AbstractService { context.getSessionManager().touch(request.getSessionId().getId()); QueryId queryId = new QueryId(request.getQueryId()); QueryInProgress queryInProgress = context.getQueryJobManager().getQueryInProgress(queryId); + + // if we cannot get a QueryInProgress instance from QueryJobManager, + // the instance can be in the finished query list. + if (queryInProgress == null) { + queryInProgress = context.getQueryJobManager().getFinishedQuery(queryId); + } + + GetQueryResultResponse.Builder builder = GetQueryResultResponse.newBuilder(); + + // If we cannot the QueryInProgress instance from the finished list, + // the query result was expired due to timeout. + // In this case, we will result in error. + if (queryInProgress == null) { + builder.setErrorMessage("No such query: " + queryId.toString()); + return builder.build(); + } + QueryInfo queryInfo = queryInProgress.getQueryInfo(); - GetQueryResultResponse.Builder builder - = GetQueryResultResponse.newBuilder(); try { //TODO After implementation Tajo's user security feature, Should be modified. @@ -448,6 +463,11 @@ public class TajoMasterClientService extends AbstractService { builder.setState(TajoProtos.QueryState.QUERY_SUCCEEDED); } else { QueryInProgress queryInProgress = context.getQueryJobManager().getQueryInProgress(queryId); + + // It will try to find a query status from a finished query list. + if (queryInProgress == null) { + queryInProgress = context.getQueryJobManager().getFinishedQuery(queryId); + } if (queryInProgress != null) { QueryInfo queryInfo = queryInProgress.getQueryInfo(); builder.setResultCode(ResultCode.OK); @@ -458,8 +478,6 @@ public class TajoMasterClientService extends AbstractService { builder.setQueryMasterHost(queryInfo.getQueryMasterHost()); builder.setQueryMasterPort(queryInfo.getQueryMasterClientPort()); } - //builder.setInitTime(queryJobManager.getInitializationTime()); - //builder.setHasResult(!queryJobManager.isCreateTableStmt()); if (queryInfo.getQueryState() == TajoProtos.QueryState.QUERY_SUCCEEDED) { builder.setFinishTime(queryInfo.getFinishTime()); } else { http://git-wip-us.apache.org/repos/asf/tajo/blob/c7e36167/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java index 0c28c52..97513a0 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java @@ -137,6 +137,12 @@ public class QueryJobManager extends CompositeService { } } + public QueryInProgress getFinishedQuery(QueryId queryId) { + synchronized(finishedQueries) { + return finishedQueries.get(queryId); + } + } + public void stopQuery(QueryId queryId) { LOG.info("Stop QueryInProgress:" + queryId); QueryInProgress queryInProgress = getQueryInProgress(queryId); http://git-wip-us.apache.org/repos/asf/tajo/blob/c7e36167/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java index c8b11a8..5eb8af2 100644 --- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -33,12 +33,14 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.ipc.ClientProtos; +import org.apache.tajo.jdbc.TajoResultSet; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.CommonTestingUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.Timeout; import java.io.IOException; import java.sql.ResultSet; @@ -608,4 +610,58 @@ public class TestTajoClient { resultSet.close(); } + + /** + * The main objective of this test is to get the status of a query which is actually finished. + * Statuses of queries regardless of its status should be available for a specified time duration. + */ + @Test(timeout = 20 * 1000) + public final void testGetQueryStatusAndResultAfterFinish() throws Exception { + String sql = "select * from lineitem"; + ClientProtos.GetQueryStatusResponse response = client.executeQuery(sql); + + assertNotNull(response); + QueryId queryId = new QueryId(response.getQueryId()); + + try { + long startTime = System.currentTimeMillis(); + while (true) { + Thread.sleep(5 * 1000); + + List<ClientProtos.BriefQueryInfo> finishedQueries = client.getFinishedQueryList(); + boolean finished = false; + if (finishedQueries != null) { + for (ClientProtos.BriefQueryInfo eachQuery: finishedQueries) { + if (eachQuery.getQueryId().equals(queryId.getProto())) { + finished = true; + break; + } + } + } + + if (finished) { + break; + } + if(System.currentTimeMillis() - startTime > 20 * 1000) { + fail("Too long time execution query"); + } + } + + QueryStatus queryStatus = client.getQueryStatus(queryId); + assertNotNull(queryStatus); + assertTrue(!TajoClient.isQueryRunnning(queryStatus.getState())); + + TajoResultSet resultSet = (TajoResultSet) client.getQueryResult(queryId); + assertNotNull(resultSet); + + int count = 0; + while(resultSet.next()) { + count++; + } + + assertEquals(5, count); + } finally { + client.closeQuery(queryId); + } + } }
