Repository: lens Updated Branches: refs/heads/master e4f26aae8 -> 79b95f0e0
LENS-915 : Update CLI to show streaming results on sync executions Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/79b95f0e Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/79b95f0e Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/79b95f0e Branch: refs/heads/master Commit: 79b95f0e04340e6b648ee71d41308fb03ea11e16 Parents: e4f26aa Author: Puneet Gupta <[email protected]> Authored: Tue Mar 29 14:18:21 2016 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Tue Mar 29 14:18:21 2016 +0530 ---------------------------------------------------------------------- .../org/apache/lens/api/query/LensQuery.java | 56 +++-- .../org/apache/lens/api/query/QueryHandle.java | 2 +- .../org/apache/lens/api/query/QueryStatus.java | 8 + .../lens/cli/commands/BaseLensCommand.java | 7 +- .../cli/commands/LensConnectionCommands.java | 5 +- .../lens/cli/commands/LensQueryCommands.java | 41 ++-- .../lens/cli/config/LensCliConfigConstants.java | 36 +++ .../apache/lens/cli/TestLensQueryCommands.java | 138 +++++++++++- .../java/org/apache/lens/client/LensClient.java | 39 +++- .../org/apache/lens/client/LensConnection.java | 2 +- .../org/apache/lens/client/LensStatement.java | 218 +++++++++++++------ .../lens/client/jdbc/LensJdbcStatement.java | 4 +- .../lens/client/model/ProxyLensQuery.java | 167 ++++++++++++++ .../src/main/resources/lens-client-default.xml | 7 + .../org/apache/lens/examples/SampleQueries.java | 4 +- .../java/org/apache/lens/rdd/LensRDDClient.java | 2 +- .../server/query/QueryExecutionServiceImpl.java | 2 +- .../lens/server/query/TestQueryService.java | 4 +- src/site/apt/user/client-config.apt | 14 +- 19 files changed, 619 insertions(+), 137 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java index e0ee761..7d079a9 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java @@ -31,7 +31,7 @@ import org.apache.lens.api.ToYAMLString; import lombok.*; /** - * The Class LensQuery. + * This class provides all the details about the query represented by the {@link LensQuery#queryHandle} */ @XmlRootElement /** @@ -79,46 +79,46 @@ import lombok.*; * Instantiates a new lens query. */ @NoArgsConstructor(access = AccessLevel.PROTECTED) -@EqualsAndHashCode +@EqualsAndHashCode(of = "queryHandle", callSuper = false) public class LensQuery extends ToYAMLString { /** - * The query handle. + * The query handle that represents this query uniquely */ @XmlElement @Getter private QueryHandle queryHandle; /** - * The user query. + * The the query submitted by the user */ @XmlElement @Getter private String userQuery; /** - * The submitted user. + * The user who submitted the query. */ @XmlElement @Getter private String submittedUser; /** - * The priority. + * The priority of the query. */ @XmlElement @Getter private Priority priority; /** - * The is persistent. + * Is true if query's result would be persisted by server. */ @XmlElement @Getter private boolean isPersistent; /** - * The selected driver class name. + * Name of the driver which executed the query (Example: hive/testDriver, jdbc/prodDriver etc) */ @XmlElement @Getter @@ -126,34 +126,39 @@ public class LensQuery extends ToYAMLString { /** * The driver query. + * It is the final query (derived form user query) that was submitted by the driver for execution. */ @XmlElement @Getter private String driverQuery; /** - * The status. + * The status of this query. + * Note: {@link QueryStatus#getStatus()} method can be used to get the {@link QueryStatus.Status} enum that defines + * the current state of the query. Also other utility methods are available to check the status of the query like + * {@link QueryStatus#queued()}, {@link QueryStatus#successful()}, {@link QueryStatus#finished()}, + * {@link QueryStatus#failed()} and {@link QueryStatus#running()} */ @XmlElement @Getter private QueryStatus status; /** - * The result set path. + * The result set path for this query if the query output was persisted by the server. */ @XmlElement @Getter private String resultSetPath; /** - * The driver op handle. + * The operation handle associated with the driver, if any. */ @XmlElement @Getter private String driverOpHandle; /** - * The query conf. + * The query conf that was used for executing this query. */ @XmlElement @Getter @@ -167,60 +172,67 @@ public class LensQuery extends ToYAMLString { private long submissionTime; /** - * The launch time. + * The query launch time. This will be submission time + time spent by query waiting in the queue */ @XmlElement @Getter private long launchTime; /** - * The driver start time. + * The query execution start time on driver. This will >= launch time. */ @XmlElement @Getter private long driverStartTime; /** - * The driver finish time. + * The the query execution end time on driver. */ @XmlElement @Getter private long driverFinishTime; /** - * The finish time. + * The query finish time on server. This will be driver finish time + any extra time spent by server (like + * formatting the result) */ @XmlElement @Getter private long finishTime; /** - * The closed time. + * The the query close time when the query is purged by the server and no more operations are pending for it. + * Note: not supported as of now. */ @XmlElement @Getter private long closedTime; /** - * The query name. + * The query name, if any. */ @XmlElement @Getter private String queryName; + /** + * @return error code in case of query failures + */ public Integer getErrorCode() { return (this.status != null) ? this.status.getErrorCode() : null; } + /** + * @return error message in case of query failures + */ public String getErrorMessage() { return (this.status != null) ? this.status.getLensErrorTOErrorMsg() : null; } + /** + * @return the query handle string + */ public String getQueryHandleString() { return (this.queryHandle != null) ? this.queryHandle.getHandleIdString() : null; } - - public boolean queued() { - return this.status.queued(); - } } http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java index 88e4b0f..81ca0e8 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java @@ -54,7 +54,7 @@ import lombok.*; * * @see java.lang.Object#hashCode() */ -@EqualsAndHashCode(callSuper = false) +@EqualsAndHashCode(of = "handleId", callSuper = false) public class QueryHandle extends QuerySubmitResult implements Serializable { /** http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java index 7a9ada1..67d1e79 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java @@ -81,26 +81,33 @@ public class QueryStatus extends ToYAMLString implements Serializable { /** * The queued. + * At this point the query is queued by the server and it waiting to be launched. The launch may be controlled by + * multiple factors like query throttling, quota, etc */ QUEUED, /** * The launched. + * At this point the query is launched for execution. */ LAUNCHED, /** * The running. + * At this point the query starts running on chosen driver */ RUNNING, /** * The executed. + * At this point execution is finished by driver, but server may still have some more operations pending + * like result persistence, if enabled. */ EXECUTED, /** * The successful. + * At this point all operations related to the query are finished successfully by driver and server. */ SUCCESSFUL, @@ -116,6 +123,7 @@ public class QueryStatus extends ToYAMLString implements Serializable { /** * The closed. + * At this point the query is purged by the server. Persistent result will still be available to the user */ CLOSED } http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java index be1ca12..9eb73b3 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java @@ -26,6 +26,7 @@ import java.util.Date; import org.apache.lens.api.ToXMLString; import org.apache.lens.api.util.PathValidator; +import org.apache.lens.cli.config.LensCliConfigConstants; import org.apache.lens.client.LensClient; import org.apache.lens.client.LensClientSingletonWrapper; @@ -40,6 +41,7 @@ import org.springframework.shell.core.ExecutionProcessor; import org.springframework.shell.event.ParseResult; import com.google.common.collect.Sets; + import lombok.extern.slf4j.Slf4j; /** @@ -47,8 +49,6 @@ import lombok.extern.slf4j.Slf4j; */ @Slf4j public class BaseLensCommand implements ExecutionProcessor { - public static final String LENS_CLI_PREFIX = "lens.cli."; - public static final String JSON_PRETTY_SUFFIX = "json.pretty"; /** The mapper. */ protected ObjectMapper mapper; @@ -149,7 +149,8 @@ public class BaseLensCommand implements ExecutionProcessor { String json = mapper.writer(pp).writeValueAsString(data); JsonNode tree = mapper.valueToTree(data); System.out.println(tree); - if (getClient().getConf().getBoolean(LENS_CLI_PREFIX + JSON_PRETTY_SUFFIX, false)) { + if (getClient().getConf().getBoolean(LensCliConfigConstants.PRINT_PRETTY_JSON, + LensCliConfigConstants.DEFAULT_PRINT_PRETTY_JSON)) { return json; } return json.replaceAll("\\[ \\{", "\n\n ").replaceAll("\\{", "").replaceAll("}", "").replaceAll("\\[", "") http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java index b760dad..e548cac 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java @@ -26,6 +26,7 @@ import javax.ws.rs.ProcessingException; import org.apache.lens.api.APIResult; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.cli.commands.annotations.UserDocumentation; +import org.apache.lens.cli.config.LensCliConfigConstants; import org.apache.lens.client.LensClient; import org.apache.lens.client.LensClientConfig; @@ -43,7 +44,9 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.Appender; import ch.qos.logback.core.ConsoleAppender; import ch.qos.logback.core.Context; + import com.google.common.base.Joiner; + import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -70,7 +73,7 @@ public class LensConnectionCommands extends BaseLensCommand { if (pair.length != 2) { return "Error: Pass parameter as <key>=<value>"; } - if (pair[0].startsWith(LENS_CLI_PREFIX) || pair[0].startsWith(LensClientConfig.CLIENT_PFX)) { + if (pair[0].startsWith(LensCliConfigConstants.LENS_CLI_PREFIX) || pair[0].startsWith(LensClientConfig.CLIENT_PFX)) { getClient().getConf().set(pair[0], pair[1]); return "Client side Set " + pair[0] + "=" + pair[1]; } else { http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java index 2c7b17f..af05951 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java @@ -28,7 +28,10 @@ import javax.ws.rs.core.Response; import org.apache.lens.api.query.*; import org.apache.lens.api.result.PrettyPrintable; import org.apache.lens.cli.commands.annotations.UserDocumentation; +import org.apache.lens.cli.config.LensCliConfigConstants; import org.apache.lens.client.LensClient; +import org.apache.lens.client.LensClient.LensClientResultSetWithStats; +import org.apache.lens.client.LensClientResultSet; import org.apache.lens.client.exceptions.LensAPIException; import org.apache.lens.client.exceptions.LensBriefErrorException; import org.apache.lens.client.model.BriefError; @@ -107,16 +110,28 @@ public class LensQueryCommands extends BaseLensCommand { try { if (async) { - QueryHandle queryHandle = getClient().executeQueryAsynch(sql, queryName).getData(); + QueryHandle queryHandle = getClient().executeQueryAsynch(sql, queryName); return queryHandle.getHandleIdString(); } else { - return formatResultSet(getClient().getResults(sql, queryName)); + LensClientResultSetWithStats resultWithStats; + long timeOutMillis = getClient().getConf().getLong(LensCliConfigConstants.QUERY_EXECUTE_TIMEOUT_MILLIS, + LensCliConfigConstants.DEFAULT_QUERY_EXECUTE_TIMEOUT_MILLIS); + LensClient.getCliLogger().info("Executing query with timeout of {} milliseconds", timeOutMillis); + QueryHandleWithResultSet result = getClient().executeQueryWithTimeout(sql, queryName, timeOutMillis); + if (result.getResult() == null) { + //Query not finished yet. Wait till it finishes and get result. + LensClient.getCliLogger().info("Couldn't complete query execution within timeout. Waiting for completion"); + resultWithStats = getClient().getSyncResults(result.getQueryHandle()); + } else { + LensClientResultSet clientResultSet = new LensClientResultSet(result.getResultMetadata(), result.getResult()); + resultWithStats = + new LensClientResultSetWithStats(clientResultSet, getClient().getQueryDetails(result.getQueryHandle())); + } + return formatResultSet(resultWithStats); } } catch (final LensAPIException e) { - BriefError briefError = new BriefError(e.getLensAPIErrorCode(), e.getLensAPIErrorMessage()); cliOutput = new IdBriefErrorTemplate(IdBriefErrorTemplateKey.REQUEST_ID, e.getLensAPIRequestId(), briefError); - } catch (final LensBriefErrorException e) { cliOutput = e.getIdBriefErrorTemplate(); } @@ -436,17 +451,17 @@ public class LensQueryCommands extends BaseLensCommand { @CliOption(key = {"async"}, mandatory = false, unspecifiedDefaultValue = "false", specifiedDefaultValue = "true", help = "<async>") boolean async, @CliOption(key = {"name"}, mandatory = false, help = "<query-name>") String queryName) { - if (async) { - QueryHandle handle = getClient().executePrepared(QueryPrepareHandle.fromString(phandle), queryName); - return handle.getHandleId().toString(); - } else { - try { - LensClient.LensClientResultSetWithStats result = getClient().getResultsFromPrepared( - QueryPrepareHandle.fromString(phandle), queryName); + try { + if (async) { + QueryHandle handle = getClient().executePrepared(QueryPrepareHandle.fromString(phandle), queryName); + return handle.getHandleId().toString(); + } else { + LensClient.LensClientResultSetWithStats result = + getClient().getResultsFromPrepared(QueryPrepareHandle.fromString(phandle), queryName); return formatResultSet(result); - } catch (Throwable t) { - return t.getMessage(); } + } catch (Throwable t) { + return t.getMessage(); } } http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-cli/src/main/java/org/apache/lens/cli/config/LensCliConfigConstants.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/config/LensCliConfigConstants.java b/lens-cli/src/main/java/org/apache/lens/cli/config/LensCliConfigConstants.java new file mode 100644 index 0000000..5651c31 --- /dev/null +++ b/lens-cli/src/main/java/org/apache/lens/cli/config/LensCliConfigConstants.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.cli.config; + +public class LensCliConfigConstants { + + private LensCliConfigConstants(){ + } + + public static final String LENS_CLI_PREFIX = "lens.cli."; + + public static final String PRINT_PRETTY_JSON = LENS_CLI_PREFIX + "json.pretty"; + + public static final boolean DEFAULT_PRINT_PRETTY_JSON = false; + + public static final String QUERY_EXECUTE_TIMEOUT_MILLIS = LENS_CLI_PREFIX + "query.execute.timeout.millis"; + + public static final long DEFAULT_QUERY_EXECUTE_TIMEOUT_MILLIS = 10000; //10 secs +} http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java index 616bf5d..dbb5e9d 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java @@ -23,6 +23,8 @@ import static org.testng.Assert.*; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.net.URL; import java.util.*; @@ -31,17 +33,23 @@ import javax.xml.datatype.DatatypeFactory; import org.apache.lens.api.APIResult; import org.apache.lens.api.metastore.*; +import org.apache.lens.api.query.LensQuery; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.query.QueryStatus; import org.apache.lens.cli.commands.LensCubeCommands; import org.apache.lens.cli.commands.LensDimensionTableCommands; import org.apache.lens.cli.commands.LensQueryCommands; +import org.apache.lens.cli.config.LensCliConfigConstants; import org.apache.lens.client.LensClient; +import org.apache.lens.client.model.ProxyLensQuery; import org.apache.lens.driver.hive.TestHiveDriver; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.query.TestQueryService.DeferredInMemoryResultFormatter; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.Path; +import org.testng.Assert; import org.testng.annotations.*; import lombok.extern.slf4j.Slf4j; @@ -92,6 +100,46 @@ public class TestLensQueryCommands extends LensCliApplicationTest { return qCom; } + @DataProvider(name = "executeSyncQueryDP") + private Object[][] executeSyncQueryDP() { + //Streaming not enabled. InMemory ResultSet Expected + LensClient client1 = new LensClient(); + client1.setConnectionParam("lens.query.enable.persistent.resultset.indriver", "false"); + client1.setConnectionParam("lens.query.enable.persistent.resultset", "false"); + client1.setConnectionParam("lens.query.enable.metrics.per.query", "true"); + client1.getConf().setLong(LensCliConfigConstants.QUERY_EXECUTE_TIMEOUT_MILLIS, 100); + LensQueryCommands qCom1 = new LensQueryCommands(); + qCom1.setClient(client1); + + //Streaming case. Query execution and formatting finishes fast. + //InMemory result set Excepted. + LensClient client2 = new LensClient(); + client2.setConnectionParam("lens.query.enable.persistent.resultset.indriver", "false"); + client2.setConnectionParam("lens.query.enable.persistent.resultset", "true"); + client2.getConf().setLong(LensCliConfigConstants.QUERY_EXECUTE_TIMEOUT_MILLIS, 20000); + LensQueryCommands qCom2 = new LensQueryCommands(); + qCom2.setClient(client2); + + //Streaming case. Query execution finishes fast, but server takes long time to format result. + //InMemory ResultSet Excepted. + //Wait for query to be successful (i,e finish both execution and formatting) + LensClient client3 = new LensClient(); + client3.setConnectionParam("lens.query.enable.persistent.resultset.indriver", "false"); + client3.setConnectionParam("lens.query.enable.persistent.resultset", "true"); + client3.setConnectionParam(LensConfConstants.QUERY_OUTPUT_FORMATTER, + DeferredInMemoryResultFormatter.class.getName()); + client3.setConnectionParam("deferPersistenceByMillis", "5000"); // property used for test only + client3.getConf().setLong(LensCliConfigConstants.QUERY_EXECUTE_TIMEOUT_MILLIS, 20000); + LensQueryCommands qCom3 = new LensQueryCommands(); + qCom3.setClient(client3); + + return new Object[][] { + { qCom1, "cube select id,name from test_dim", true, false }, + { qCom1, "cube select id,name1 from invalid_test_dim", false, true }, // this query should fail; + { qCom2, "cube select id,name from test_dim", true, true }, + { qCom3, "cube select id,name from test_dim", true, true }, }; + } + private void closeClientConnection(LensQueryCommands queryCommands) { queryCommands.getClient().closeConnection(); } @@ -99,13 +147,43 @@ public class TestLensQueryCommands extends LensCliApplicationTest { /** * Test execute sync query */ - @Test(dataProvider = "queryCommands") - public void executeSyncQuery(LensQueryCommands qCom) { + @Test(dataProvider = "executeSyncQueryDP") + public void executeSyncQuery(LensQueryCommands qCom, String sql, boolean shouldPass, boolean closeConn) + throws Exception{ assertEquals(qCom.getAllPreparedQueries("all", "", -1, -1), "No prepared queries"); - String sql = "cube select id,name from test_dim"; - String result = qCom.executeQuery(sql, false, "testQuery2"); - assertTrue(result.contains("1\tfirst"), result); - closeClientConnection(qCom); + String successfulQueries = qCom.getAllQueries("SUCCESSFUL", null, "all", null, -1, Long.MAX_VALUE); + int noOfSuccQueriesSoFar; + if (successfulQueries.contains("No queries")) { + noOfSuccQueriesSoFar = 0; + } else { + noOfSuccQueriesSoFar = Integer.parseInt(successfulQueries.substring(successfulQueries.lastIndexOf(": ") + 2)); + } + try { + String result = qCom.executeQuery(sql, false, "testQuerySync"); + assertTrue(result.contains("1\tfirst"), result); + } catch (Exception e) { + if (closeConn) { + closeClientConnection(qCom); + } + if (shouldPass) { + fail("Unexpected failure", e); + } else { + return; //simulated failure scenario + } + } + // Wait for query to reach successful state + int chkCount = 0; + while (!qCom.getAllQueries("SUCCESSFUL", null, "all", null, -1, Long.MAX_VALUE).contains( + "Total number of queries: " + (noOfSuccQueriesSoFar + 1))) { + Thread.sleep(2000); + chkCount++; + if (chkCount > 30) { + fail("Unable to get successful status for query even after 30 checks"); + } + } + if (closeConn) { + closeClientConnection(qCom); + } } /** @@ -152,7 +230,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { //Fetch again. Should not get resultset result = qCom.getQueryResults(handle, null, true); log.debug("Prepared Query Result is " + result); - assertTrue(result.contains("Resultset not available for the query"), "Query is not purged yet " + handle); + assertTrue(result.contains("Failed"), "Query is not purged yet " + handle); result = qCom.destroyPreparedQuery(qh); @@ -171,6 +249,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { assertFalse(handles2.contains(qh), handles2); result = qCom.destroyPreparedQuery(handles); assertEquals("Successfully destroyed " + handles, result); + closeClientConnection(qCom); } @@ -204,6 +283,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { log.debug(result); assertTrue(result.contains(explainPlan)); + closeClientConnection(qCom); } @@ -223,6 +303,8 @@ public class TestLensQueryCommands extends LensCliApplicationTest { result = qCom.explainAndPrepare(sql, ""); assertTrue(result.contains("Explain FAILED:")); + + closeClientConnection(qCom); } /** @@ -401,6 +483,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { downloadResult(qCom, qCom.getClient().getStatement().getQueryHandleString(), "testQuery3", "\"1\",\"first\""); System.out.println("@@END_PERSISTENT_RESULT_TEST-------------"); qCom.getClient().setConnectionParam("lens.query.enable.persistent.resultset.indriver", "false"); + closeClientConnection(qCom); } @@ -450,7 +533,48 @@ public class TestLensQueryCommands extends LensCliApplicationTest { assertFalse(result.contains("(-")); System.out.println("@@END_FINISHED_PURGED_RESULT_TEST-------------"); qCom.getClient().setConnectionParam("lens.query.enable.persistent.resultset", "false"); + closeClientConnection(qCom); } + @Test + public void testProxyLensQuery() throws Exception { + LensClient client = new LensClient(); + QueryHandle handle = client.executeQueryAsynch("cube select id,name from test_dim", "proxyTestQuery"); + client.getStatement().waitForQueryToComplete(handle); + LensQuery query = client.getQueryDetails(handle); + ProxyLensQuery proxyQuery = new ProxyLensQuery(client.getStatement(), handle); + Assert.assertEquals(query.getStatus().successful(), proxyQuery.getStatus().successful()); + Assert.assertEquals(query.getSubmissionTime(), proxyQuery.getSubmissionTime()); + Assert.assertEquals(query.getFinishTime(), proxyQuery.getFinishTime()); + + //Check is any new getters are added to LensQuery. If yes, ProxyLensQuery should be updated too + Set<String> lensQueryMethods = new HashSet<String>(); + for (Method m :query.getClass().getDeclaredMethods()) { + if (Modifier.isPublic(m.getModifiers()) && !m.getName().startsWith("hash") && !m.getName().startsWith("equals")) { + lensQueryMethods.add(m.getName()); + } + } + + Set<String> proxyLensQueryMethods = new HashSet<String>(); + for (Method m :proxyQuery.getClass().getDeclaredMethods()) { + if (Modifier.isPublic(m.getModifiers())) { + proxyLensQueryMethods.add(m.getName()); + } + } + + log.info("Methods in LensQuery: " + lensQueryMethods + "\nMethods in ProxyLensQuery: " + proxyLensQueryMethods); + assertTrue(lensQueryMethods.containsAll(proxyLensQueryMethods), + "Methods in LensQuery and ProxyLensQuery do not match"); + + + //Check equals and hashCode override + ProxyLensQuery proxyQuery2 = new ProxyLensQuery(client.getStatement(), handle); + Assert.assertEquals(proxyQuery, proxyQuery2); + Assert.assertEquals(proxyQuery, query); + Assert.assertEquals(proxyQuery.hashCode(), proxyQuery2.hashCode()); + Assert.assertEquals(proxyQuery.hashCode(), query.hashCode()); + + client.closeConnection(); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-client/src/main/java/org/apache/lens/client/LensClient.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java index 39d771c..9626820 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java @@ -64,7 +64,7 @@ public class LensClient { @Getter private PathValidator pathValidator; - public static Logger getCliLooger() { + public static Logger getCliLogger() { return LoggerFactory.getLogger(CLILOGGER); } @@ -100,13 +100,36 @@ public class LensClient { return mc; } - public LensAPIResult<QueryHandle> executeQueryAsynch(String sql, String queryName) throws LensAPIException { + public QueryHandle executeQueryAsynch(String sql, String queryName) throws LensAPIException { log.debug("Executing query {}", sql); - LensAPIResult<QueryHandle> lensAPIResult = statement.execute(sql, false, queryName); - LensQuery query = statement.getQuery(); - log.debug("Adding query to statementMap {}", query.getQueryHandle()); - statementMap.put(query.getQueryHandle(), statement); - return lensAPIResult; + QueryHandle handle = statement.executeQuery(sql, false, queryName); + statementMap.put(handle, statement); + return handle; + } + + /** + * Execute query with timeout option. + * If the query does not finish within the timeout time, server returns the query handle which can be used to + * track further progress. + * + * @param sql : query/command to be executed + * @param queryName : optional query name + * @param timeOutMillis : timeout milliseconds for the query execution. + * @return + * @throws LensAPIException + */ + public QueryHandleWithResultSet executeQueryWithTimeout(String sql, String queryName, long timeOutMillis) + throws LensAPIException { + log.info("Executing query {} with timeout of {} milliseconds", sql, timeOutMillis); + QueryHandleWithResultSet result = statement.executeQuery(sql, queryName, timeOutMillis); + statementMap.put(result.getQueryHandle(), statement); + if (result.getStatus().failed()) { + IdBriefErrorTemplate errorResult = new IdBriefErrorTemplate(IdBriefErrorTemplateKey.QUERY_ID, + result.getQueryHandle().getHandleIdString(), new BriefError(result.getStatus() + .getErrorCode(), result.getStatus().getErrorMessage())); + throw new LensBriefErrorException(errorResult); + } + return result; } public Date getLatestDateOfCube(String cubeName, String timePartition) { @@ -139,7 +162,7 @@ public class LensClient { public LensClientResultSetWithStats getResults(String sql, String queryName) throws LensAPIException { log.debug("Executing query {}", sql); - statement.execute(sql, true, queryName); + statement.executeQuery(sql, true, queryName); return getResultsFromStatement(statement); } http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-client/src/main/java/org/apache/lens/client/LensConnection.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java index eeb473a..d67e64e 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java @@ -311,7 +311,7 @@ public class LensConnection { return value.getElements(); } - LensConnectionParams getLensConnectionParams() { + public LensConnectionParams getLensConnectionParams() { return this.params; } http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-client/src/main/java/org/apache/lens/client/LensStatement.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java index 33c26e1..f06bcd1 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java @@ -31,9 +31,9 @@ import org.apache.lens.api.APIResult; import org.apache.lens.api.LensConf; import org.apache.lens.api.query.*; import org.apache.lens.api.query.QueryStatus.Status; - import org.apache.lens.api.result.LensAPIResult; import org.apache.lens.client.exceptions.LensAPIException; +import org.apache.lens.client.model.ProxyLensQuery; import org.apache.commons.lang.StringUtils; @@ -58,59 +58,42 @@ public class LensStatement { private LensQuery query; /** - * Execute. - * - * @param sql the sql - * @param waitForQueryToComplete the wait for query to complete - * @param queryName the query name - */ - public LensAPIResult<QueryHandle> execute(String sql, boolean waitForQueryToComplete, - String queryName) throws LensAPIException { - LensAPIResult<QueryHandle> lensAPIResult = executeQuery(sql, waitForQueryToComplete, queryName); - this.query = getQuery(lensAPIResult.getData()); - return lensAPIResult; - } - - /** - * Execute. - * - * @param sql the sql - * @param queryName the query name - */ - public void execute(String sql, String queryName) throws LensAPIException { - QueryHandle handle = executeQuery(sql, true, queryName).getData(); - this.query = getQuery(handle); - } - - /** - * Execute query. + * This method can be used for executing a query. If waitForQueryToComplete is false, the call to this method returns + * immediately after submitting the query to the server without waiting for it to complete execution. + * <p> + * {@link #getStatus(QueryHandle)} can be used to track to track the query progress and + * {@link #getQuery(QueryHandle)} can be used to get complete details (including status) about the query. * * @param sql the sql * @param waitForQueryToComplete the wait for query to complete * @param queryName the query name * @return the query handle */ - public LensAPIResult<QueryHandle> executeQuery(String sql, boolean waitForQueryToComplete, - String queryName) throws LensAPIException { + public QueryHandle executeQuery(String sql, boolean waitForQueryToComplete, String queryName) + throws LensAPIException { - LensAPIResult<QueryHandle> lensAPIResult = executeQuery(sql, queryName); + QueryHandle handle = submitQuery(sql, queryName); if (waitForQueryToComplete) { - waitForQueryToComplete(lensAPIResult.getData()); + waitForQueryToComplete(handle); } - return lensAPIResult; + return handle; } /** - * Execute query. + * This method can be used for executing a prepared query. If waitForQueryToComplete is false, the call to this method + * returns immediately after submitting the query to the server without waiting for it to complete execution. + * <p> + * {@link #getStatus(QueryHandle)} can be used to track to track the query progress and + * {@link #getQuery(QueryHandle)} can be used to get complete details (including status) about the query. * - * @param phandle the phandle + * @param phandle the prepared query handle * @param waitForQueryToComplete the wait for query to complete * @param queryName the query name * @return the query handle */ public QueryHandle executeQuery(QueryPrepareHandle phandle, boolean waitForQueryToComplete, String queryName) { - QueryHandle handle = executeQuery(phandle, queryName); + QueryHandle handle = submitQuery(phandle, queryName); if (waitForQueryToComplete) { waitForQueryToComplete(handle); @@ -119,6 +102,57 @@ public class LensStatement { } /** + * This method can be used for executing query. The method waits for timeOutMillis time OR query execution to succeed, + * which ever happens first, before returning the response to the caller. + * <p> + * If the query execution finishes before timeout, user can check the query Status (SUCCESSFUL/FAILED) using + * {@link QueryHandleWithResultSet#getStatus()} and access the result of SUCCESSFUL query via + * {@link QueryHandleWithResultSet#getResult()} and {@link QueryHandleWithResultSet#getResultMetadata()}. + * <p> + * If the query does not finish within the timeout, user can use {@link #getStatus(QueryHandle)} to track + * the query progress and {@link #getQuery(QueryHandle)} to get complete details (including status) about + * the query. Once the query has reached SUCCESSFUL state, user can access the results via + * {@link #getResultSet(LensQuery)} and {@link #getResultSetMetaData(LensQuery)} + * + * @param sql : query/command to be executed + * @param queryName : optional query name + * @param timeOutMillis : timeout milliseconds + * @return QueryHandleWithResultSet + * @throws LensAPIException + */ + public QueryHandleWithResultSet executeQuery(String sql, String queryName, long timeOutMillis) + throws LensAPIException { + if (!connection.isOpen()) { + throw new IllegalStateException("Lens Connection has to be established before querying"); + } + + Client client = connection.buildClient(); + FormDataMultiPart mp = new FormDataMultiPart(); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), connection + .getSessionHandle(), MediaType.APPLICATION_XML_TYPE)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), sql)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "EXECUTE_WITH_TIMEOUT")); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("timeoutmillis").build(), "" + timeOutMillis)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("queryName").build(), queryName == null ? "" + : queryName)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(), + MediaType.APPLICATION_XML_TYPE)); + WebTarget target = getQueryWebTarget(client); + + Response response = + target.request(MediaType.APPLICATION_XML_TYPE).post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); + + if (response.getStatus() == Response.Status.OK.getStatusCode()) { + QueryHandleWithResultSet result = + response.readEntity(new GenericType<LensAPIResult<QueryHandleWithResultSet>>() {}).getData(); + this.query = new ProxyLensQuery(this, result.getQueryHandle()); + return result; + } + + throw new LensAPIException(response.readEntity(LensAPIResult.class)); + } + + /** * Prepare query. * * @param sql the sql @@ -200,27 +234,27 @@ public class LensStatement { * @param handle the handle */ public void waitForQueryToComplete(QueryHandle handle) { - LensClient.getCliLooger().info("Query handle: {}", handle); - query = getQuery(handle); - while (query.queued()) { - query = getQuery(handle); - LensClient.getCliLooger().debug("Query {} status: {}", handle, query.getStatus()); + LensClient.getCliLogger().info("Query handle: {}", handle); + LensQuery queryDetails = getQuery(handle); + while (queryDetails.getStatus().queued()) { + queryDetails = getQuery(handle); + LensClient.getCliLogger().debug("Query {} status: {}", handle, queryDetails.getStatus()); try { Thread.sleep(connection.getLensConnectionParams().getQueryPollInterval()); } catch (InterruptedException e) { throw new IllegalStateException(e); } } - LensClient.getCliLooger().info("User query: '{}' was submitted to {}", query.getUserQuery(), - query.getSelectedDriverName()); - if (query.getDriverQuery() != null) { - LensClient.getCliLooger().info(" Driver query: '{}' and Driver handle: {}", query.getDriverQuery(), - query.getDriverOpHandle()); + LensClient.getCliLogger().info("User query: '{}' was submitted to {}", queryDetails.getUserQuery(), + queryDetails.getSelectedDriverName()); + if (queryDetails.getDriverQuery() != null) { + LensClient.getCliLogger().info(" Driver query: '{}' and Driver handle: {}", queryDetails.getDriverQuery(), + queryDetails.getDriverOpHandle()); } - while (!query.getStatus().finished() - && !(query.getStatus().getStatus().equals(Status.CLOSED))) { - query = getQuery(handle); - LensClient.getCliLooger().info("Query Status:{} ", query.getStatus()); + while (!queryDetails.getStatus().finished() + && !(queryDetails.getStatus().getStatus().equals(Status.CLOSED))) { + queryDetails = getQuery(handle); + LensClient.getCliLogger().info("Query Status:{} ", queryDetails.getStatus()); try { Thread.sleep(connection.getLensConnectionParams().getQueryPollInterval()); } catch (InterruptedException e) { @@ -261,9 +295,8 @@ public class LensStatement { try { Client client = connection.buildClient(); WebTarget target = getQueryWebTarget(client); - this.query = target.path(handle.toString()).queryParam("sessionid", connection.getSessionHandle()).request() + return target.path(handle.toString()).queryParam("sessionid", connection.getSessionHandle()).request() .get(LensQuery.class); - return query; } catch (Exception e) { log.error("Failed to get query status, cause:", e); throw new IllegalStateException("Failed to get query status, cause:" + e.getMessage()); @@ -295,7 +328,7 @@ public class LensStatement { * @param queryName the query name * @return the query handle */ - private LensAPIResult<QueryHandle> executeQuery(String sql, String queryName) throws LensAPIException { + private QueryHandle submitQuery(String sql, String queryName) throws LensAPIException { if (!connection.isOpen()) { throw new IllegalStateException("Lens Connection has to be established before querying"); } @@ -305,7 +338,7 @@ public class LensStatement { mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), connection .getSessionHandle(), MediaType.APPLICATION_XML_TYPE)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), sql)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "EXECUTE")); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("queryName").build(), queryName == null ? "" : queryName)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(), @@ -315,7 +348,9 @@ public class LensStatement { Response response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); if (response.getStatus() == Response.Status.OK.getStatusCode()) { - return response.readEntity(new GenericType<LensAPIResult<QueryHandle>>() {}); + QueryHandle handle = response.readEntity(new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); + this.query = new ProxyLensQuery(this, handle); + return handle; } throw new LensAPIException(response.readEntity(LensAPIResult.class)); @@ -327,8 +362,9 @@ public class LensStatement { * @param phandle the phandle * @param queryName the query name * @return the query handle + * @throws LensAPIException */ - public QueryHandle executeQuery(QueryPrepareHandle phandle, String queryName) { + private QueryHandle submitQuery(QueryPrepareHandle phandle, String queryName) { if (!connection.isOpen()) { throw new IllegalStateException("Lens Connection has to be " + "established before querying"); } @@ -344,8 +380,9 @@ public class LensStatement { mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(), MediaType.APPLICATION_XML_TYPE)); QueryHandle handle = target.request() - .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), QueryHandle.class); + .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), QueryHandle.class); + this.query = new ProxyLensQuery(this, handle); return handle; } @@ -420,8 +457,11 @@ public class LensStatement { return handles; } + /** + * Gets the result set meta data for the most recently executed query. + */ public QueryResultSetMetadata getResultSetMetaData() { - return this.getResultSetMetaData(query); + return this.getResultSetMetaData(this.getQuery()); } /** @@ -447,12 +487,18 @@ public class LensStatement { } } + /** + * Gets result set for the most recently executed query. + */ public QueryResult getResultSet() { - return this.getResultSet(this.query); + return this.getResultSet(this.getQuery()); } + /** + * Gets http result set for the most recently executed query. + */ public Response getHttpResultSet() { - return this.getHttpResultSet(this.query); + return this.getHttpResultSet(this.getQuery()); } /** @@ -501,12 +547,12 @@ public class LensStatement { } /** - * Kill. + * Kill the most recently submitted query via any executeQuery methods. * * @return true, if successful */ public boolean kill() { - return this.kill(query); + return this.kill(this.getQuery()); } /** @@ -536,7 +582,7 @@ public class LensStatement { * @return true, if successful */ public boolean closeResultSet() { - if (!query.getStatus().isResultSetAvailable()) { + if (!this.getQuery().getStatus().isResultSetAvailable()) { return false; } Client client = connection.buildClient(); @@ -565,35 +611,73 @@ public class LensStatement { } public boolean isIdle() { - return query == null || query.getStatus().finished(); + return query == null || this.getQuery().getStatus().finished(); } /** - * Was query successful. + * Was the most recently executed query successful. * * @return true, if successful */ public boolean wasQuerySuccessful() { - return query.getStatus().getStatus().equals(QueryStatus.Status.SUCCESSFUL); + return this.getQuery().getStatus().getStatus().equals(QueryStatus.Status.SUCCESSFUL); + } + + /** + * Gets status for query represented by handle. + */ + public QueryStatus getStatus(QueryHandle handle) { + return getQuery(handle).getStatus(); } + /** + * Gets the status for the most recently executed query. + */ public QueryStatus getStatus() { return getQuery().getStatus(); } + /** + * Gets details of the most recently executed query through {@link #executeQuery(QueryPrepareHandle, boolean, String)} + * or {@link #executeQuery(String, boolean, String)} or {@link #executeQuery(String, String, long)} + * <p> + * Note: Cached query details are returned if the query has finished. If the query has still not finished it fetches + * the latest details from server again. + */ public LensQuery getQuery() { + if (this.query != null && !this.query.getStatus().finished()) { + // Get Updated Query if the query has not finished yet. + this.query = getQuery(this.query.getQueryHandle()); + } return this.query; } + /** + *Gets the error code, if any, for the most recently executed query. + */ public int getErrorCode() { - return this.query.getErrorCode(); + return this.getQuery().getErrorCode(); } + /** + * Gets the error message, if any, for the most recently executed query. + */ public String getErrorMessage() { - return this.query.getErrorMessage(); + return this.getQuery().getErrorMessage(); } + /** + * Gets the query handle string for the most recently executed query. + */ public String getQueryHandleString() { return this.query.getQueryHandleString(); } + + /** + * Gets the user for the lens session + */ + public String getUser() { + return this.connection.getLensConnectionParams().getUser(); + } } + http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcStatement.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcStatement.java b/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcStatement.java index 10f7155..fb265e5 100644 --- a/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcStatement.java +++ b/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcStatement.java @@ -58,7 +58,7 @@ public class LensJdbcStatement implements Statement { @Override public ResultSet executeQuery(String s) throws SQLException { try { - statement.execute(s, null); + statement.executeQuery(s, true, null); } catch (LensAPIException e) { log.error("Execution Failed for Statement:{}", s, e); } @@ -115,7 +115,7 @@ public class LensJdbcStatement implements Statement { throw new SQLException("Cannot execute statemes on closed statements"); } try { - statement.execute(s, true, null); + statement.executeQuery(s, true, null); } catch (Throwable t) { throw new SQLException(t); } http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-client/src/main/java/org/apache/lens/client/model/ProxyLensQuery.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/model/ProxyLensQuery.java b/lens-client/src/main/java/org/apache/lens/client/model/ProxyLensQuery.java new file mode 100644 index 0000000..e8b3488 --- /dev/null +++ b/lens-client/src/main/java/org/apache/lens/client/model/ProxyLensQuery.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.client.model; + +import org.apache.lens.api.LensConf; +import org.apache.lens.api.Priority; +import org.apache.lens.api.query.LensQuery; +import org.apache.lens.api.query.QueryHandle; +import org.apache.lens.api.query.QueryStatus; +import org.apache.lens.client.LensStatement; + +import lombok.extern.slf4j.Slf4j; + +/** + * This class can be used to create Proxy Lens Query objects. The Proxy objects support lazy initialization + * of members of this class given a query handle and LensStatement. + * <p> + * In most cases the query handle information should suffice which is available locally, and only in a few cases + * like {@link org.apache.lens.client.LensClient.LensClientResultSetWithStats}, extra information needs to be fetched + * from Lens Server. + * <p> + * Note:This class if not meant to be instantiated by lens apps (using lens-client to interact with lens server) + * directly + */ +@Slf4j +public class ProxyLensQuery extends LensQuery { + + private LensStatement statement; + private QueryHandle queryHandle; + private boolean isFullyInitialized; + private LensQuery actualLensQuery; + + public ProxyLensQuery(LensStatement statement, QueryHandle queryHandle) { + this.statement = statement; + this.queryHandle = queryHandle; + } + + @Override + public QueryHandle getQueryHandle() { + return this.queryHandle; + } + + @Override + public String getSubmittedUser() { + return this.statement.getUser(); + } + + @Override + public String getUserQuery() { + return getQuery().getUserQuery(); + } + + @Override + public Priority getPriority() { + return getQuery().getPriority(); + } + + @Override + public boolean isPersistent() { + return getQuery().isPersistent(); + } + + @Override + public String getSelectedDriverName() { + return getQuery().getSelectedDriverName(); + } + + @Override + public String getDriverQuery() { + return getQuery().getDriverQuery(); + } + + @Override + public QueryStatus getStatus() { + return getQuery().getStatus(); + } + + @Override + public String getResultSetPath() { + return getQuery().getResultSetPath(); + } + + @Override + public String getDriverOpHandle() { + return getQuery().getDriverOpHandle(); + } + + @Override + public LensConf getQueryConf() { + return getQuery().getQueryConf(); + } + + @Override + public long getSubmissionTime() { + return getQuery().getSubmissionTime(); + } + + @Override + public long getLaunchTime() { + return getQuery().getLaunchTime(); + } + + @Override + public long getDriverStartTime() { + return getQuery().getDriverStartTime(); + } + + @Override + public long getDriverFinishTime() { + return getQuery().getDriverFinishTime(); + } + + @Override + public long getFinishTime() { + return getQuery().getFinishTime(); + } + + @Override + public long getClosedTime() { + return getQuery().getClosedTime(); + } + + @Override + public String getQueryName() { + return getQuery().getQueryName(); + } + + @Override + public Integer getErrorCode() { + return getQuery().getErrorCode(); + } + + @Override + public String getErrorMessage() { + return getQuery().getErrorMessage(); + } + + @Override + public String getQueryHandleString() { + return getQuery().getQueryHandleString(); + } + + private synchronized LensQuery getQuery() { + if (!isFullyInitialized) { + this.actualLensQuery = statement.getQuery(queryHandle); + this.isFullyInitialized = true; + } + return actualLensQuery; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-client/src/main/resources/lens-client-default.xml ---------------------------------------------------------------------- diff --git a/lens-client/src/main/resources/lens-client-default.xml b/lens-client/src/main/resources/lens-client-default.xml index 35b2d28..5a578a7 100644 --- a/lens-client/src/main/resources/lens-client-default.xml +++ b/lens-client/src/main/resources/lens-client-default.xml @@ -56,4 +56,11 @@ <value>false</value> <description>Should CLI try to prettify the JSON of an object before priting.</description> </property> + <property> + <name>lens.cli.query.execute.timeout.millis</name> + <value>10000</value> + <description>This property defines the timeout value when sync or --async false option + is used to execute a query. The default value is 10 seconds. + </description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java b/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java index 805a282..e88ed28 100644 --- a/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java +++ b/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java @@ -134,7 +134,7 @@ public class SampleQueries { total++; System.out.println("Query:" + query); try { - QueryHandle handle = queryClient.executeQuery(query, true, null).getData(); + QueryHandle handle = queryClient.executeQuery(query, true, null); System.out.println("Status:" + queryClient.getQuery().getStatus()); System.out.println("Total time in millis:" + (queryClient.getQuery().getFinishTime() - queryClient.getQuery().getSubmissionTime())); @@ -160,7 +160,7 @@ public class SampleQueries { retCode = 1; } } catch (Exception e) { - LensClient.getCliLooger().error("Exception for example query : \"{}\"", query, e); + LensClient.getCliLogger().error("Exception for example query : \"{}\"", query, e); retCode = 1; } System.out.println("--------------------"); http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java ---------------------------------------------------------------------- diff --git a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java index b4f43ec..860b219 100644 --- a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java +++ b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java @@ -164,7 +164,7 @@ public class LensRDDClient { * @throws LensAPIException the lens exception */ public QueryHandle createLensRDDAsync(String query) throws LensAPIException { - return getClient().executeQueryAsynch(query, "").getData(); + return getClient().executeQueryAsynch(query, ""); } /** http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 581530f..47efb1c 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -540,7 +540,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE driverRS = ctx.getSelectedDriver().fetchResultSet(getCtx()); } catch (Exception e) { log.error( - "Error while getting result ser form driver {}. Driver result set based purging logic will be ignored", + "Error while getting result set form driver {}. Driver result set based purging logic will be ignored", ctx.getSelectedDriver(), e); } } http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index 49de59c..df13ba2 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -1359,7 +1359,7 @@ public class TestQueryService extends LensJerseyTest { false, true, MediaType.APPLICATION_XML_TYPE); } - private static class DeferredInMemoryResultFormatter extends FileSerdeFormatter { + public static class DeferredInMemoryResultFormatter extends FileSerdeFormatter { /** * Defer init so that this output formatter takes significant time. */ @@ -1370,7 +1370,7 @@ public class TestQueryService extends LensJerseyTest { } } - private static class DeferredPersistentResultFormatter extends FilePersistentFormatter { + public static class DeferredPersistentResultFormatter extends FilePersistentFormatter { /** * Defer init so that this output formatter takes significant time. */ http://git-wip-us.apache.org/repos/asf/lens/blob/79b95f0e/src/site/apt/user/client-config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/user/client-config.apt b/src/site/apt/user/client-config.apt index 714db18..4ed41b4 100644 --- a/src/site/apt/user/client-config.apt +++ b/src/site/apt/user/client-config.apt @@ -26,16 +26,18 @@ Lens client configuration *--+--+---+--+ |1|lens.cli.json.pretty|false|Should CLI try to prettify the JSON of an object before priting.| *--+--+---+--+ -|2|lens.client.dbname|default|Default lens database| +|2|lens.cli.query.execute.timeout.millis|10000|This property defines the timeout value when sync or --async false option is used to execute a query. The default value is 10 seconds.| *--+--+---+--+ -|3|lens.client.query.poll.interval|10|Interval at which query progress will be polled. Interval has to be given in milliseconds| +|3|lens.client.dbname|default|Default lens database| *--+--+---+--+ -|4|lens.client.requestfilter.ws.filter.impl|org.apache.lens.client.RequestFilter|Implementation class for Request Filter| +|4|lens.client.query.poll.interval|10000|Interval at which query progress will be polled. Interval has to be given in milliseconds| *--+--+---+--+ -|5|lens.client.user.name|anonymous|Lens client user name| +|5|lens.client.requestfilter.ws.filter.impl|org.apache.lens.client.RequestFilter|Implementation class for Request Filter| *--+--+---+--+ -|6|lens.client.ws.request.filternames|requestfilter|These JAX-RS filters would be started in the specified order when lens-client starts| +|6|lens.client.user.name|anonymous|Lens client user name| *--+--+---+--+ -|7|lens.server.base.url|http://0.0.0.0:9999/lensapi|The base url for the lens server| +|7|lens.client.ws.request.filternames|requestfilter|These JAX-RS filters would be started in the specified order when lens-client starts| +*--+--+---+--+ +|8|lens.server.base.url|http://0.0.0.0:9999/lensapi|The base url for the lens server| *--+--+---+--+ The configuration parameters and their default values
