Repository: incubator-lens Updated Branches: refs/heads/master 063a3766f -> d47af610c
LENS-26 : Add cli option to download the result into a file ( Rajat Khandelwal via amareshwari) Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/d47af610 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/d47af610 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/d47af610 Branch: refs/heads/master Commit: d47af610cf329815795dd7151319bebf4a5394d5 Parents: 063a376 Author: Rajat Khandelwal <[email protected]> Authored: Fri May 22 10:32:13 2015 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Fri May 22 10:32:13 2015 +0530 ---------------------------------------------------------------------- checkstyle/src/main/resources/checkstyle.xml | 6 +- .../lens/cli/commands/BaseLensCommand.java | 11 +- .../lens/cli/commands/LensCRUDCommand.java | 4 +- .../LensCRUDStoragePartitionCommand.java | 6 +- .../lens/cli/commands/LensQueryCommands.java | 58 +++++++-- .../apache/lens/cli/TestLensQueryCommands.java | 120 +++++++++++-------- .../lens/cli/doc/TestGenerateCLIUserDoc.java | 7 +- .../java/org/apache/lens/client/LensClient.java | 57 +++++---- .../org/apache/lens/client/LensStatement.java | 32 ++++- .../org/apache/lens/driver/hive/HiveDriver.java | 10 +- .../lens/driver/hive/HiveInMemoryResultSet.java | 19 ++- .../apache/lens/driver/jdbc/JDBCResultSet.java | 23 +++- .../server/api/driver/InMemoryResultSet.java | 1 + .../lens/server/api/driver/MockDriver.java | 5 + .../server/query/QueryExecutionServiceImpl.java | 10 +- src/site/apt/user/cli.apt | 2 +- 16 files changed, 250 insertions(+), 121 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/checkstyle/src/main/resources/checkstyle.xml ---------------------------------------------------------------------- diff --git a/checkstyle/src/main/resources/checkstyle.xml b/checkstyle/src/main/resources/checkstyle.xml index 236f977..d6ff839 100644 --- a/checkstyle/src/main/resources/checkstyle.xml +++ b/checkstyle/src/main/resources/checkstyle.xml @@ -240,9 +240,9 @@ <!-- allow warnings to be suppressed --> <module name="SuppressionCommentFilter"> - <property name="offCommentFormat" value="SUSPEND CHECKSTYLE CHECK ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck"/> - <property name="onCommentFormat" value="RESUME CHECKSTYLE CHECK ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck"/> - <property name="checkFormat" value="ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck"/> + <property name="offCommentFormat" value="SUSPEND CHECKSTYLE CHECK ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck|InnerAssignmentCheck"/> + <property name="onCommentFormat" value="RESUME CHECKSTYLE CHECK ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck|InnerAssignmentCheck"/> + <property name="checkFormat" value="ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck|InnerAssignmentCheck"/> </module> <!--module name="SuppressionFilter"> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java index 8707fec..d6055cd 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java @@ -140,14 +140,21 @@ public class BaseLensCommand implements ExecutionProcessor { .replaceAll("]", "\n").replaceAll(",", "").replaceAll("\"", "").replaceAll("\n\n", "\n"); } - public String getValidPath(String path) { + public String getValidPath(String path, boolean shouldBeDirectory, boolean shouldExist) { + path = path.replaceAll("/$", ""); if (path.startsWith("~")) { path = path.replaceFirst("~", System.getProperty("user.home")); } File f = new File(path); - if (!f.exists()) { + if (shouldExist && !f.exists()) { throw new RuntimeException("Path " + path + " doesn't exist."); } + if (shouldBeDirectory && !f.isDirectory()) { + throw new RuntimeException("Path " + path + " is not a directory"); + } + if (!shouldBeDirectory && f.isDirectory()) { + throw new RuntimeException("Path " + path + " is a directory"); + } return f.getAbsolutePath(); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDCommand.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDCommand.java index 05922e3..9cb7d69 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDCommand.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDCommand.java @@ -38,7 +38,7 @@ public abstract class LensCRUDCommand<T> extends BaseLensCommand { } public String create(String path, boolean ignoreIfExists) { - return doCreate(getValidPath(path), ignoreIfExists).getStatus().toString().toLowerCase(); + return doCreate(getValidPath(path, false, true), ignoreIfExists).getStatus().toString().toLowerCase(); } public String describe(String name) { @@ -50,7 +50,7 @@ public abstract class LensCRUDCommand<T> extends BaseLensCommand { } public String update(String entity, String path) { - return doUpdate(entity, getValidPath(path)).getStatus().toString().toLowerCase(); + return doUpdate(entity, getValidPath(path, false, true)).getStatus().toString().toLowerCase(); } public String drop(String name, boolean cascade) { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java index 0d53e21..208081d 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java @@ -53,7 +53,7 @@ public abstract class LensCRUDStoragePartitionCommand<T> extends LensCRUDCommand } public String addStorage(String tableName, String path) { - return doAddStorage(tableName, getValidPath(path)).toString().toLowerCase(); + return doAddStorage(tableName, getValidPath(path, false, true)).toString().toLowerCase(); } public String getStorage(String tableName, String storage) { @@ -81,11 +81,11 @@ public abstract class LensCRUDStoragePartitionCommand<T> extends LensCRUDCommand } public String addPartition(String tableName, String storageName, String path) { - return doAddPartition(tableName, storageName, getValidPath(path)).toString().toLowerCase(); + return doAddPartition(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase(); } public String addPartitions(String tableName, String storageName, String path) { - return doAddPartitions(tableName, storageName, getValidPath(path)).toString().toLowerCase(); + return doAddPartitions(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase(); } public String dropPartitions(String tableName, String storageName, String filter) { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java index 64e9ca4..6c4a1b5 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java @@ -18,18 +18,18 @@ */ package org.apache.lens.cli.commands; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.UnsupportedEncodingException; +import java.io.*; import java.nio.charset.Charset; import java.util.List; import java.util.UUID; +import javax.ws.rs.core.Response; + import org.apache.lens.api.query.*; import org.apache.lens.cli.commands.annotations.UserDocumentation; import org.apache.lens.client.LensClient; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.springframework.shell.core.annotation.CliCommand; @@ -40,6 +40,7 @@ import com.google.common.base.Joiner; /** * The Class LensQueryCommands. + * SUSPEND CHECKSTYLE CHECK InnerAssignmentCheck */ @Component @UserDocumentation(title = "Query Management", @@ -200,10 +201,10 @@ public class LensQueryCommands extends BaseLensCommand { return "Explain FAILED:" + plan.getErrorMsg(); } if (StringUtils.isNotBlank(location)) { - String validPath = getValidPath(location); - OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(validPath), Charset.defaultCharset()); - osw.write(plan.getPlanString()); - osw.close(); + String validPath = getValidPath(location, false, false); + try (OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(validPath), Charset.defaultCharset())) { + osw.write(plan.getPlanString()); + } return "Saved to " + validPath; } return plan.getPlanString(); @@ -262,13 +263,44 @@ public class LensQueryCommands extends BaseLensCommand { * @param qh the qh * @return the query results */ - @CliCommand(value = "query results", help = "get results of async query with query handle <query_handle>") + @CliCommand(value = "query results", + help = "get results of async query with query handle <query_handle>. Can " + + "optionally save the results to a file by providing <save_location>") public String getQueryResults( - @CliOption(key = {"", "query_handle"}, mandatory = true, help = "<query_handle>") String qh) { + @CliOption(key = {"", "query_handle"}, mandatory = true, help = "<query_handle>") String qh, + @CliOption(key = {"save_location"}, mandatory = false, help = "<save_location>") String location) { + QueryHandle queryHandle = new QueryHandle(UUID.fromString(qh)); try { - LensClient.LensClientResultSetWithStats result = getClient() - .getAsyncResults(new QueryHandle(UUID.fromString(qh))); - return formatResultSet(result); + String prefix = ""; + if (StringUtils.isNotBlank(location)) { + location = getValidPath(location, true, true); + Response response = getClient().getHttpResults(queryHandle); + if (response.getStatus() == Response.Status.OK.getStatusCode()) { + String disposition = (String) response.getHeaders().get("content-disposition").get(0); + String fileName = disposition.split("=")[1].trim(); + location = getValidPath(location + File.separator + fileName, false, false); + try (InputStream stream = response.readEntity(InputStream.class); + FileOutputStream outStream = new FileOutputStream(new File(location))) { + IOUtils.copy(stream, outStream); + } + return "Saved to " + location; + } else { + LensClient.LensClientResultSetWithStats results = getClient().getAsyncResults(queryHandle); + if (results.getResultSet().getResult() instanceof InMemoryQueryResult) { + location = getValidPath(location + File.separator + qh + ".csv", false, false); + try (OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(location), + Charset.defaultCharset())) { + osw.write(formatResultSet(results)); + } + return "Saved to " + location; + } else { + return "Can't download the result because it's available in driver's persistence.\n" + + formatResultSet(results); + } + } + } else { + return formatResultSet(getClient().getAsyncResults(queryHandle)); + } } catch (Throwable t) { return t.getMessage(); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java index d7b6372..5724d24 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java @@ -18,9 +18,12 @@ */ package org.apache.lens.cli; -import java.io.File; +import static org.testng.Assert.*; + +import java.io.*; import java.net.URL; import java.util.GregorianCalendar; +import java.util.Scanner; import java.util.UUID; import javax.ws.rs.BadRequestException; @@ -42,7 +45,6 @@ import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testng.Assert; import org.testng.annotations.Test; /** @@ -65,6 +67,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { + " test_dim\n" + " name\n" + " TOK_WHERE\n" + " =\n" + " .\n" + " TOK_TABLE_OR_COL\n" + " test_dim\n" + " dt\n" + " 'latest'"; + private File resDir; /** * Test query commands. @@ -78,6 +81,8 @@ public class TestLensQueryCommands extends LensCliApplicationTest { setup(client); LensQueryCommands qCom = new LensQueryCommands(); qCom.setClient(client); + resDir = new File("target/results"); + assertTrue(resDir.exists() || resDir.mkdirs()); testExecuteSyncQuery(qCom); testExecuteAsyncQuery(qCom); testExplainQuery(qCom); @@ -92,7 +97,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { client.setConnectionParam("lens.query.enable.metrics.per.query", "true"); qCom.setClient(client); String result = qCom.getAllPreparedQueries("all", "", -1, -1); - Assert.assertEquals(result, "No prepared queries"); + assertEquals(result, "No prepared queries"); testExecuteSyncQuery(qCom); testExecuteAsyncQuery(qCom); testExplainQuery(qCom); @@ -114,19 +119,19 @@ public class TestLensQueryCommands extends LensCliApplicationTest { String sql = "cube select id, name from test_dim"; String result = qCom.getAllPreparedQueries("testPreparedName", "all", submitTime, Long.MAX_VALUE); - Assert.assertEquals(result, "No prepared queries"); + assertEquals(result, "No prepared queries"); final String qh = qCom.prepare(sql, "testPreparedName"); result = qCom.getAllPreparedQueries("testPreparedName", "all", submitTime, System.currentTimeMillis()); - Assert.assertEquals(qh, result); + assertEquals(qh, result); result = qCom.getPreparedStatus(qh); - Assert.assertTrue(result.contains("User query:cube select id, name from test_dim")); - Assert.assertTrue(result.contains(qh)); + assertTrue(result.contains("User query:cube select id, name from test_dim")); + assertTrue(result.contains(qh)); result = qCom.executePreparedQuery(qh, false, "testPrepQuery1"); LOG.warn("XXXXXX Prepared query sync result is " + result); - Assert.assertTrue(result.contains("1\tfirst")); + assertTrue(result.contains("1\tfirst")); String handle = qCom.executePreparedQuery(qh, true, "testPrepQuery2"); LOG.debug("Perpared query handle is " + handle); @@ -135,29 +140,33 @@ public class TestLensQueryCommands extends LensCliApplicationTest { } String status = qCom.getStatus(handle); LOG.debug("Prepared Query Status is " + status); - Assert.assertTrue(status.contains("Status : SUCCESSFUL")); + assertTrue(status.contains("Status : SUCCESSFUL")); - result = qCom.getQueryResults(handle); + result = qCom.getQueryResults(handle, null); + LOG.debug("Prepared Query Result is " + result); + assertTrue(result.contains("1\tfirst")); + // Fetch again. + result = qCom.getQueryResults(handle, null); LOG.debug("Prepared Query Result is " + result); - Assert.assertTrue(result.contains("1\tfirst")); + assertTrue(result.contains("1\tfirst")); result = qCom.destroyPreparedQuery(qh); LOG.debug("destroy result is " + result); - Assert.assertEquals("Successfully destroyed " + qh, result); + assertEquals("Successfully destroyed " + qh, result); result = qCom.getAllPreparedQueries("testPreparedName", "all", submitTime, Long.MAX_VALUE); - Assert.assertEquals(result, "No prepared queries"); + assertEquals(result, "No prepared queries"); final String qh2 = qCom.explainAndPrepare(sql, "testPrepQuery3"); - Assert.assertTrue(qh2.contains(explainPlan)); + assertTrue(qh2.contains(explainPlan)); String handles = qCom.getAllPreparedQueries("testPrepQuery3", "all", -1, Long.MAX_VALUE); - Assert.assertFalse(handles.contains("No prepared queries"), handles); + assertFalse(handles.contains("No prepared queries"), handles); String handles2 = qCom.getAllPreparedQueries("testPrepQuery3", "all", -1, submitTime - 1); - Assert.assertFalse(handles2.contains(qh), handles2); + assertFalse(handles2.contains(qh), handles2); result = qCom.destroyPreparedQuery(handles); - Assert.assertEquals("Successfully destroyed " + handles, result); + assertEquals("Successfully destroyed " + handles, result); } /** @@ -172,7 +181,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { qCom.setClient(client); String sql = "cube select id, name from test_dim"; final String result = qCom.explainAndPrepare(sql, "testFailPrepared"); - Assert.assertTrue(result.contains("Explain FAILED:Error while processing statement: FAILED: Hive Internal Error:" + assertTrue(result.contains("Explain FAILED:Error while processing statement: FAILED: Hive Internal Error:" + " java.lang.ClassNotFoundException(org.apache.lens.driver.hive.TestHiveDriver.FailHook)")); } @@ -187,7 +196,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { String result = qCom.explainQuery(sql, ""); LOG.debug(result); - Assert.assertTrue(result.contains(explainPlan)); + assertTrue(result.contains(explainPlan)); } @@ -202,10 +211,10 @@ public class TestLensQueryCommands extends LensCliApplicationTest { String result = qCom.explainQuery(sql, ""); LOG.debug(result); - Assert.assertTrue(result.contains("Explain FAILED:")); + assertTrue(result.contains("Explain FAILED:")); result = qCom.explainAndPrepare(sql, ""); - Assert.assertTrue(result.contains("Explain FAILED:")); + assertTrue(result.contains("Explain FAILED:")); } /** @@ -222,61 +231,68 @@ public class TestLensQueryCommands extends LensCliApplicationTest { String user = qCom.getClient().getLensStatement(new QueryHandle(UUID.fromString(qh))).getQuery().getSubmittedUser(); String result = qCom.getAllQueries("", "testQuery1", user, -1, Long.MAX_VALUE); // this is because previous query has run two query handle will be there - Assert.assertTrue(result.contains(qh), result); - Assert.assertTrue(result.contains("Total number of queries")); + assertTrue(result.contains(qh), result); + assertTrue(result.contains("Total number of queries")); String[] resultSplits = result.split("\n"); // assert on the number of queries - Assert.assertEquals(String.valueOf(resultSplits.length - 1), resultSplits[resultSplits.length - 1].split(": ")[1]); + assertEquals(String.valueOf(resultSplits.length - 1), resultSplits[resultSplits.length - 1].split(": ")[1]); String details = qCom.getDetails(qh); - Assert.assertTrue(details.contains("driverQuery"), details); + assertTrue(details.contains("driverQuery"), details); // Check that query name searching is 'ilike' String result2 = qCom.getAllQueries("", "query", "all", -1, Long.MAX_VALUE); - Assert.assertTrue(result2.contains(qh), result2); + assertTrue(result2.contains(qh), result2); while (!client.getQueryStatus(qh).finished()) { Thread.sleep(5000); } - Assert.assertTrue(qCom.getStatus(qh).contains("Status : SUCCESSFUL")); + assertTrue(qCom.getStatus(qh).contains("Status : SUCCESSFUL")); details = qCom.getDetails(qh); - Assert.assertTrue(details.contains("driverQuery")); + assertTrue(details.contains("driverQuery")); + + result = qCom.getQueryResults(qh, null); + assertTrue(result.contains("1\tfirst")); + + downloadResult(qCom, qh, result); + // re-download should also succeed + downloadResult(qCom, qh, result); - result = qCom.getQueryResults(qh); - Assert.assertTrue(result.contains("1\tfirst")); // Kill query is not tested as there is no deterministic way of killing a query result = qCom.getAllQueries("SUCCESSFUL", "", "all", -1, Long.MAX_VALUE); - Assert.assertTrue(result.contains(qh), result); + assertTrue(result.contains(qh), result); result = qCom.getAllQueries("FAILED", "", "all", -1, Long.MAX_VALUE); if (!result.contains("No queries")) { // Make sure valid query handles are returned String[] handles = StringUtils.split(result, "\n"); for (String handle : handles) { - QueryHandle.fromString(handle.trim()); + if (!handle.contains("Total number of queries")) { + QueryHandle.fromString(handle.trim()); + } } } String queryName = client.getLensStatement(new QueryHandle(UUID.fromString(qh))).getQuery().getQueryName(); - Assert.assertTrue("testQuery1".equalsIgnoreCase(queryName), queryName); + assertTrue("testQuery1".equalsIgnoreCase(queryName), queryName); result = qCom.getAllQueries("", "", "", submitTime, System.currentTimeMillis()); - Assert.assertTrue(result.contains(qh), result); + assertTrue(result.contains(qh), result); result = qCom.getAllQueries("", "fooBar", "all", submitTime, System.currentTimeMillis()); - Assert.assertTrue(result.contains("No queries"), result); + assertTrue(result.contains("No queries"), result); result = qCom.getAllQueries("SUCCESSFUL", "", "all", submitTime, System.currentTimeMillis()); - Assert.assertTrue(result.contains(qh)); + assertTrue(result.contains(qh)); result = qCom.getAllQueries("SUCCESSFUL", "", "all", submitTime - 5000, submitTime - 1); // should not give query since its not in the range - Assert.assertFalse(result.contains(qh)); + assertFalse(result.contains(qh)); try { // Should fail with bad request since fromDate > toDate result = qCom.getAllQueries("SUCCESSFUL", "", "all", submitTime + 5000, submitTime); - Assert.fail("Call should have failed with BadRequestException, instead got " + result); + fail("Call should have failed with BadRequestException, instead got " + result); } catch (BadRequestException exc) { // pass } @@ -284,6 +300,14 @@ public class TestLensQueryCommands extends LensCliApplicationTest { System.out.println("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"); } + private void downloadResult(LensQueryCommands qCom, String qh, String expected) throws IOException { + assertTrue(qCom.getQueryResults(qh, resDir.getAbsolutePath()).contains("Saved")); + assertEquals(readFile(resDir.getAbsolutePath() + File.separator + qh + ".csv").trim(), expected.trim()); + } + private String readFile(String path) throws FileNotFoundException { + return new Scanner(new File(path)).useDelimiter("\\Z").next(); + } + /** * Sets the up. * @@ -313,7 +337,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { timePart.getPartSpecElement().add(partElement); xp.setTimePartitionSpec(timePart); APIResult result = client.addPartitionToDim("dim_table", "local", xp); - Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); + assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); } /** @@ -324,7 +348,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { private void testExecuteSyncQuery(LensQueryCommands qCom) { String sql = "cube select id,name from test_dim"; String result = qCom.executeQuery(sql, false, "testQuery2"); - Assert.assertTrue(result.contains("1\tfirst"), result); + assertTrue(result.contains("1\tfirst"), result); } /** @@ -340,11 +364,11 @@ public class TestLensQueryCommands extends LensCliApplicationTest { try { String result = qCom.executeQuery(query, false, "testQuery3"); System.out.println("@@ RESULT " + result); - Assert.assertNotNull(result); - Assert.assertFalse(result.contains("Failed to get resultset")); + assertNotNull(result); + assertFalse(result.contains("Failed to get resultset")); } catch (Exception exc) { exc.printStackTrace(); - Assert.fail("Exception not expected: " + exc.getMessage()); + fail("Exception not expected: " + exc.getMessage()); } System.out.println("@@END_PERSISTENT_RESULT_TEST-------------"); } @@ -364,17 +388,17 @@ public class TestLensQueryCommands extends LensCliApplicationTest { while (!client.getQueryStatus(qh).finished()) { Thread.sleep(5000); } - Assert.assertTrue(qCom.getStatus(qh).contains("Status : SUCCESSFUL")); + assertTrue(qCom.getStatus(qh).contains("Status : SUCCESSFUL")); - String result = qCom.getQueryResults(qh); + String result = qCom.getQueryResults(qh, null); System.out.println("@@ RESULT " + result); - Assert.assertNotNull(result); + assertNotNull(result); // This is to check for positive processing time - Assert.assertFalse(result.contains("(-")); + assertFalse(result.contains("(-")); } catch (Exception exc) { exc.printStackTrace(); - Assert.fail("Exception not expected: " + exc.getMessage()); + fail("Exception not expected: " + exc.getMessage()); } System.out.println("@@END_FINISHED_PURGED_RESULT_TEST-------------"); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/test/java/org/apache/lens/cli/doc/TestGenerateCLIUserDoc.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/doc/TestGenerateCLIUserDoc.java b/lens-cli/src/test/java/org/apache/lens/cli/doc/TestGenerateCLIUserDoc.java index e5fe067..1910be9 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/doc/TestGenerateCLIUserDoc.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/doc/TestGenerateCLIUserDoc.java @@ -21,10 +21,7 @@ package org.apache.lens.cli.doc; import java.io.*; import java.lang.annotation.Annotation; import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashSet; -import java.util.TreeSet; +import java.util.*; import org.apache.lens.cli.commands.*; import org.apache.lens.cli.commands.annotations.UserDocumentation; @@ -48,7 +45,7 @@ public class TestGenerateCLIUserDoc { BufferedWriter bw = new BufferedWriter(new FileWriter(new File(APT_FILE))); StringBuilder sb = new StringBuilder(); sb.append(getCLIIntroduction()).append("\n\n\n"); - ArrayList<Class<? extends BaseLensCommand>> classes = Lists.newArrayList( + List<Class<? extends BaseLensCommand>> classes = Lists.newArrayList( LensConnectionCommands.class, LensDatabaseCommands.class, LensStorageCommands.class, http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-client/src/main/java/org/apache/lens/client/LensClient.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java index 9b0c935..cd8bf16 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java @@ -22,6 +22,8 @@ import java.util.Date; import java.util.HashMap; import java.util.List; +import javax.ws.rs.core.Response; + import org.apache.lens.api.APIResult; import org.apache.lens.api.metastore.*; import org.apache.lens.api.query.*; @@ -30,15 +32,18 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.google.common.collect.Maps; +import lombok.Getter; public class LensClient { private static final Log LOG = LogFactory.getLog(LensClient.class); private static final String DEFAULT_PASSWORD = ""; private final LensClientConfig conf; + @Getter private final LensMetadataClient mc; private String password; - private LensConnection conn; + @Getter + private LensConnection connection; private final HashMap<QueryHandle, LensStatement> statementMap = Maps.newHashMap(); private final LensStatement statement; @@ -63,8 +68,8 @@ public class LensClient { this.conf.set(LensClientConfig.SESSION_CLUSTER_USER, System.getProperty("user.name")); } connectToLensServer(); - mc = new LensMetadataClient(conn); - statement = new LensStatement(conn); + mc = new LensMetadataClient(connection); + statement = new LensStatement(connection); } public LensClient(Credentials cred) { @@ -80,10 +85,6 @@ public class LensClient { return query.getQueryHandle(); } - public LensConnection getConnection() { - return conn; - } - public Date getLatestDateOfCube(String cubeName, String timePartition) { return mc.getLatestDateOfCube(cubeName, timePartition); } @@ -150,16 +151,24 @@ public class LensClient { return getResultsFromHandle(q); } + public Response getHttpResults() { + return statement.getHttpResultSet(); + } + + public Response getHttpResults(QueryHandle q) { + return statement.getHttpResultSet(statement.getQuery(q)); + } + public LensStatement getLensStatement(QueryHandle query) { return this.statementMap.get(query); } public QueryStatus getQueryStatus(QueryHandle query) { - return new LensStatement(conn).getQuery(query).getStatus(); + return new LensStatement(connection).getQuery(query).getStatus(); } public LensQuery getQueryDetails(QueryHandle handle) { - return new LensStatement(conn).getQuery(handle); + return new LensStatement(connection).getQuery(handle); } public QueryStatus getQueryStatus(String q) { @@ -171,7 +180,7 @@ public class LensClient { } public QueryPlan getQueryPlan(String q) { - return new LensStatement(conn).explainQuery(q); + return new LensStatement(connection).explainQuery(q); } public boolean killQuery(QueryHandle q) { @@ -190,15 +199,15 @@ public class LensClient { } public List<QueryHandle> getQueries(String state, String queryName, String user, long fromDate, long toDate) { - return new LensStatement(conn).getAllQueries(state, queryName, user, fromDate, toDate); + return new LensStatement(connection).getAllQueries(state, queryName, user, fromDate, toDate); } private void connectToLensServer() { LOG.debug("Connecting to lens server " + new LensConnectionParams(conf)); - conn = new LensConnection(new LensConnectionParams(conf)); - conn.open(password); - LOG.debug("Successfully connected to server " + conn); + connection = new LensConnection(new LensConnectionParams(conf)); + connection.open(password); + LOG.debug("Successfully connected to server " + connection); } @@ -269,36 +278,36 @@ public class LensClient { } public APIResult setConnectionParam(String key, String val) { - return this.conn.setConnectionParams(key, val); + return this.connection.setConnectionParams(key, val); } public List<String> getConnectionParam() { - return this.conn.getConnectionParams(); + return this.connection.getConnectionParams(); } public List<String> getConnectionParam(String key) { - return this.conn.getConnectionParams(key); + return this.connection.getConnectionParams(key); } public APIResult closeConnection() { LOG.debug("Closing lens connection: " + new LensConnectionParams(conf)); - return this.conn.close(); + return this.connection.close(); } public APIResult addJarResource(String path) { - return this.conn.addResourceToConnection("jar", path); + return this.connection.addResourceToConnection("jar", path); } public APIResult removeJarResource(String path) { - return this.conn.removeResourceFromConnection("jar", path); + return this.connection.removeResourceFromConnection("jar", path); } public APIResult addFileResource(String path) { - return this.conn.addResourceToConnection("file", path); + return this.connection.addResourceToConnection("file", path); } public APIResult removeFileResource(String path) { - return this.conn.removeResourceFromConnection("file", path); + return this.connection.removeResourceFromConnection("file", path); } public APIResult createFactTable(String factSpec) { @@ -523,11 +532,11 @@ public class LensClient { } public boolean isConnectionOpen() { - return this.conn.isOpen(); + return this.connection.isOpen(); } public List<String> listResources(String type) { - return this.conn.listResourcesFromConnection(type); + return this.connection.listResourcesFromConnection(type); } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-client/src/main/java/org/apache/lens/client/LensStatement.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java index 5ce3a82..4472a80 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java @@ -26,6 +26,7 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.GenericType; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.lens.api.APIResult; import org.apache.lens.api.query.*; @@ -268,7 +269,7 @@ public class LensStatement { } Client client = ClientBuilder.newBuilder().register(MultiPartFeature.class).register(LensJAXBContextResolver.class) - .build(); + .build(); FormDataMultiPart mp = new FormDataMultiPart(); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), connection @@ -281,7 +282,7 @@ public class LensStatement { WebTarget target = getQueryWebTarget(client); return target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensResponse<QueryHandle, NoErrorPayload>>(){}).getData(); + new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData(); } /** @@ -332,7 +333,7 @@ public class LensStatement { WebTarget target = getQueryWebTarget(client); QueryPlan handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensResponse<QueryPlan, NoErrorPayload>>() {}).getData(); + new GenericType<LensResponse<QueryPlan, NoErrorPayload>>() {}).getData(); return handle; } @@ -405,6 +406,10 @@ public class LensStatement { return this.getResultSet(this.query); } + public Response getHttpResultSet() { + return this.getHttpResultSet(this.query); + } + /** * Gets the result set. * @@ -427,6 +432,27 @@ public class LensStatement { } /** + * Gets the http result set. + * + * @param query the query + * @return the http result set + */ + public Response getHttpResultSet(LensQuery query) { + if (query.getStatus().getStatus() != QueryStatus.Status.SUCCESSFUL) { + throw new IllegalArgumentException("Result set metadata " + "can be only queries for successful queries"); + } + Client client = ClientBuilder.newClient(); + + try { + WebTarget target = getQueryWebTarget(client); + return target.path(query.getQueryHandle().toString()).path("httpresultset") + .queryParam("sessionid", connection.getSessionHandle()).request().get(); + } catch (Exception e) { + throw new IllegalStateException("Failed to get resultset, cause:" + e.getMessage()); + } + } + + /** * Kill. * * @return true, if successful http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index 43985ba..8bec5e0 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -801,12 +801,10 @@ public class HiveDriver implements LensDriver { OperationHandle op = getHiveHandle(context.getQueryHandle()); LOG.info("Creating result set for hiveHandle:" + op); try { - if (op.hasResultSet() || context.isDriverPersistent()) { - if (context.isDriverPersistent()) { - return new HivePersistentResultSet(new Path(context.getHdfsoutPath()), op, getClient()); - } else { - return new HiveInMemoryResultSet(op, getClient(), closeAfterFetch); - } + if (context.isDriverPersistent()) { + return new HivePersistentResultSet(new Path(context.getHdfsoutPath()), op, getClient()); + } else if (op.hasResultSet()) { + return new HiveInMemoryResultSet(op, getClient(), closeAfterFetch); } else { // queries that do not have result return null; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java index a953ec6..4d631ec 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java @@ -61,6 +61,7 @@ public class HiveInMemoryResultSet extends InMemoryResultSet { /** The num columns. */ int numColumns; + private FetchOrientation orientation; /** * Instantiates a new hive in memory result set. @@ -77,6 +78,7 @@ public class HiveInMemoryResultSet extends InMemoryResultSet { this.closeAfterFecth = closeAfterFecth; this.metadata = client.getResultSetMetadata(opHandle); this.numColumns = metadata.getColumnDescriptors().size(); + this.seekToStart(); } /* @@ -98,16 +100,23 @@ public class HiveInMemoryResultSet extends InMemoryResultSet { return hrsMeta; } + @Override + public boolean seekToStart() { + orientation = FetchOrientation.FETCH_FIRST; + return true; + } + /* - * (non-Javadoc) - * - * @see org.apache.lens.server.api.driver.InMemoryResultSet#hasNext() - */ + * (non-Javadoc) + * + * @see org.apache.lens.server.api.driver.InMemoryResultSet#hasNext() + */ @Override public boolean hasNext() throws LensException { if (fetchedRowsItr == null || !fetchedRowsItr.hasNext()) { try { - rowSet = client.fetchResults(opHandle, FetchOrientation.FETCH_NEXT, fetchSize); + rowSet = client.fetchResults(opHandle, orientation, fetchSize); + orientation = FetchOrientation.FETCH_NEXT; noMoreResults = rowSet.numRows() == 0; if (noMoreResults) { if (closeAfterFecth) { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java index 36a48b9..bbcc3f1 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java @@ -76,10 +76,11 @@ public class JDBCResultSet extends InMemoryResultSet { * @param resultSet the result set * @param closeAfterFetch the close after fetch */ - public JDBCResultSet(QueryResult queryResult, ResultSet resultSet, boolean closeAfterFetch) { + public JDBCResultSet(QueryResult queryResult, ResultSet resultSet, boolean closeAfterFetch) throws LensException { this.queryResult = queryResult; this.resultSet = resultSet; this.closeAfterFetch = closeAfterFetch; + seekToStart(); } private ResultSetMetaData getRsMetadata() throws LensException { @@ -294,11 +295,23 @@ public class JDBCResultSet extends InMemoryResultSet { } } + @Override + public boolean seekToStart() throws LensException { + try { + if (!resultSet.isBeforeFirst()) { + resultSet.beforeFirst(); + } + return true; + } catch (SQLException e) { + throw new LensException(e); + } + } + /* - * (non-Javadoc) - * - * @see org.apache.lens.server.api.driver.InMemoryResultSet#hasNext() - */ + * (non-Javadoc) + * + * @see org.apache.lens.server.api.driver.InMemoryResultSet#hasNext() + */ @Override public synchronized boolean hasNext() throws LensException { try { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-server-api/src/main/java/org/apache/lens/server/api/driver/InMemoryResultSet.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/InMemoryResultSet.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/InMemoryResultSet.java index f11d883..3b76126 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/InMemoryResultSet.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/InMemoryResultSet.java @@ -31,6 +31,7 @@ import org.apache.lens.server.api.error.LensException; */ public abstract class InMemoryResultSet extends LensResultSet { + public abstract boolean seekToStart() throws LensException; /** * Whether there is another result row available. * http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java index 0f4e990..30c1a71 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java @@ -308,6 +308,11 @@ public class MockDriver implements LensDriver { } @Override + public boolean seekToStart() throws LensException { + return false; + } + + @Override public boolean hasNext() throws LensException { // TODO Auto-generated method stub return false; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 343cd01..4cf1fa9 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -1288,6 +1288,9 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu } } } + if (resultSets.get(queryHandle) instanceof InMemoryResultSet) { + ((InMemoryResultSet) resultSets.get(queryHandle)).seekToStart(); + } return resultSets.get(queryHandle); } } @@ -2288,7 +2291,12 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu .type(MediaType.APPLICATION_OCTET_STREAM).build(); } } else { - throw new NotFoundException("Http result not available for query:" + queryHandle.toString()); + String entity = ""; + if (result instanceof InMemoryResultSet || result instanceof PersistentResultSet) { + entity = "Result is available in driver's " + + (result instanceof InMemoryResultSet ? "memory" : "persistence") + "."; + } + return Response.status(Response.Status.NOT_FOUND).entity(entity).build(); } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/src/site/apt/user/cli.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt index 8efb58b..2016bfd 100644 --- a/src/site/apt/user/cli.apt +++ b/src/site/apt/user/cli.apt @@ -324,7 +324,7 @@ User CLI Commands *--+--+ |query list [--state \<query-status\>] [--name \<query-name\>] [--user \<user-who-submitted-query\>] [--fromDate \<submission-time-is-after\>] [--toDate \<submission-time-is-before\>]|Get all queries. Various filter options can be provided(optionally), as can be seen from the command syntax| *--+--+ -|query results [--query_handle] \<query_handle\>|get results of async query with query handle <<<query_handle>>>| +|query results [--query_handle] \<query_handle\> [--save_location \<save_location\>]|get results of async query with query handle <<<query_handle>>>. Can optionally save the results to a file by providing <<<save_location>>>| *--+--+ |query status [--query_handle] \<query_handle\>|Fetch status of executed query having query handle <<<query_handle>>>| *--+--+
