Repository: lens Updated Branches: refs/heads/master d457dd0de -> 669e87272
http://git-wip-us.apache.org/repos/asf/lens/blob/669e8727/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java index 0945598..e266ef2 100644 --- a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java +++ b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java @@ -22,9 +22,13 @@ package org.apache.lens.server.common; import static org.apache.lens.server.common.FormDataMultiPartFactory.createFormDataMultiPartForFact; import static org.apache.lens.server.common.FormDataMultiPartFactory.createFormDataMultiPartForSession; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + import javax.ws.rs.client.Entity; import javax.ws.rs.client.Invocation; import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.GenericType; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -36,7 +40,9 @@ import org.apache.lens.api.metastore.XCube; import org.apache.lens.api.metastore.XFactTable; import org.apache.lens.api.query.LensQuery; import org.apache.lens.api.query.QueryHandle; +import org.apache.lens.api.query.QueryResult; import org.apache.lens.api.query.QueryStatus; +import org.apache.lens.api.result.LensAPIResult; import org.glassfish.jersey.media.multipart.FormDataMultiPart; @@ -53,49 +59,72 @@ public class RestAPITestUtil { public static LensSessionHandle openFooBarSession(final WebTarget target) { return openSession(target, "foo", "bar"); } + public static LensSessionHandle openSession(final WebTarget target, final String userName, final String passwd) { return openSession(target, userName, passwd, new LensConf()); } public static LensSessionHandle openSession(final WebTarget target, final String userName, final String passwd, - final LensConf conf) { + final LensConf conf) { final FormDataMultiPart mp = createFormDataMultiPartForSession(Optional.<LensSessionHandle>absent(), - Optional.of(userName), Optional.of(passwd), Optional.of(conf)); + Optional.of(userName), Optional.of(passwd), Optional.of(conf)); return target.path("session").request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - LensSessionHandle.class); + LensSessionHandle.class); } public static Response estimate(final WebTarget target, final Optional<LensSessionHandle> sessionId, - final Optional<String> query) { + final Optional<String> query) { - return postQuery(target, sessionId, query, Optional.of("estimate")); + return postQuery(target, sessionId, query, Optional.of("estimate"), Optional.<LensConf>absent()); } public static Response execute(final WebTarget target, final Optional<LensSessionHandle> sessionId, final Optional<String> query) { - return postQuery(target, sessionId, query, Optional.of("execute")); + return execute(target, sessionId, query, Optional.<LensConf>absent()); } - public static Response postQuery(final WebTarget target, final Optional<LensSessionHandle> sessionId, - final Optional<String> query, final Optional<String> operation) { + public static Response execute(final WebTarget target, final Optional<LensSessionHandle> sessionId, + final Optional<String> query, final Optional<LensConf> lensConf) { + return postQuery(target, sessionId, query, Optional.of("execute"), lensConf); + } + + public static <T> T executeAndGetHandle(final WebTarget target, final Optional<LensSessionHandle> sessionId, + final Optional<String> query, final Optional<LensConf> lensConf) { + Response resp = postQuery(target, sessionId, query, Optional.of("execute"), lensConf); + assertEquals(resp.getStatus(), Response.Status.OK.getStatusCode()); + T handle = resp.readEntity(new GenericType<LensAPIResult<T>>() {}).getData(); + assertNotNull(handle); + return handle; + } - return postQuery(target, sessionId, query, operation, new LensConf()); + public static Response postQuery(final WebTarget target, final Optional<LensSessionHandle> sessionId, + final Optional<String> query, final Optional<String> operation) { + return postQuery(target, sessionId, query, operation, Optional.<LensConf>absent()); } public static Response postQuery(final WebTarget target, final Optional<LensSessionHandle> sessionId, - final Optional<String> query, final Optional<String> operation, final LensConf conf) { + final Optional<String> query, final Optional<String> operation, Optional<LensConf> lensConfOptional) { FormDataMultiPart mp = FormDataMultiPartFactory - .createFormDataMultiPartForQuery(sessionId, query, operation, conf); + .createFormDataMultiPartForQuery(sessionId, query, operation, lensConfOptional.or(new LensConf())); return target.path("queryapi/queries").request(MediaType.APPLICATION_XML).post( - Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); + Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); } - public static void closeSessionFailFast(final WebTarget target, final LensSessionHandle sessionId) { + public static LensQuery executeAndWaitForQueryToFinish(WebTarget target, LensSessionHandle lensSessionId, + String query, Optional<LensConf> conf, Optional<QueryStatus.Status> statusOptional) throws InterruptedException { + QueryHandle handle = executeAndGetHandle(target, Optional.of(lensSessionId), Optional.of(query), conf); + if (statusOptional.isPresent()) { + return waitForQueryToFinish(target, lensSessionId, handle, statusOptional.get()); + } else { + return waitForQueryToFinish(target, lensSessionId, handle); + } + } + public static void closeSessionFailFast(final WebTarget target, final LensSessionHandle sessionId) { APIResult result = closeSession(target, sessionId); checkResponse(result); } @@ -114,7 +143,7 @@ public class RestAPITestUtil { public static APIResult createCube(final WebTarget target, final LensSessionHandle sessionId, final XCube cube) { return target.path("metastore").path("cubes").queryParam("sessionid", sessionId).request(MediaType.APPLICATION_XML) - .post(Entity.xml(cubeObjectFactory.createXCube(cube)), APIResult.class); + .post(Entity.xml(cubeObjectFactory.createXCube(cube)), APIResult.class); } public static void createCubeFailFast(final WebTarget target, final LensSessionHandle sessionId, final XCube cube) { @@ -123,64 +152,64 @@ public class RestAPITestUtil { } public static APIResult createFact(final WebTarget target, final LensSessionHandle sessionId, - final XFactTable factTable) { + final XFactTable factTable) { FormDataMultiPart mp = createFormDataMultiPartForFact(sessionId, factTable); return target.path("metastore").path("facts").queryParam("sessionid", sessionId).request(MediaType.APPLICATION_XML) - .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class); + .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class); } public static void createFactFailFast(final WebTarget target, final LensSessionHandle sessionId, - final XFactTable factTable) { + final XFactTable factTable) { APIResult result = createFact(target, sessionId, factTable); checkResponse(result); } public static APIResult setCurrentDatabase(final WebTarget target, final LensSessionHandle sessionId, - final String dbName) { + final String dbName) { WebTarget dbTarget = target.path("metastore").path("databases/current"); return dbTarget.queryParam("sessionid", sessionId).request(MediaType.APPLICATION_XML) - .put(Entity.xml(dbName), + .put(Entity.xml(dbName), APIResult.class); } public static void setCurrentDatabaseFailFast(final WebTarget target, final LensSessionHandle sessionId, - final String dbName) { + final String dbName) { APIResult result = setCurrentDatabase(target, sessionId, dbName); checkResponse(result); } public static APIResult createDatabase(final WebTarget target, final LensSessionHandle sessionId, - final String dbName) { + final String dbName) { WebTarget dbTarget = target.path("metastore").path("databases"); return dbTarget.queryParam("sessionid", sessionId).request(MediaType.APPLICATION_XML) - .post(Entity.xml(dbName), APIResult.class); + .post(Entity.xml(dbName), APIResult.class); } public static void createDatabaseFailFast(final WebTarget target, final LensSessionHandle sessionId, - final String dbName) { + final String dbName) { APIResult result = createDatabase(target, sessionId, dbName); checkResponse(result); } public static void createAndSetCurrentDbFailFast(final WebTarget target, final LensSessionHandle sessionId, - final String dbName) { + final String dbName) { createDatabaseFailFast(target, sessionId, dbName); setCurrentDatabaseFailFast(target, sessionId, dbName); } public static APIResult dropDatabaseFailFast(final WebTarget target, final LensSessionHandle sessionId, - String dbName) { + String dbName) { WebTarget dbTarget = target.path("metastore").path("databases").path(dbName); return dbTarget.queryParam("cascade", "true") - .queryParam("sessionid", sessionId).request(MediaType.APPLICATION_XML).delete(APIResult.class); + .queryParam("sessionid", sessionId).request(MediaType.APPLICATION_XML).delete(APIResult.class); } private static void checkResponse(final APIResult result) { @@ -191,22 +220,45 @@ public class RestAPITestUtil { public static LensQuery waitForQueryToFinish(final WebTarget target, final LensSessionHandle lensSessionHandle, final QueryHandle handle) throws InterruptedException { - - LensQuery lensQuery = getLensQuery(target, lensSessionHandle, handle); - - QueryStatus stat = lensQuery.getStatus(); - while (!stat.finished()) { - lensQuery = getLensQuery(target, lensSessionHandle, handle); - stat = lensQuery.getStatus(); + LensQuery ctx = getLensQuery(target, lensSessionHandle, handle); + while (!ctx.getStatus().finished()) { + ctx = getLensQuery(target, lensSessionHandle, handle); Thread.sleep(1000); } + return ctx; + } + + public static LensQuery waitForQueryToFinish(final WebTarget target, final LensSessionHandle lensSessionHandle, + final QueryHandle handle, QueryStatus.Status status) throws InterruptedException { + LensQuery lensQuery = waitForQueryToFinish(target, lensSessionHandle, handle); + assertEquals(lensQuery.getStatus().getStatus(), status); return lensQuery; } public static LensQuery getLensQuery(final WebTarget target, final LensSessionHandle lensSessionHandle, final QueryHandle handle) { - return target.path("queryapi/queries").path(handle.toString()).queryParam("sessionid", lensSessionHandle).request() - .get(LensQuery.class); + .get(LensQuery.class); + } + + public static QueryResult getLensQueryResult(final WebTarget target, final LensSessionHandle lensSessionHandle, + final QueryHandle handle) { + return target.path("queryapi/queries").path(handle.toString()).path("resultset") + .queryParam("sessionid", lensSessionHandle).request().get(QueryResult.class); + } + + public static Response getLensQueryHttpResult(final WebTarget target, final LensSessionHandle lensSessionHandle, + final QueryHandle handle) { + return target.path("queryapi/queries").path(handle.toString()).path("httpresultset") + .queryParam("sessionid", lensSessionHandle).request().get(Response.class); + } + + public static LensConf getLensConf(Object... args) { + assertEquals(args.length % 2, 0); + LensConf conf = new LensConf(); + for (int i = 0; i < args.length; i += 2) { + conf.addProperty(args[i], args[i + 1]); + } + return conf; } } http://git-wip-us.apache.org/repos/asf/lens/blob/669e8727/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java index b8012be..1dab35e 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java @@ -92,7 +92,6 @@ public class TestEventService { * the genericEvent event occurs, that object's appropriate * method is invoked. * - * @see GenericEventEvent */ class GenericEventListener extends AsyncEventListener<LensEvent> { @@ -119,7 +118,6 @@ public class TestEventService { * the mockFailed event occurs, that object's appropriate * method is invoked. * - * @see MockFailedEvent */ class MockFailedListener implements LensEventListener<QueryFailed> { @@ -146,7 +144,6 @@ public class TestEventService { * the mockEnded event occurs, that object's appropriate * method is invoked. * - * @see MockEndedEvent */ class MockEndedListener implements LensEventListener<QueryEnded> { @@ -290,7 +287,7 @@ public class TestEventService { */ @Test public void testAddListener() { - int listenersBefore = ((EventServiceImpl) service).getEventListeners().keySet().size(); + int listenersBefore = service.getEventListeners().keySet().size(); genericEventListener = new GenericEventListener(); service.addListenerForType(genericEventListener, LensEvent.class); endedListener = new MockEndedListener(); @@ -417,8 +414,10 @@ public class TestEventService { QueryHandle query = new QueryHandle(UUID.randomUUID()); String user = "user"; long now = System.currentTimeMillis(); - QueryFailed failed = new QueryFailed(now, QueryStatus.Status.RUNNING, QueryStatus.Status.FAILED, query, user, null); - QuerySuccess success = new QuerySuccess(now, QueryStatus.Status.RUNNING, QueryStatus.Status.SUCCESSFUL, query); + QueryFailed failed + = new QueryFailed(null, now, QueryStatus.Status.RUNNING, QueryStatus.Status.FAILED, query, user, null); + QuerySuccess success + = new QuerySuccess(null, now, QueryStatus.Status.RUNNING, QueryStatus.Status.SUCCESSFUL, query); QueuePositionChange positionChange = new QueuePositionChange(now, 1, 0, query); try { http://git-wip-us.apache.org/repos/asf/lens/blob/669e8727/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java index 82ce205..3bb90ef 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java @@ -18,28 +18,25 @@ */ package org.apache.lens.server.query; -import static org.apache.lens.server.common.RestAPITestUtil.execute; -import static org.apache.lens.server.common.RestAPITestUtil.waitForQueryToFinish; +import static org.apache.lens.server.common.RestAPITestUtil.*; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Application; import javax.ws.rs.core.GenericType; -import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; -import org.apache.lens.api.query.LensQuery; +import org.apache.lens.api.query.InMemoryQueryResult; import org.apache.lens.api.query.QueryHandle; -import org.apache.lens.api.query.QueryStatus; +import org.apache.lens.api.query.QueryResult; import org.apache.lens.api.query.QueryStatus.Status; import org.apache.lens.api.result.LensAPIResult; import org.apache.lens.server.LensJerseyTest; @@ -52,16 +49,10 @@ import org.apache.lens.server.common.TestResourceFile; import org.apache.hadoop.hive.conf.HiveConf; import org.glassfish.jersey.client.ClientConfig; -import org.glassfish.jersey.media.multipart.FormDataBodyPart; -import org.glassfish.jersey.media.multipart.FormDataContentDisposition; -import org.glassfish.jersey.media.multipart.FormDataMultiPart; import org.glassfish.jersey.media.multipart.MultiPartFeature; import org.subethamail.wiser.Wiser; import org.subethamail.wiser.WiserMessage; -import org.testng.Assert; -import org.testng.annotations.AfterTest; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; +import org.testng.annotations.*; import com.google.common.base.Optional; @@ -93,7 +84,7 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest { wiser.setHostname("localhost"); wiser.setPort(25000); queryService = LensServices.get().getService(QueryExecutionService.NAME); - Map<String, String> sessionconf = new HashMap<String, String>(); + Map<String, String> sessionconf = new HashMap<>(); sessionconf.put("test.session.key", "svalue"); sessionconf.put(LensConfConstants.QUERY_MAIL_NOTIFY, "true"); sessionconf.put(LensConfConstants.QUERY_RESULT_EMAIL_CC, "foo1@localhost,foo2@localhost,foo3@localhost"); @@ -101,6 +92,12 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest { // automatically createTable(TEST_TABLE); loadData(TEST_TABLE, TestResourceFile.TEST_DATA2_FILE.getValue()); + wiser.start(); + } + + @BeforeMethod + public void clearWiserMessages() { + wiser.getMessages().clear(); } /* @@ -110,6 +107,7 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest { */ @AfterTest public void tearDown() throws Exception { + wiser.stop(); dropTable(TEST_TABLE); queryService.closeSession(lensSessionId); super.tearDown(); @@ -171,28 +169,29 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest { private QueryHandle launchAndWaitForQuery(LensConf conf, String query, Status expectedStatus) throws InterruptedException { - final WebTarget target = target().path("queryapi/queries"); - final FormDataMultiPart mp = new FormDataMultiPart(); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - MediaType.APPLICATION_XML_TYPE)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), query)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, - MediaType.APPLICATION_XML_TYPE)); - final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensAPIResult<QueryHandle>>(){}).getData(); + return executeAndWaitForQueryToFinish(target(), lensSessionId, query, Optional.of(conf), + Optional.of(expectedStatus)).getQueryHandle(); + } + + private WiserMessage getMessage() throws InterruptedException { + List<WiserMessage> messages = new ArrayList<>(); + for (int i = 0; i < NUM_ITERS; i++) { + messages = wiser.getMessages(); + if (messages.size() >= 4) { + break; + } + Thread.sleep(2000); + } + assertEquals(messages.size(), 4); + return messages.get(0); + } - Assert.assertNotNull(handle); - LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request() - .get(LensQuery.class); - QueryStatus stat = ctx.getStatus(); - while (!stat.finished()) { - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - stat = ctx.getStatus(); - Thread.sleep(1000); + private void assertKeywordsContains(Object o, Object... keywords) { + String string = o.toString(); + for (Object keyword : keywords) { + // Mail message converts \n to \r\n + assertTrue(string.contains(keyword.toString().replaceAll("\\n", "\r\n")), o + " doesn't contain " + keyword); } - Assert.assertEquals(ctx.getStatus().getStatus(), expectedStatus); - return handle; } /** @@ -201,91 +200,70 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest { * @throws InterruptedException the interrupted exception */ @Test - public void testEmailNotification() throws InterruptedException { - wiser.start(); - - LensConf conf = new LensConf(); - + public void testLaunchFailure() throws InterruptedException { // launch failure final Response response = execute(target(), Optional.of(lensSessionId), Optional.of("select fail from non_exist")); assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); QueryHandle handle = response.readEntity(new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); - LensQuery lensQuery = waitForQueryToFinish(target(), lensSessionId, handle); - - List<WiserMessage> messages = new ArrayList<WiserMessage>(); - for (int i = 0; i < NUM_ITERS; i++) { - messages = wiser.getMessages(); - if (messages.size() >= 4) { - break; - } - Thread.sleep(2000); - } - - Assert.assertEquals(messages.size(), 4); - Assert.assertTrue(messages.get(0).toString().contains(handle.toString())); - Assert.assertTrue(messages.get(0).toString().contains("Launching query failed")); - Assert.assertTrue(messages.get(0).toString().contains("Reason")); + assertKeywordsContains(getMessage(), handle, "Launching query failed", "Reason"); + } + @Test + public void testFormattingFailure() throws InterruptedException { // formatting failure - conf = new LensConf(); - conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); - conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); - conf.addProperty(LensConfConstants.QUERY_OUTPUT_SERDE, "NonexistentSerde.class"); - handle = launchAndWaitForQuery(conf, "select ID, IDSTR from " + TEST_TABLE, - QueryStatus.Status.FAILED); - - messages = new ArrayList<WiserMessage>(); - for (int i = 0; i < NUM_ITERS; i++) { - messages = wiser.getMessages(); - if (messages.size() >= 8) { - break; - } - Thread.sleep(2000); - } - - Assert.assertEquals(messages.size(), 8); - Assert.assertTrue(messages.get(4).toString().contains(handle.toString())); - Assert.assertTrue(messages.get(4).toString().contains("Result formatting failed!")); - Assert.assertTrue(messages.get(4).toString().contains("Reason")); + LensConf conf = getLensConf( + LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true", + LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false", + LensConfConstants.QUERY_OUTPUT_SERDE, "NonexistentSerde.class"); + QueryHandle handle = launchAndWaitForQuery(conf, "select ID, IDSTR from " + TEST_TABLE, Status.FAILED); + assertKeywordsContains(getMessage(), handle, "Result formatting failed!", "Reason"); + } + @Test + public void testExecutionFailure() throws InterruptedException { // execution failure - conf = new LensConf(); - conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true"); - conf.addProperty(HiveConf.ConfVars.COMPRESSRESULT.name(), "true"); - conf.addProperty("mapred.compress.map.output", "true"); - conf.addProperty("mapred.map.output.compression.codec", "nonexisting"); - handle = launchAndWaitForQuery(conf, "select count(ID) from " + TEST_TABLE, QueryStatus.Status.FAILED); - messages = new ArrayList<WiserMessage>(); - for (int i = 0; i < NUM_ITERS; i++) { - messages = wiser.getMessages(); - if (messages.size() >= 8) { - break; - } - Thread.sleep(2000); - } + LensConf conf = getLensConf( + LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true", + HiveConf.ConfVars.COMPRESSRESULT.name(), "true", + "mapred.compress.map.output", "true", + "mapred.map.output.compression.codec", "nonexisting"); + QueryHandle handle = launchAndWaitForQuery(conf, "select count(ID) from " + TEST_TABLE, Status.FAILED); + assertKeywordsContains(getMessage(), handle, "Query execution failed!", "Reason"); + } - Assert.assertEquals(messages.size(), 12); - Assert.assertTrue(messages.get(8).toString().contains(handle.toString())); - Assert.assertTrue(messages.get(8).toString().contains("Query execution failed!")); - Assert.assertTrue(messages.get(8).toString().contains("Reason")); + @DataProvider(name = "success-tests") + public Object[][] persistenceConfigDataProvider() { + return new Object[][]{ + {false, false, }, + {true, false, }, + {false, true, }, + {true, true, }, + }; + } + @Test(dataProvider = "success-tests") + public void testSuccessfulQuery(Boolean lensPersistence, Boolean driverPersistence) throws InterruptedException { // successful query - conf = new LensConf(); - conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); - conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true"); - handle = launchAndWaitForQuery(conf, "select ID, IDSTR from " + TEST_TABLE, QueryStatus.Status.SUCCESSFUL); - messages = new ArrayList<WiserMessage>(); - for (int i = 0; i < NUM_ITERS; i++) { - messages = wiser.getMessages(); - if (messages.size() >= 16) { - break; - } - Thread.sleep(2000); + LensConf conf = getLensConf( + LensConfConstants.QUERY_PERSISTENT_RESULT_SET, lensPersistence, + LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, driverPersistence); + QueryHandle handle = launchAndWaitForQuery(conf, "select ID, IDSTR from " + TEST_TABLE, Status.SUCCESSFUL); + String expectedKeywords; + if (lensPersistence || driverPersistence) { + QueryResult result = getLensQueryResult(target(), lensSessionId, handle); + expectedKeywords = result.toPrettyString(); + } else { + expectedKeywords = InMemoryQueryResult.DECLARATION; + } + WiserMessage message = getMessage(); + assertKeywordsContains(message, handle, "Query SUCCESSFUL", expectedKeywords); + if (lensPersistence) { + assertKeywordsContains(message, new String[]{"Downloadable from", "httpresultset"}); + assertEquals(getLensQueryHttpResult(target(), lensSessionId, handle).getStatus(), + Response.Status.OK.getStatusCode()); + } else { + assertEquals(getLensQueryHttpResult(target(), lensSessionId, handle).getStatus(), + Response.Status.NOT_FOUND.getStatusCode()); } - Assert.assertEquals(messages.size(), 16); - Assert.assertTrue(messages.get(12).toString().contains(handle.toString())); - Assert.assertTrue(messages.get(12).toString().contains("Query SUCCESSFUL")); - Assert.assertTrue(messages.get(12).toString().contains("Result available at")); - wiser.stop(); } } http://git-wip-us.apache.org/repos/asf/lens/blob/669e8727/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index fa8d6ee..ea75ffb 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -18,10 +18,9 @@ */ package org.apache.lens.server.query; -import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.*; -import static org.apache.lens.server.common.RestAPITestUtil.execute; -import static org.apache.lens.server.common.RestAPITestUtil.waitForQueryToFinish; +import static org.apache.lens.server.common.RestAPITestUtil.*; import static org.testng.Assert.*; @@ -31,7 +30,10 @@ import java.util.*; import javax.ws.rs.NotFoundException; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.*; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.GenericType; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.lens.api.APIResult; import org.apache.lens.api.LensConf; @@ -39,7 +41,9 @@ import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.jaxb.LensJAXBContextResolver; import org.apache.lens.api.query.*; import org.apache.lens.api.query.QueryStatus.Status; -import org.apache.lens.api.result.*; +import org.apache.lens.api.result.LensAPIResult; +import org.apache.lens.api.result.LensErrorTO; +import org.apache.lens.api.result.QueryCostTO; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.driver.hive.HiveDriver; import org.apache.lens.server.LensJerseyTest; @@ -62,17 +66,24 @@ import org.apache.lens.server.session.HiveSessionService; import org.apache.lens.server.session.LensSessionImpl; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.glassfish.jersey.client.ClientConfig; -import org.glassfish.jersey.media.multipart.*; +import org.glassfish.jersey.media.multipart.FormDataBodyPart; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.glassfish.jersey.media.multipart.MultiPartFeature; import org.glassfish.jersey.test.TestProperties; -import org.testng.annotations.*; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; import com.codahale.metrics.MetricRegistry; - import com.google.common.base.Optional; import lombok.extern.slf4j.Slf4j; @@ -102,6 +113,7 @@ public class TestQueryService extends LensJerseyTest { return classes; } } + /* * (non-Javadoc) * @@ -218,14 +230,14 @@ public class TestQueryService extends LensJerseyTest { LensConf conf = new LensConf(); final FormDataMultiPart mp = new FormDataMultiPart(); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - MediaType.APPLICATION_XML_TYPE)); + MediaType.APPLICATION_XML_TYPE)); mp.bodyPart( - new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select ID from non_exist_table")); + new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select ID from non_exist_table")); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, - MediaType.APPLICATION_XML_TYPE)); + MediaType.APPLICATION_XML_TYPE)); final Response response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); - assertEquals(response.getStatus(), Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()); + assertEquals(response.getStatus(), INTERNAL_SERVER_ERROR.getStatusCode()); } /** @@ -235,19 +247,13 @@ public class TestQueryService extends LensJerseyTest { */ @Test public void testLaunchFail() throws InterruptedException { - - final Response response = execute(target(), Optional.of(lensSessionId), Optional.of("select fail from non_exist")); - assertEquals(response.getStatus(), Response.Status.OK.getStatusCode()); - - QueryHandle handle = response.readEntity(new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); - LensQuery lensQuery = waitForQueryToFinish(target(), lensSessionId, handle); - + LensQuery lensQuery = executeAndWaitForQueryToFinish(target(), lensSessionId, "select fail from non_exist", + Optional.<LensConf>absent(), Optional.of(Status.FAILED)); assertTrue(lensQuery.getSubmissionTime() > 0); assertEquals(lensQuery.getLaunchTime(), 0); assertEquals(lensQuery.getDriverStartTime(), 0); assertEquals(lensQuery.getDriverFinishTime(), 0); assertTrue(lensQuery.getFinishTime() > 0); - assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.FAILED); } // test with execute async post, get all queries, get query context, @@ -262,22 +268,13 @@ public class TestQueryService extends LensJerseyTest { public void testQueriesAPI() throws InterruptedException { // test post execute op final WebTarget target = target().path("queryapi/queries"); - final FormDataMultiPart mp = new FormDataMultiPart(); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - MediaType.APPLICATION_XML_TYPE)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select ID from " + TEST_TABLE)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(), - MediaType.APPLICATION_XML_TYPE)); long queuedQueries = metricsSvc.getQueuedQueries(); long runningQueries = metricsSvc.getRunningQueries(); long finishedQueries = metricsSvc.getFinishedQueries(); - final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensAPIResult<QueryHandle>>() { - }).getData(); - assertNotNull(handle); + QueryHandle handle = executeAndGetHandle(target(), Optional.of(lensSessionId), Optional.of("select ID from " + + TEST_TABLE), Optional.<LensConf>absent()); // Get all queries // XML @@ -285,7 +282,7 @@ public class TestQueryService extends LensJerseyTest { .get(new GenericType<List<QueryHandle>>() {}); assertTrue(allQueriesXML.size() >= 1); - List<QueryHandle> allQueries = (List<QueryHandle>) target.queryParam("sessionid", lensSessionId).request() + List<QueryHandle> allQueries = target.queryParam("sessionid", lensSessionId).request() .get(new GenericType<List<QueryHandle>>() {}); assertTrue(allQueries.size() >= 1); assertTrue(allQueries.contains(handle)); @@ -375,7 +372,7 @@ public class TestQueryService extends LensJerseyTest { MediaType.APPLICATION_XML_TYPE)); final QueryPlan plan2 = ptarget.request().post(Entity.entity(mp2, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensAPIResult<QueryPlan>>() {}).getData(); + new GenericType<LensAPIResult<QueryPlan>>() {}).getData(); assertEquals(plan2.getTablesQueried().size(), 1); assertTrue(plan2.getTablesQueried().get(0).endsWith(TEST_TABLE.toLowerCase())); assertNotNull(plan2.getPrepareHandle()); @@ -395,33 +392,33 @@ public class TestQueryService extends LensJerseyTest { final FormDataMultiPart mp = new FormDataMultiPart(); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - MediaType.APPLICATION_XML_TYPE)); + MediaType.APPLICATION_XML_TYPE)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select NO_ID from " - + TEST_TABLE)); + + TEST_TABLE)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "explain")); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(), - MediaType.APPLICATION_XML_TYPE)); + MediaType.APPLICATION_XML_TYPE)); final Response responseExplain = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); - assertEquals(responseExplain.getStatus(), Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()); + assertEquals(responseExplain.getStatus(), INTERNAL_SERVER_ERROR.getStatusCode()); // Test explain and prepare final WebTarget ptarget = target().path("queryapi/preparedqueries"); final FormDataMultiPart mp2 = new FormDataMultiPart(); mp2.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - MediaType.APPLICATION_XML_TYPE)); + MediaType.APPLICATION_XML_TYPE)); mp2.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select NO_ID from " - + TEST_TABLE)); + + TEST_TABLE)); mp2.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "explain_and_prepare")); mp2.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(), - MediaType.APPLICATION_XML_TYPE)); + MediaType.APPLICATION_XML_TYPE)); final Response responseExplainAndPrepare = target.request().post( - Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); + Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); - assertEquals(responseExplainAndPrepare.getStatus(), Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()); + assertEquals(responseExplainAndPrepare.getStatus(), INTERNAL_SERVER_ERROR.getStatusCode()); } // post to preparedqueries @@ -451,7 +448,7 @@ public class TestQueryService extends LensJerseyTest { MediaType.APPLICATION_XML_TYPE)); final QueryPrepareHandle pHandle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensAPIResult<QueryPrepareHandle>>() {}).getData(); + new GenericType<LensAPIResult<QueryPrepareHandle>>() {}).getData(); // Get all prepared queries List<QueryPrepareHandle> allQueries = (List<QueryPrepareHandle>) target.queryParam("sessionid", lensSessionId) @@ -493,32 +490,11 @@ public class TestQueryService extends LensJerseyTest { .post(Entity.entity(confpart, MediaType.MULTIPART_FORM_DATA_TYPE), QueryHandle.class); assertNotEquals(handle1, handle2); - LensQuery ctx1 = target().path("queryapi/queries").path(handle1.toString()).queryParam("sessionid", lensSessionId) - .request().get(LensQuery.class); + LensQuery ctx1 = waitForQueryToFinish(target(), lensSessionId, handle1, Status.SUCCESSFUL); assertEquals(ctx1.getQueryName().toLowerCase(), "testquery1"); - // wait till the query finishes - QueryStatus stat = ctx1.getStatus(); - while (!stat.finished()) { - ctx1 = target().path("queryapi/queries").path(handle1.toString()).queryParam("sessionid", lensSessionId) - .request().get(LensQuery.class); - stat = ctx1.getStatus(); - Thread.sleep(1000); - } - assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); - LensQuery ctx2 = target().path("queryapi/queries").path(handle2.toString()).queryParam("sessionid", lensSessionId) - .request().get(LensQuery.class); - assertNotNull(ctx2); + LensQuery ctx2 = waitForQueryToFinish(target(), lensSessionId, handle2, Status.SUCCESSFUL); assertEquals(ctx2.getQueryName().toLowerCase(), "testqueryname2"); - // wait till the query finishes - stat = ctx2.getStatus(); - while (!stat.finished()) { - ctx2 = target().path("queryapi/queries").path(handle1.toString()).queryParam("sessionid", lensSessionId) - .request().get(LensQuery.class); - stat = ctx2.getStatus(); - Thread.sleep(1000); - } - assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); // destroy prepared APIResult result = target.path(pHandle.toString()).queryParam("sessionid", lensSessionId).request() @@ -550,7 +526,7 @@ public class TestQueryService extends LensJerseyTest { final QueryPlan plan = target.request() .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensAPIResult<QueryPlan>>() {}).getData(); + new GenericType<LensAPIResult<QueryPlan>>() {}).getData(); assertEquals(plan.getTablesQueried().size(), 1); assertTrue(plan.getTablesQueried().get(0).endsWith(TEST_TABLE.toLowerCase())); @@ -588,29 +564,8 @@ public class TestQueryService extends LensJerseyTest { .post(Entity.entity(confpart, MediaType.MULTIPART_FORM_DATA_TYPE), QueryHandle.class); assertNotEquals(handle1, handle2); - LensQuery ctx1 = target().path("queryapi/queries").path(handle1.toString()).queryParam("sessionid", lensSessionId) - .request().get(LensQuery.class); - // wait till the query finishes - QueryStatus stat = ctx1.getStatus(); - while (!stat.finished()) { - ctx1 = target().path("queryapi/queries").path(handle1.toString()).queryParam("sessionid", lensSessionId) - .request().get(LensQuery.class); - stat = ctx1.getStatus(); - Thread.sleep(1000); - } - assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); - - LensQuery ctx2 = target().path("queryapi/queries").path(handle2.toString()).queryParam("sessionid", lensSessionId) - .request().get(LensQuery.class); - // wait till the query finishes - stat = ctx2.getStatus(); - while (!stat.finished()) { - ctx2 = target().path("queryapi/queries").path(handle1.toString()).queryParam("sessionid", lensSessionId) - .request().get(LensQuery.class); - stat = ctx2.getStatus(); - Thread.sleep(1000); - } - assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + waitForQueryToFinish(target(), lensSessionId, handle1, Status.SUCCESSFUL); + waitForQueryToFinish(target(), lensSessionId, handle2, Status.SUCCESSFUL); // destroy prepared APIResult result = target.path(plan.getPrepareHandle().toString()).queryParam("sessionid", lensSessionId).request() @@ -660,7 +615,7 @@ public class TestQueryService extends LensJerseyTest { assertTrue(lensQuery.getStatus().getStatus().equals(Status.QUEUED) || lensQuery.getStatus().getStatus().equals(Status.LAUNCHED) || lensQuery.getStatus().getStatus().equals(Status.RUNNING) - || lensQuery.getStatus().getStatus().equals(Status.SUCCESSFUL)); + || lensQuery.getStatus().getStatus().equals(Status.SUCCESSFUL), lensQuery.getStatus().toString()); // wait till the query finishes QueryStatus stat = lensQuery.getStatus(); @@ -729,14 +684,7 @@ public class TestQueryService extends LensJerseyTest { new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); // Get query - lensQuery = target.path(handle3.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - // wait till the query finishes - stat = lensQuery.getStatus(); - while (!stat.finished()) { - lensQuery = target.path(handle3.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - stat = lensQuery.getStatus(); - Thread.sleep(1000); - } + waitForQueryToFinish(target(), lensSessionId, handle3, Status.SUCCESSFUL); validateHttpEndPoint(target(), null, handle3, null); } @@ -918,7 +866,7 @@ public class TestQueryService extends LensJerseyTest { List<String> actualRows = Arrays.asList(result.split("\n")); validatePersistentResult(actualRows); } else { - assertEquals(Response.Status.SEE_OTHER.getStatusCode(), response.getStatus()); + assertEquals(SEE_OTHER.getStatusCode(), response.getStatus()); assertTrue(response.getHeaderString("Location").contains(redirectUrl)); } } @@ -934,10 +882,10 @@ public class TestQueryService extends LensJerseyTest { try { Response response = parent.path("queryapi/queries/" + handle.toString() + "/httpresultset") .queryParam("sessionid", lensSessionId).request().get(); - if (Response.Status.NOT_FOUND.getStatusCode() != response.getStatus()) { + if (NOT_FOUND.getStatusCode() != response.getStatus()) { fail("Expected not found excepiton, but got:" + response.getStatus()); } - assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + assertEquals(response.getStatus(), NOT_FOUND.getStatusCode()); } catch (NotFoundException e) { // expected log.error("Resource not found.", e); @@ -975,27 +923,13 @@ public class TestQueryService extends LensJerseyTest { assertNotNull(handle); // Get query - LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request() - .get(LensQuery.class); - assertTrue(ctx.getStatus().getStatus().equals(Status.QUEUED) - || ctx.getStatus().getStatus().equals(Status.LAUNCHED) || ctx.getStatus().getStatus().equals(Status.RUNNING) - || ctx.getStatus().getStatus().equals(Status.SUCCESSFUL)); - - // wait till the query finishes - QueryStatus stat = ctx.getStatus(); - while (!stat.finished()) { - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - stat = ctx.getStatus(); - Thread.sleep(1000); - } - assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + waitForQueryToFinish(target(), lensSessionId, handle, Status.SUCCESSFUL); // fetch results validateResultSetMetadata(handle, "", new String[][]{{"ID", "INT"}, {"IDSTR", "STRING"}}, target(), lensSessionId); - InMemoryQueryResult resultset = target.path(handle.toString()).path("resultset") .queryParam("sessionid", lensSessionId).request().get(InMemoryQueryResult.class); validateInmemoryResult(resultset); @@ -1030,20 +964,7 @@ public class TestQueryService extends LensJerseyTest { assertNotNull(dropHandle); // Get query - LensQuery ctx = target.path(dropHandle.toString()).queryParam("sessionid", lensSessionId).request() - .get(LensQuery.class); - assertTrue(ctx.getStatus().getStatus().equals(Status.QUEUED) - || ctx.getStatus().getStatus().equals(Status.LAUNCHED) || ctx.getStatus().getStatus().equals(Status.RUNNING) - || ctx.getStatus().getStatus().equals(Status.SUCCESSFUL)); - - // wait till the query finishes - QueryStatus stat = ctx.getStatus(); - while (!stat.finished()) { - ctx = target.path(dropHandle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - stat = ctx.getStatus(); - Thread.sleep(1000); - } - assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + waitForQueryToFinish(target(), lensSessionId, dropHandle, Status.SUCCESSFUL); final FormDataMultiPart mp = new FormDataMultiPart(); conf = new LensConf(); @@ -1061,19 +982,7 @@ public class TestQueryService extends LensJerseyTest { assertNotNull(handle); // Get query - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - assertTrue(ctx.getStatus().getStatus().equals(Status.QUEUED) - || ctx.getStatus().getStatus().equals(Status.LAUNCHED) || ctx.getStatus().getStatus().equals(Status.RUNNING) - || ctx.getStatus().getStatus().equals(Status.SUCCESSFUL)); - - // wait till the query finishes - stat = ctx.getStatus(); - while (!stat.finished()) { - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - stat = ctx.getStatus(); - Thread.sleep(1000); - } - assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + waitForQueryToFinish(target(), lensSessionId, handle, Status.SUCCESSFUL); String select = "SELECT * FROM temp_output"; final FormDataMultiPart fetch = new FormDataMultiPart(); @@ -1089,16 +998,7 @@ public class TestQueryService extends LensJerseyTest { assertNotNull(handle2); // Get query - ctx = target.path(handle2.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - - // wait till the query finishes - stat = ctx.getStatus(); - while (!stat.finished()) { - ctx = target.path(handle2.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - stat = ctx.getStatus(); - Thread.sleep(1000); - } - assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + waitForQueryToFinish(target(), lensSessionId, handle2, Status.SUCCESSFUL); // fetch results validateResultSetMetadata(handle2, "temp_output.", new String[][]{{"ID", "INT"}, {"IDSTR", "STRING"}}, @@ -1236,10 +1136,10 @@ public class TestQueryService extends LensJerseyTest { // set a timeout value enough for tests mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("timeoutmillis").build(), "300000")); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(), - MediaType.APPLICATION_XML_TYPE)); + MediaType.APPLICATION_XML_TYPE)); Response response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); - assertEquals(response.getStatus(), Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()); + assertEquals(response.getStatus(), INTERNAL_SERVER_ERROR.getStatusCode()); } /** @@ -1423,7 +1323,7 @@ public class TestQueryService extends LensJerseyTest { LensErrorTO expectedLensErrorTO = LensErrorTO.composedOf( - LensCubeErrorCode.NEITHER_CUBE_NOR_DIMENSION.getLensErrorInfo().getErrorCode(), + LensCubeErrorCode.NEITHER_CUBE_NOR_DIMENSION.getLensErrorInfo().getErrorCode(), "Neither cube nor dimensions accessed in the query", TestDataUtils.MOCK_STACK_TRACE); ErrorResponseExpectedData expectedData = new ErrorResponseExpectedData(BAD_REQUEST, expectedLensErrorTO); @@ -1499,6 +1399,7 @@ public class TestQueryService extends LensJerseyTest { "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-PARALLEL_ESTIMATE")), reg.getGauges().keySet().toString()); } + @Test public void testQueryRejection() throws InterruptedException, IOException { final WebTarget target = target().path("queryapi/queries"); @@ -1515,4 +1416,40 @@ public class TestQueryService extends LensJerseyTest { Response response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); assertEquals(response.getStatus(), 400); } + + /** + * Test query purger + * + * @throws InterruptedException the interrupted exception + * @throws IOException Signals that an I/O exception has occurred. + */ + @Test + public void testQueryPurger() throws InterruptedException, IOException { + waitForPurge(); + LensConf conf = getLensConf(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); + // test post execute op + LensQuery ctx1 = executeAndWaitForQueryToFinish(target(), lensSessionId, + "select ID, IDSTR from " + TEST_TABLE, + Optional.of(conf), Optional.of(Status.SUCCESSFUL)); + LensQuery ctx2 = executeAndWaitForQueryToFinish(target(), lensSessionId, + "select ID, IDSTR from " + TEST_TABLE, + Optional.of(conf), Optional.of(Status.SUCCESSFUL)); + LensQuery ctx3 = executeAndWaitForQueryToFinish(target(), lensSessionId, + "select ID, IDSTR from " + TEST_TABLE, + Optional.of(conf), Optional.of(Status.SUCCESSFUL)); + waitForPurge(3, queryService.finishedQueries); + assertEquals(queryService.finishedQueries.size(), 3); + getLensQueryResult(target(), lensSessionId, ctx3.getQueryHandle()); + waitForPurge(2, queryService.finishedQueries); + assertTrue(queryService.finishedQueries.size() == 2); + getLensQueryResult(target(), lensSessionId, ctx2.getQueryHandle()); + waitForPurge(1, queryService.finishedQueries); + assertTrue(queryService.finishedQueries.size() == 1); + getLensQueryResult(target(), lensSessionId, ctx1.getQueryHandle()); + } + + @AfterMethod + private void waitForPurge() throws InterruptedException { + waitForPurge(0, queryService.finishedQueries); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/669e8727/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java index 98a658e..987129c 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java @@ -130,9 +130,9 @@ public class TestResultFormatting extends LensJerseyTest { conf.addProperty(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName()); testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, null); - conf.addProperty(LensConfConstants.RESULT_FS_READ_URL, "filereadurl://"); + queryService.conf.set(LensConfConstants.RESULT_FS_READ_URL, "filereadurl://"); testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, "filereadurl://"); - + queryService.conf.unset(LensConfConstants.RESULT_FS_READ_URL); } // test with execute async post with result formatter, get query, get results @@ -149,8 +149,9 @@ public class TestResultFormatting extends LensJerseyTest { conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true"); testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, null); - conf.addProperty(LensConfConstants.RESULT_FS_READ_URL, "filereadurl://"); + queryService.conf.set(LensConfConstants.RESULT_FS_READ_URL, "filereadurl://"); testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, "filereadurl://"); + queryService.conf.unset(LensConfConstants.RESULT_FS_READ_URL); } /** @@ -262,4 +263,8 @@ public class TestResultFormatting extends LensJerseyTest { } } + @AfterTest + public void waitForPurge() throws InterruptedException { + waitForPurge(0, queryService.finishedQueries); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/669e8727/lens-server/src/test/resources/lens-site.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml index f728810..cc887ef 100644 --- a/lens-server/src/test/resources/lens-site.xml +++ b/lens-server/src/test/resources/lens-site.xml @@ -127,9 +127,9 @@ </property> <property> - <!-- Keep only the last query in memory --> - <name>lens.server.max.finished.queries</name> - <value>1</value> + <!-- run every second --> + <name>lens.server.purge.interval</name> + <value>1000</value> </property> <property> http://git-wip-us.apache.org/repos/asf/lens/blob/669e8727/src/site/apt/admin/config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt index 3500441..3a0678f 100644 --- a/src/site/apt/admin/config.apt +++ b/src/site/apt/admin/config.apt @@ -101,43 +101,43 @@ Lens server configuration *--+--+---+--+ |37|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.| *--+--+---+--+ -|38|lens.server.max.finished.queries|100|Maximum number of finished queries which lens server will keep in memory before purging.| +|38|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service| *--+--+---+--+ -|39|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service| +|39|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource| *--+--+---+--+ -|40|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource| +|40|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.| *--+--+---+--+ -|41|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.| +|41|lens.server.metrics.ganglia.host| |The ganglia host name| *--+--+---+--+ -|42|lens.server.metrics.ganglia.host| |The ganglia host name| +|42|lens.server.metrics.ganglia.port| |The ganglia port| *--+--+---+--+ -|43|lens.server.metrics.ganglia.port| |The ganglia port| +|43|lens.server.metrics.graphite.host| |The graphite host name| *--+--+---+--+ -|44|lens.server.metrics.graphite.host| |The graphite host name| +|44|lens.server.metrics.graphite.port| |The graphite port| *--+--+---+--+ -|45|lens.server.metrics.graphite.port| |The graphite port| +|45|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds| *--+--+---+--+ -|46|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds| +|46|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.| *--+--+---+--+ -|47|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.| +|47|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource| *--+--+---+--+ -|48|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource| +|48|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location| *--+--+---+--+ -|49|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location| +|49|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.| *--+--+---+--+ -|50|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.| +|50|lens.server.query.launching.constraint.factories|org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory|Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all constraints pass.| *--+--+---+--+ -|51|lens.server.query.launching.constraint.factories|org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory|Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all constraints pass.| -*--+--+---+--+ -|52|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ | +|51|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ | | | | |Use cases will be to use extra intelligence to convert user query to optimized cube query. \ | | | | |Or define shortcuts for certain frequently used queries :) | *--+--+---+--+ -|53|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service| +|52|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service| +*--+--+---+--+ +|53|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in logback xml for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger| *--+--+---+--+ -|54|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in logback xml for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger| +|54|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource| *--+--+---+--+ -|55|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource| +|55|lens.server.querypurger.sleep.interval|10000|The interval(milliseconds) with which purger to run periodically. Default 10 sec.| *--+--+---+--+ |56|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service| *--+--+---+--+ @@ -167,7 +167,7 @@ Lens server configuration *--+--+---+--+ |69|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.| *--+--+---+--+ -|70|lens.server.servicenames|session,query,metastore,scheduler,quota,savedquery|These services would be started in the specified order when lens-server starts up| +|70|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up| *--+--+---+--+ |71|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs| *--+--+---+--+ @@ -237,6 +237,6 @@ Lens server configuration *--+--+---+--+ |104|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| *--+--+---+--+ -|105|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index,log,savedquery|These JAX-RS resources would be started in the specified order when lens-server starts up| +|105|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| *--+--+---+--+ The configuration parameters and their default values
