Repository: lens Updated Branches: refs/heads/master 934f84573 -> d7f7a2eee
LENS-1190: Support Query End Notification to a HTTP end point. Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d7f7a2ee Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d7f7a2ee Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d7f7a2ee Branch: refs/heads/master Commit: d7f7a2eee0e1829e152b30f10b1539425bf053bb Parents: 934f845 Author: Puneet Gupta <[email protected]> Authored: Mon Aug 8 13:46:54 2016 +0530 Committer: Puneet <[email protected]> Committed: Mon Aug 8 13:46:54 2016 +0530 ---------------------------------------------------------------------- .../lens/server/api/LensConfConstants.java | 57 ++++- .../lens/server/query/QueryEndHttpNotifier.java | 68 ++++++ .../server/query/QueryEventHttpNotifier.java | 229 +++++++++++++++++++ .../server/query/QueryExecutionServiceImpl.java | 2 + .../src/main/resources/lensserver-default.xml | 14 ++ .../src/main/resources/lenssession-default.xml | 25 ++ .../query/TestQueryNotifictaionResource.java | 80 +++++++ .../lens/server/query/TestQueryService.java | 36 ++- src/site/apt/admin/config.apt | 174 +++++++------- src/site/apt/admin/session-config.apt | 52 +++-- 10 files changed, 626 insertions(+), 111 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/d7f7a2ee/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 72e2b61..57d13f9 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -18,6 +18,8 @@ */ package org.apache.lens.server.api; +import javax.ws.rs.core.MediaType; + import org.apache.lens.server.api.error.LensException; /** @@ -893,10 +895,10 @@ public final class LensConfConstants { public static final int DEFAULT_LAUNCHER_POOL_MIN_THREADS = 3; /** - * Key used to get maximum number of threads in the laucnher thread pool + * Key used to get maximum number of threads in the launcher thread pool */ public static final String LAUNCHER_POOL_MAX_THREADS = SERVER_PFX + "launcher.pool.max.threads"; - // keeping the default to hundred, we may never grow till there, it would go to max for concurrrent queries allowed on + // keeping the default to hundred, we may never grow till there, it would go to max for concurrent queries allowed on // all drivers together. public static final int DEFAULT_LAUNCHER_POOL_MAX_THREADS = 100; @@ -1098,4 +1100,55 @@ public final class LensConfConstants { * Query current time for the scheduled query. */ public static final String QUERY_CURRENT_TIME_IN_MILLIS = QUERY_PFX + "current.time.millis"; + + /** + * This is the connection timeout for all HTTP Notifications sent by lens server + */ + public static final String HTTP_NOTIFICATION_CONN_TIMEOUT_MILLIS = SERVER_PFX + + "http.notification.conn.timeout.millis"; + + /** + * Default connection timeout is 5 secs + */ + public static final int DEFAULT_HTTP_NOTIFICATION_CONN_TIMEOUT_MILLIS = 5000; //5 secs + + /** + * This is the read timeout for all HTTP Notifications sent by lens server + */ + public static final String HTTP_NOTIFICATION_READ_TIMEOUT_MILLIS = SERVER_PFX + + "http.notification.read.timeout.millis"; + + /** + * Default read timeout is 5 secs + */ + public static final int DEFAULT_HTTP_NOTIFICATION_READ_TIMEOUT_MILLIS = 10000; //10 secs + + /** + * This is the media type for Query Http notifications + */ + public static final String QUERY_HTTP_NOTIFICATION_MEDIATYPE = QUERY_PFX + "http.notification.mediatype"; + + /** + * This is the default media type for all Query Http notifications + */ + public static final String DEFAULT_QUERY_HTTP_NOTIFICATION_MEDIATYPE = MediaType.APPLICATION_JSON; + + /** + * These are the end points for Query http notifications. Users can specify more than one comma separated end points + */ + public static final String QUERY_HTTP_NOTIFICATION_URLS = QUERY_PFX + "http.notification.urls"; + + /** + * This is the prefix for Query http Notification. User is expected to add notification type along with prefix + */ + public static final String QUERY_HTTP_NOTIFICATION_TYPE_PFX = QUERY_PFX + "http.notification.type."; + + /** + * This is the property for enabling Query finished Http notification. This will include successful, failed and + * cancelled queries. + */ + public static final String QUERY_HTTP_NOTIFICATION_TYPE_FINISHED = QUERY_HTTP_NOTIFICATION_TYPE_PFX + "FINISHED"; + + + } http://git-wip-us.apache.org/repos/asf/lens/blob/d7f7a2ee/lens-server/src/main/java/org/apache/lens/server/query/QueryEndHttpNotifier.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndHttpNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndHttpNotifier.java new file mode 100644 index 0000000..5d2ddbe --- /dev/null +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndHttpNotifier.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.server.query; + + +import java.util.Map; + +import org.apache.lens.api.query.QueryStatus; +import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.query.QueryEnded; +import org.apache.lens.server.api.query.QueryEvent; +import org.apache.lens.server.model.LogSegregationContext; + +import org.apache.hadoop.conf.Configuration; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class QueryEndHttpNotifier extends QueryEventHttpNotifier<QueryEnded> { + + private final LogSegregationContext logSegregationContext; + private static final int CORE_POOL_SIZE = 3; + + public QueryEndHttpNotifier(Configuration config, LogSegregationContext logSegregationContext) { + super(config, CORE_POOL_SIZE); + this.logSegregationContext = logSegregationContext; + } + + @Override + public void process(QueryEnded event) { + logSegregationContext.setLogSegragationAndQueryId(event.getQueryContext().getQueryHandleString()); + super.process(event, event.getQueryContext()); + } + + @Override + protected boolean isHttpNotificationEnabled(QueryEvent event, QueryContext queryContext) { + return (event.getCurrentValue() != QueryStatus.Status.CLOSED) // CLOSED QueryEnded events to be skipped + && super.isHttpNotificationEnabled(event, queryContext); + } + + @Override + protected void updateExtraEventDetails(QueryEvent event, QueryContext queryContext, + Map<String, Object> eventDetails) { + //Nothing specific as of now for finished queries. We can attach query results later if required. + } + + @Override + protected NotificationType getNotificationType() { + return NotificationType.FINISHED; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/d7f7a2ee/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java new file mode 100644 index 0000000..f264603 --- /dev/null +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.server.query; + + +import static org.apache.lens.server.api.LensConfConstants.*; + +import java.util.HashMap; +import java.util.Map; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.events.AsyncEventListener; +import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.query.QueryEnded; +import org.apache.lens.server.api.query.QueryEvent; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; + +import org.glassfish.jersey.client.ClientProperties; +import org.glassfish.jersey.media.multipart.FormDataBodyPart; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.glassfish.jersey.moxy.json.MoxyJsonFeature; + +import lombok.extern.slf4j.Slf4j; + +/** + * Base class for all QueryEvent HTTP Notifications. Subclasses are expected to override + * @{link #getNotificationType} and {@link #updateExtraEventDetails(QueryEvent, QueryContext, Map)}. + * Subclasses can also override @{link {@link #isHttpNotificationEnabled(QueryEvent, QueryContext)}} if specific + * logic is required to check whether http notification is enabled/intended for an event. + * + */ + +@Slf4j +public abstract class QueryEventHttpNotifier<T extends QueryEvent> extends AsyncEventListener<T> { + + private final Configuration config; + + public QueryEventHttpNotifier(Configuration config, int poolSize) { + super(poolSize); + this.config = config; + } + + /** + * Type of query notifications that are supported. + * We can add to this list once we support more notification types. + */ + protected static enum NotificationType { + FINISHED, + LAUNCHED + } + + /** + * Return the type of query Notification handled. + * Expecting subclasses to pass appropriate type. + * @return + */ + protected abstract NotificationType getNotificationType(); + + /** + * Checks if the events needs a HTTP notification. + * + * @param event + * @param queryContext + * @return + */ + protected boolean isHttpNotificationEnabled(QueryEvent event, QueryContext queryContext) { + + if (queryContext == null) { + log.warn("Could not find the context for {} for event:{}. {} HTTP Notification will be generated", + event.getQueryHandle(), event.getCurrentValue(), getNotificationType()); + return false; + } + + boolean isQueryHTTPNotificationEnabled = queryContext.getConf().getBoolean( + QUERY_HTTP_NOTIFICATION_TYPE_PFX + getNotificationType().name(), false); + if (!isQueryHTTPNotificationEnabled) { + log.info("{} HTTP notification for query {} is not enabled", + getNotificationType(), queryContext.getQueryHandleString()); + return false; + } + + return true; + } + + /** + * Processes each incoming event + * + * @param event + * @param queryContext + */ + protected void process(QueryEnded event, QueryContext queryContext) { + if (!isHttpNotificationEnabled(event, queryContext)) { + return; + } + + String httpEndPointDetails = queryContext.getConf().get(QUERY_HTTP_NOTIFICATION_URLS); + String[] httpEndPoints = null; + if (StringUtils.isEmpty(httpEndPointDetails)) { + log.warn("HTTP notification end points not set for query {}. Skipping {} notification", + queryContext.getQueryHandleString(), getNotificationType()); + return; + } else { + httpEndPoints = httpEndPointDetails.trim().split("\\s*,\\s*"); + } + String mediaType = queryContext.getConf().get(QUERY_HTTP_NOTIFICATION_MEDIATYPE, + DEFAULT_QUERY_HTTP_NOTIFICATION_MEDIATYPE); + + Map<String, Object> eventDetails = new HashMap<>(); + updateBasicEventDetails(event, queryContext, eventDetails); + updateExtraEventDetails(event, queryContext, eventDetails); + + int responseCode; + for (String httpEndPoint : httpEndPoints) { + try { + responseCode = notifyEvent(httpEndPoint, eventDetails, MediaType.valueOf(mediaType)); + log.info("{} HTTP Notification sent successfully for query {} to {}. Response code {}", getNotificationType(), + queryContext.getQueryHandleString(), httpEndPoint, responseCode); + } catch (LensException e) { + log.error("Error while sending {} HTTP Notification for Query {} to {}", getNotificationType(), + queryContext.getQueryHandleString(), httpEndPoint, e); + } + } + } + + /** + * Basic event details are filled here which are common to all events. + * + * @param event + * @param queryContext + * @param eventDetails + */ + private void updateBasicEventDetails(QueryEvent event, QueryContext queryContext, + Map<String, Object> eventDetails) { + eventDetails.put("eventtype", getNotificationType().name()); + eventDetails.put("eventtime", event.getEventTime()); + eventDetails.put("query", queryContext.toLensQuery()); + } + + /** + * Subclasses are expected to provide extra (more specific) event details. + * Example for FINISHED notification, STATUS can be provided + * + * @param event + * @param queryContext + * @param eventDetails + */ + protected abstract void updateExtraEventDetails(QueryEvent event, QueryContext queryContext, + Map<String, Object> eventDetails); + + /** + * Notifies the http end point about the event. Callers can choose the media type + * + * @param httpEndPoint + * @param eventDetails + * @param mediaType + * @return response code in case of success + * @throws LensException in case of exception or http response status != 2XX + */ + private int notifyEvent(String httpEndPoint, Map<String, Object> eventDetails, MediaType mediaType) + throws LensException { + + final WebTarget target = buildClient().target(httpEndPoint); + FormDataMultiPart mp = new FormDataMultiPart(); + for (Map.Entry<String, Object> eventDetail : eventDetails.entrySet()) { + //Using plain format for primitive data types. + MediaType resolvedMediaType = (eventDetail.getValue() instanceof Number + || eventDetail.getValue() instanceof String) ? MediaType.TEXT_PLAIN_TYPE : mediaType; + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name(eventDetail.getKey()).build(), + eventDetail.getValue(), resolvedMediaType)); + } + Response response; + try { + response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); + } catch (Exception e) { + throw new LensException("Error while publishing Http notification", e); + } + //2XX = SUCCESS + if (!(response.getStatus() >= 200 && response.getStatus() < 300)) { + throw new LensException("Error while publishing Http notification. Response code " + response.getStatus()); + } + return response.getStatus(); + } + + /** + * Builds a rest client which has a connection and read timeout of 5 and 10 secs respectively by default. + * + * @return + */ + private Client buildClient() { + ClientBuilder cb = ClientBuilder.newBuilder().register(MultiPartFeature.class).register(MoxyJsonFeature.class) + .register(MoxyJsonConfigurationContextResolver.class); + Client client = cb.build(); + //Set Timeouts + client.property(ClientProperties.CONNECT_TIMEOUT, + config.getInt(HTTP_NOTIFICATION_CONN_TIMEOUT_MILLIS, DEFAULT_HTTP_NOTIFICATION_CONN_TIMEOUT_MILLIS)); + client.property(ClientProperties.READ_TIMEOUT, + config.getInt(HTTP_NOTIFICATION_READ_TIMEOUT_MILLIS, DEFAULT_HTTP_NOTIFICATION_READ_TIMEOUT_MILLIS)); + return client; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/d7f7a2ee/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 84dcecd..1752414 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -365,6 +365,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE QueryEnded.class); getEventService().addListenerForType( new QueryEndNotifier(this, getCliService().getHiveConf(), this.logSegregationContext), QueryEnded.class); + getEventService().addListenerForType( + new QueryEndHttpNotifier(getCliService().getHiveConf(), this.logSegregationContext), QueryEnded.class); log.info("Registered query result formatter"); } http://git-wip-us.apache.org/repos/asf/lens/blob/d7f7a2ee/lens-server/src/main/resources/lensserver-default.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml index e9525fd..d672c88 100644 --- a/lens-server/src/main/resources/lensserver-default.xml +++ b/lens-server/src/main/resources/lensserver-default.xml @@ -875,4 +875,18 @@ <description>Query current time in millis. This is used to resolve 'now'. If value is set to zero, 'now' is resolved to current value</description> </property> + + <property> + <name>lens.server.http.notification.conn.timeout.millis</name> + <value>5000</value> + <description>This is the connection timeout for all HTTP Notifications sent by lens server. + Default connection timeout is 5 secs</description> + </property> + <property> + <name>lens.server.http.notification.read.timeout.millis</name> + <value>10000</value> + <description>This is the read timeout for all HTTP Notifications sent by lens server. + Default read timeout is 10 secs</description> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/d7f7a2ee/lens-server/src/main/resources/lenssession-default.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/lenssession-default.xml b/lens-server/src/main/resources/lenssession-default.xml index 62fca51..b6bfa2e 100644 --- a/lens-server/src/main/resources/lenssession-default.xml +++ b/lens-server/src/main/resources/lenssession-default.xml @@ -242,6 +242,31 @@ </description> </property> + <property> + <name>lens.query.http.notification.mediatype</name> + <value>application/json</value> + <description>This is the media type for Query Http notifications. Accepted types are "application/json" and + "application/xml". The default value is "application/json"</description> + </property> + + <property> + <name>lens.query.http.notification.type.FINISHED</name> + <value>false</value> + <description>Setting this property to true will enable query FINISHED notifications which includes + SUCCESSFUL, FAILED and CANCELLED queries. The notification will have eventtype = "FINISHED", + eventtime = long event time and query = org.apache.lens.api.query.LensQuery instance. The mediatype for + eventtype and eventtime will be TEXT/PLAIN and the mediatype for query will be based on property + lens.query.http.notification.mediatype. Default value of this property is false. </description> + </property> + + <property> + <name>lens.query.http.notification.urls</name> + <value></value> + <description>These are the http end points for Query http notifications. Users can specify more than one comma + separated end points for a query. If this property is not set, no http notification will be sent out by lens + server for the query.</description> + </property> + <!-- properties for session --> <property> <name>lens.session.aux.jars</name> http://git-wip-us.apache.org/repos/asf/lens/blob/d7f7a2ee/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java new file mode 100644 index 0000000..3a3f2f1 --- /dev/null +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.server.query; + +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; + +import org.apache.lens.api.query.*; +import org.apache.lens.server.api.error.LensException; + + +import org.glassfish.jersey.media.multipart.FormDataParam; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +@Path("/queryapi/notifictaion") +public class TestQueryNotifictaionResource { + + @Getter + private static int finishedCount = 0; + @Getter + private static int successfulCount = 0; + @Getter + private static int failedCount = 0; + @Getter + private static int cancelledCount = 0; + + @POST + @Path("finished") + @Consumes({MediaType.MULTIPART_FORM_DATA}) + @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN}) + public void prepareQuery( + @FormDataParam("eventtype") String eventtype, + @FormDataParam("eventtime") String eventtime, + @FormDataParam("query") LensQuery query) throws LensException { + + System.out.println("@@@@ Received Finished Event for queryid: " + query.getQueryHandleString() + + " queryname:" + query.getQueryName() + " user:" + query.getSubmittedUser() + + " status:" + query.getStatus() + " eventtype:" + eventtype); + + finishedCount++; + + if (query.getStatus().successful()) { + successfulCount++; + } else if (query.getStatus().failed()) { + failedCount++; + } else if (query.getStatus().cancelled()) { + cancelledCount++; + } + } + + public static void clearState() { + finishedCount = 0; + successfulCount = 0; + cancelledCount = 0; + failedCount = 0; + } + + +} http://git-wip-us.apache.org/repos/asf/lens/blob/d7f7a2ee/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index 9f40a01..a1ecaa6 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -114,6 +114,7 @@ public class TestQueryService extends LensJerseyTest { final Set<Class<?>> classes = super.getClasses(); classes.add(GenericExceptionMapper.class); classes.add(LensJAXBContextResolver.class); + classes.add(TestQueryNotifictaionResource.class); return classes; } } @@ -1387,7 +1388,7 @@ public class TestQueryService extends LensJerseyTest { } /** * Data provider for test case - * {@link #testExecuteWithTimeoutAndPreFetchAndServerPersistence(long, int, boolean, long)} + * {@link #testExecuteWithTimeoutAndPreFetchAndServerPersistence(long, int, boolean, long, String, int)} */ @DataProvider public Object[][] executeWithTimeoutAndPreFetechAndServerPersistenceDP() { @@ -1890,4 +1891,37 @@ public class TestQueryService extends LensJerseyTest { private void waitForPurge() throws InterruptedException { waitForPurge(0, queryService.finishedQueries); } + + + @Test(dataProvider = "mediaTypeData") + public void testFinishedNotifictaion(MediaType mt) throws LensException, InterruptedException { + String query = "select ID, IDSTR, count(*) from " + TEST_TABLE + " group by ID, IDSTR"; + String endpoint = getBaseUri() + "/queryapi/notifictaion/finished"; + LensConf conf = new LensConf(); + conf.addProperty(LensConfConstants.QUERY_HTTP_NOTIFICATION_TYPE_FINISHED, "true"); + conf.addProperty(LensConfConstants.QUERY_HTTP_NOTIFICATION_MEDIATYPE, mt); + conf.addProperty(LensConfConstants.QUERY_HTTP_NOTIFICATION_URLS, endpoint + " , " + endpoint); + + //Test for SUCCESSFUL FINISH notification + queryService.execute(lensSessionId, query, 20000, conf, "testHttpNotifictaionQuery"); + + //TEST for CANCELLED FINISH notification + conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); + conf.addProperty(LensConfConstants.QUERY_OUTPUT_FORMATTER, DeferredPersistentResultFormatter.class.getName()); + conf.addProperty("deferPersistenceByMillis", 5000); // defer persistence for 5 secs + QueryHandle hanlde = queryService.executeAsync(lensSessionId, query, conf, "testHttpNotifictaionQuery"); + queryService.cancelQuery(lensSessionId, hanlde); + + //Test for FAILED FINISH notification + conf.addProperty(LensConfConstants.QUERY_OUTPUT_FORMATTER, "wrong.formatter"); + queryService.execute(lensSessionId, query, 20000, conf, "testHttpNotifictaionQuery"); + + Thread.sleep(3000); //Keep some time for notifications to get delivered + assertEquals(TestQueryNotifictaionResource.getSuccessfulCount(), 2); + assertEquals(TestQueryNotifictaionResource.getCancelledCount(), 2); + assertEquals(TestQueryNotifictaionResource.getFailedCount(), 2); + assertEquals(TestQueryNotifictaionResource.getFinishedCount(), 6); + + TestQueryNotifictaionResource.clearState(); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/d7f7a2ee/src/site/apt/admin/config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt index db859e6..4291798 100644 --- a/src/site/apt/admin/config.apt +++ b/src/site/apt/admin/config.apt @@ -93,176 +93,180 @@ Lens server configuration *--+--+---+--+ |33|lens.server.hdfs.output.retention|1 day|hdfs output retention period. Default 1 day| *--+--+---+--+ -|34|lens.server.index.ws.resource.impl|org.apache.lens.server.IndexResource|Implementation class for Index Resource| +|34|lens.server.http.notification.conn.timeout.millis|5000|This is the connection timeout for all HTTP Notifications sent by lens server. Default connection timeout is 5 secs| *--+--+---+--+ -|35|lens.server.inmemory.resultset.ttl.secs|300|This property defines the TTL(time to live) in seconds for all result sets of type InMemoryResultSet beyond which they are eligible for purging irrespective of whether the result set has been read or not. The default value is 300 seconds (5 minutes).| +|35|lens.server.http.notification.read.timeout.millis|10000|This is the read timeout for all HTTP Notifications sent by lens server. Default read timeout is 10 secs| *--+--+---+--+ -|36|lens.server.launcher.pool.keepalive.millis|60000|Thread keep alive time in milliseconds for the query launcher thread pool. If there are no query launches for this period,then cached threads will be released from the pool.| +|36|lens.server.index.ws.resource.impl|org.apache.lens.server.IndexResource|Implementation class for Index Resource| *--+--+---+--+ -|37|lens.server.launcher.pool.max.threads|100|Maximum number of threads in the query launcher thread pool. Keeping the default to hundred, we may never grow till there, it would go to max for concurrrent queries allowed on all drivers together. This value should be greater than the max concurrent queries allowed on all drivers.| +|37|lens.server.inmemory.resultset.ttl.secs|300|This property defines the TTL(time to live) in seconds for all result sets of type InMemoryResultSet beyond which they are eligible for purging irrespective of whether the result set has been read or not. The default value is 300 seconds (5 minutes).| *--+--+---+--+ -|38|lens.server.launcher.pool.min.threads|3|Minimum number of threads in the query launcher thread pool| +|38|lens.server.launcher.pool.keepalive.millis|60000|Thread keep alive time in milliseconds for the query launcher thread pool. If there are no query launches for this period,then cached threads will be released from the pool.| *--+--+---+--+ -|39|lens.server.log.ws.resource.impl|org.apache.lens.server.LogResource|Implementation class for Log Resource| +|39|lens.server.launcher.pool.max.threads|100|Maximum number of threads in the query launcher thread pool. Keeping the default to hundred, we may never grow till there, it would go to max for concurrrent queries allowed on all drivers together. This value should be greater than the max concurrent queries allowed on all drivers.| *--+--+---+--+ -|40|lens.server.mail.from.address|[email protected]|The from field in the notifier mail to the submitter.| +|40|lens.server.launcher.pool.min.threads|3|Minimum number of threads in the query launcher thread pool| *--+--+---+--+ -|41|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail| +|41|lens.server.log.ws.resource.impl|org.apache.lens.server.LogResource|Implementation class for Log Resource| *--+--+---+--+ -|42|lens.server.mail.port|25|SMTP Port| +|42|lens.server.mail.from.address|[email protected]|The from field in the notifier mail to the submitter.| *--+--+---+--+ -|43|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.| +|43|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail| *--+--+---+--+ -|44|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.| +|44|lens.server.mail.port|25|SMTP Port| *--+--+---+--+ -|45|lens.server.max.sessions.per.user|10|Number of sessions can be allowed for each user. User has to close one of the active sessions to open a new session once limit is reached. Otherwise Server throws an exception by saying that opened session limit has been already reached for user.| +|45|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.| *--+--+---+--+ -|46|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service| +|46|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.| *--+--+---+--+ -|47|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource| +|47|lens.server.max.sessions.per.user|10|Number of sessions can be allowed for each user. User has to close one of the active sessions to open a new session once limit is reached. Otherwise Server throws an exception by saying that opened session limit has been already reached for user.| *--+--+---+--+ -|48|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.| +|48|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service| *--+--+---+--+ -|49|lens.server.metrics.ganglia.host| |The ganglia host name| +|49|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource| *--+--+---+--+ -|50|lens.server.metrics.ganglia.port| |The ganglia port| +|50|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.| *--+--+---+--+ -|51|lens.server.metrics.graphite.host| |The graphite host name| +|51|lens.server.metrics.ganglia.host| |The ganglia host name| *--+--+---+--+ -|52|lens.server.metrics.graphite.port| |The graphite port| +|52|lens.server.metrics.ganglia.port| |The ganglia port| *--+--+---+--+ -|53|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds| +|53|lens.server.metrics.graphite.host| |The graphite host name| *--+--+---+--+ -|54|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.| +|54|lens.server.metrics.graphite.port| |The graphite port| *--+--+---+--+ -|55|lens.server.moxyjson.ws.feature.impl|org.glassfish.jersey.moxy.json.MoxyJsonFeature|Enable Moxy json feature| +|55|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds| *--+--+---+--+ -|56|lens.server.moxyjsonconfigresovler.ws.feature.impl|org.apache.lens.api.util.MoxyJsonConfigurationContextResolver|Moxy json configuration resolver| +|56|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.| *--+--+---+--+ -|57|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource| +|57|lens.server.moxyjson.ws.feature.impl|org.glassfish.jersey.moxy.json.MoxyJsonFeature|Enable Moxy json feature| *--+--+---+--+ -|58|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location| +|58|lens.server.moxyjsonconfigresovler.ws.feature.impl|org.apache.lens.api.util.MoxyJsonConfigurationContextResolver|Moxy json configuration resolver| *--+--+---+--+ -|59|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.| +|59|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource| *--+--+---+--+ -|60|lens.server.query.launching.constraint.factories|org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory|Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all constraints pass.| +|60|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location| *--+--+---+--+ -|61|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ | +|61|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.| +*--+--+---+--+ +|62|lens.server.query.launching.constraint.factories|org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory|Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all constraints pass.| +*--+--+---+--+ +|63|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ | | | | |Use cases will be to use extra intelligence to convert user query to optimized cube query. \ | | | | |Or define shortcuts for certain frequently used queries :) | *--+--+---+--+ -|62|lens.server.query.resultset.retention|1 day|Lens query resultset retention period. Default 1 day| +|64|lens.server.query.resultset.retention|1 day|Lens query resultset retention period. Default 1 day| *--+--+---+--+ -|63|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service| +|65|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service| *--+--+---+--+ -|64|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in logback xml for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger| +|66|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in logback xml for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger| *--+--+---+--+ -|65|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource| +|67|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource| *--+--+---+--+ -|66|lens.server.querypurger.sleep.interval|10000|The interval(milliseconds) with which purger to run periodically. Default 10 sec.| +|68|lens.server.querypurger.sleep.interval|10000|The interval(milliseconds) with which purger to run periodically. Default 10 sec.| *--+--+---+--+ -|67|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service| +|69|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service| *--+--+---+--+ -|68|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource| +|70|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource| *--+--+---+--+ -|69|lens.server.requestlogger.ws.filter.impl|org.apache.lens.server.LensRequestLoggingFilter|Implementation class for Request logging Filter| +|71|lens.server.requestlogger.ws.filter.impl|org.apache.lens.server.LensRequestLoggingFilter|Implementation class for Request logging Filter| *--+--+---+--+ -|70|lens.server.resultset.purge.enabled|false|Whether to purge the query results| +|72|lens.server.resultset.purge.enabled|false|Whether to purge the query results| *--+--+---+--+ -|71|lens.server.resultsetpurger.sleep.interval.secs|3600|Periodicity for Query result purger runs. Default 1 hour.| +|73|lens.server.resultsetpurger.sleep.interval.secs|3600|Periodicity for Query result purger runs. Default 1 hour.| *--+--+---+--+ -|72|lens.server.savedquery.jdbc.dialectclass|org.apache.lens.server.query.save.SavedQueryDao$HSQLDialect|Dialect of the target DB, Default is HSQL. Override with the target DB used.| +|74|lens.server.savedquery.jdbc.dialectclass|org.apache.lens.server.query.save.SavedQueryDao$HSQLDialect|Dialect of the target DB, Default is HSQL. Override with the target DB used.| *--+--+---+--+ -|73|lens.server.savedquery.list.default.count|20|Key denoting the default fetch value of saved query list api.| +|75|lens.server.savedquery.list.default.count|20|Key denoting the default fetch value of saved query list api.| *--+--+---+--+ -|74|lens.server.savedquery.list.default.offset|0|Key denoting the default start value of saved query list api.| +|76|lens.server.savedquery.list.default.offset|0|Key denoting the default start value of saved query list api.| *--+--+---+--+ -|75|lens.server.savedquery.service.impl|org.apache.lens.server.query.save.SavedQueryServiceImpl|Implementation class for saved query service| +|77|lens.server.savedquery.service.impl|org.apache.lens.server.query.save.SavedQueryServiceImpl|Implementation class for saved query service| *--+--+---+--+ -|76|lens.server.savedquery.ws.resource.impl|org.apache.lens.server.query.save.SavedQueryResource|Implementation class for Saved query Resource| +|78|lens.server.savedquery.ws.resource.impl|org.apache.lens.server.query.save.SavedQueryResource|Implementation class for Saved query Resource| *--+--+---+--+ -|77|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service| +|79|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service| *--+--+---+--+ -|78|lens.server.scheduler.store.class|org.apache.lens.server.scheduler.SchedulerDAO$SchedulerHsqlDBStore|A subclass of SchedulerDBStore class used for storing scheduler related information.| +|80|lens.server.scheduler.store.class|org.apache.lens.server.scheduler.SchedulerDAO$SchedulerHsqlDBStore|A subclass of SchedulerDBStore class used for storing scheduler related information.| *--+--+---+--+ -|79|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource| +|81|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource| *--+--+---+--+ -|80|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.| +|82|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.| *--+--+---+--+ -|81|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter| +|83|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter| *--+--+---+--+ -|82|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.| +|84|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.| *--+--+---+--+ -|83|lens.server.servicenames|session,query,savedquery,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up| +|85|lens.server.servicenames|session,query,savedquery,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up| *--+--+---+--+ -|84|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs| +|86|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs| *--+--+---+--+ -|85|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service| +|87|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service| *--+--+---+--+ -|86|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.| +|88|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.| *--+--+---+--+ -|87|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource| +|89|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource| *--+--+---+--+ -|88|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.| +|90|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.| *--+--+---+--+ -|89|lens.server.state.persistence.enabled|true|If flag is enabled, state of all the services will be persisted periodically to a location specified by lens.server.persist.location and on server restart all the services will be started from last saved state.| +|91|lens.server.state.persistence.enabled|true|If flag is enabled, state of all the services will be persisted periodically to a location specified by lens.server.persist.location and on server restart all the services will be started from last saved state.| *--+--+---+--+ -|90|lens.server.state.persistence.interval.millis|300000|Lens server state persistence time interval in milliseconds| +|92|lens.server.state.persistence.interval.millis|300000|Lens server state persistence time interval in milliseconds| *--+--+---+--+ -|91|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.| +|93|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.| *--+--+---+--+ -|92|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.| +|94|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.| *--+--+---+--+ -|93|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.| +|95|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.| *--+--+---+--+ -|94|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.| +|96|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.| *--+--+---+--+ -|95|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| +|97|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| *--+--+---+--+ -|96|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| +|98|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| *--+--+---+--+ -|97|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.| +|99|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.| *--+--+---+--+ -|98|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.| +|100|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.| *--+--+---+--+ -|99|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| +|101|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| *--+--+---+--+ -|100|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| +|102|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| *--+--+---+--+ -|101|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| +|103|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| *--+--+---+--+ -|102|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| +|104|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| *--+--+---+--+ -|103|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| +|105|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| *--+--+---+--+ -|104|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| +|106|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| *--+--+---+--+ -|105|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| +|107|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| *--+--+---+--+ -|106|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| +|108|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| *--+--+---+--+ -|107|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| +|109|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| *--+--+---+--+ -|108|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| +|110|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| *--+--+---+--+ -|109|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| +|111|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| *--+--+---+--+ -|110|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| +|112|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| *--+--+---+--+ -|111|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| +|113|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| *--+--+---+--+ -|112|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| +|114|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| *--+--+---+--+ -|113|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| +|115|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| *--+--+---+--+ -|114|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| +|116|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| *--+--+---+--+ -|115|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| +|117|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| *--+--+---+--+ -|116|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| +|118|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| *--+--+---+--+ -|117|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| +|119|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| *--+--+---+--+ -|118|lens.server.ws.resourcenames|session,metastore,query,savedquery,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| +|120|lens.server.ws.resourcenames|session,metastore,query,savedquery,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| *--+--+---+--+ The configuration parameters and their default values http://git-wip-us.apache.org/repos/asf/lens/blob/d7f7a2ee/src/site/apt/admin/session-config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/session-config.apt b/src/site/apt/admin/session-config.apt index fc6916f..c4b3c04 100644 --- a/src/site/apt/admin/session-config.apt +++ b/src/site/apt/admin/session-config.apt @@ -60,50 +60,56 @@ Lens session configuration *--+--+---+--+ |18|lens.query.hdfs.output.path|hdfsout|The directory under the parent result directory, in which HiveDriver will persist the results, if persisting by driver is enabled. This directory should exist and should have world writable permissions sothat all users will be able put query outputs here.| *--+--+---+--+ -|19|lens.query.output.charset.encoding|UTF-8|The charset encoding for formatting query result. It supports all the encodings supported by java.io.OutputStreamWriter.| +|19|lens.query.http.notification.mediatype|application/json|This is the media type for Query Http notifications. Accepted types are "application/json" and "application/xml". The default value is "application/json"| *--+--+---+--+ -|20|lens.query.output.compression.codec|org.apache.hadoop.io.compress.GzipCodec|The codec used to compress the query output, if compression is enabled| +|20|lens.query.http.notification.type.FINISHED|false|Setting this property to true will enable query FINISHED notifications which includes SUCCESSFUL, FAILED and CANCELLED queries. The notification will have eventtype = "FINISHED", eventtime = long event time and query = org.apache.lens.api.query.LensQuery instance. The mediatype for eventtype and eventtime will be TEXT/PLAIN and the mediatype for query will be based on property lens.query.http.notification.mediatype. Default value of this property is false.| *--+--+---+--+ -|21|lens.query.output.enable.compression|false|Whether to compress the query result output| +|21|lens.query.http.notification.urls| |These are the http end points for Query http notifications. Users can specify more than one comma separated end points for a query. If this property is not set, no http notification will be sent out by lens server for the query.| *--+--+---+--+ -|22|lens.query.output.file.extn|.csv|The extension name for the persisted query output file. If file is compressed, the extension from compression codec will be appended to this extension.| +|22|lens.query.output.charset.encoding|UTF-8|The charset encoding for formatting query result. It supports all the encodings supported by java.io.OutputStreamWriter.| *--+--+---+--+ -|23|lens.query.output.footer| |The value of custom footer that should be written, if any. This footer will be added in formatting driver persisted results.| +|23|lens.query.output.compression.codec|org.apache.hadoop.io.compress.GzipCodec|The codec used to compress the query output, if compression is enabled| *--+--+---+--+ -|24|lens.query.output.formatter| |The query result output formatter for the query. If no value is specified, then org.apache.lens.lib.query.FileSerdeFormatter will be used to format in-memory result sets, org.apache.lens.lib.query.FilePersistentFormatter will be used to format driver persisted result sets.| +|24|lens.query.output.enable.compression|false|Whether to compress the query result output| *--+--+---+--+ -|25|lens.query.output.header| |The value of custom header that should be written, if any. If no value column names will be used as header.| +|25|lens.query.output.file.extn|.csv|The extension name for the persisted query output file. If file is compressed, the extension from compression codec will be appended to this extension.| *--+--+---+--+ -|26|lens.query.output.write.footer|false|Whether to write footer as part of query result. When enabled, total number of rows will be written as part of header.| +|26|lens.query.output.footer| |The value of custom footer that should be written, if any. This footer will be added in formatting driver persisted results.| *--+--+---+--+ -|27|lens.query.output.write.header|false|Whether to write header as part of query result formatting. When enabled the user given header will be added in case of driver persisted results, and column names chosen will be added as header for in-memory results.| +|27|lens.query.output.formatter| |The query result output formatter for the query. If no value is specified, then org.apache.lens.lib.query.FileSerdeFormatter will be used to format in-memory result sets, org.apache.lens.lib.query.FilePersistentFormatter will be used to format driver persisted result sets.| *--+--+---+--+ -|28|lens.query.prefetch.inmemory.resultset|true|When set to true, specified number of rows of result set will be pre-fetched if the result set is of type InMemoryResultSet and query execution is not asynchronous i.e. query should be launched with operation as EXECUTE_WITH_TIMEOUT. Suggested usage of this property: It can be used by client to stream as well as persist results in server for queries that finish fast and produce results with fewer rows (should be less than number of rows pre-fetched). Note that the results are streamed to the client early, without waiting for persistence to finish. Default value of this property is true.| +|28|lens.query.output.header| |The value of custom header that should be written, if any. If no value column names will be used as header.| *--+--+---+--+ -|29|lens.query.prefetch.inmemory.resultset.rows|100|Specifies the number of rows to pre-fetch when lens.query.prefetch.inmemory.resultset is set to true. Default value is 100 rows.| +|29|lens.query.output.write.footer|false|Whether to write footer as part of query result. When enabled, total number of rows will be written as part of header.| *--+--+---+--+ -|30|lens.query.result.email.cc| |When query ends, the result/failure reason will be sent to the user via email. The mail would be cc'ed to the addresses provided in this field.| +|30|lens.query.output.write.header|false|Whether to write header as part of query result formatting. When enabled the user given header will be added in case of driver persisted results, and column names chosen will be added as header for in-memory results.| *--+--+---+--+ -|31|lens.query.result.fs.read.url| |Http read URL for FileSystem on which result is present, if available. For example webhdfs as http read url should http://host:port/webhdfs/v1. Currently we support only webhdfs url as the http url for HDFS file system| +|31|lens.query.prefetch.inmemory.resultset|true|When set to true, specified number of rows of result set will be pre-fetched if the result set is of type InMemoryResultSet and query execution is not asynchronous i.e. query should be launched with operation as EXECUTE_WITH_TIMEOUT. Suggested usage of this property: It can be used by client to stream as well as persist results in server for queries that finish fast and produce results with fewer rows (should be less than number of rows pre-fetched). Note that the results are streamed to the client early, without waiting for persistence to finish. Default value of this property is true.| *--+--+---+--+ -|32|lens.query.result.output.dir.format| |The format of the output if result is persisted in hdfs. The format should be expressed in HQL.| +|32|lens.query.prefetch.inmemory.resultset.rows|100|Specifies the number of rows to pre-fetch when lens.query.prefetch.inmemory.resultset is set to true. Default value is 100 rows.| *--+--+---+--+ -|33|lens.query.result.output.serde|org.apache.lens.lib.query.CSVSerde|The default serde class name that should be used by org.apache.lens.lib.query.FileSerdeFormatter for formatting the output| +|33|lens.query.result.email.cc| |When query ends, the result/failure reason will be sent to the user via email. The mail would be cc'ed to the addresses provided in this field.| *--+--+---+--+ -|34|lens.query.result.parent.dir|file:///tmp/lensreports|The directory for storing persisted result of query. This directory should exist and should have writable permissions by lens server| +|34|lens.query.result.fs.read.url| |Http read URL for FileSystem on which result is present, if available. For example webhdfs as http read url should http://host:port/webhdfs/v1. Currently we support only webhdfs url as the http url for HDFS file system| *--+--+---+--+ -|35|lens.query.result.size.format.threshold|10737418240|The maximum allowed size of the query result. If exceeds, no server side formatting would be done.| +|35|lens.query.result.output.dir.format| |The format of the output if result is persisted in hdfs. The format should be expressed in HQL.| *--+--+---+--+ -|36|lens.query.result.split.multiple|false|Whether to split the result into multiple files. If enabled, each file will be restricted to max rows configured. All the files will be available as zip.| +|36|lens.query.result.output.serde|org.apache.lens.lib.query.CSVSerde|The default serde class name that should be used by org.apache.lens.lib.query.FileSerdeFormatter for formatting the output| *--+--+---+--+ -|37|lens.query.result.split.multiple.maxrows|100000|The maximum number of rows allowed in each file, when splitting the result into multiple files is enabled.| +|37|lens.query.result.parent.dir|file:///tmp/lensreports|The directory for storing persisted result of query. This directory should exist and should have writable permissions by lens server| *--+--+---+--+ -|38|lens.session.aux.jars| |List of comma separated jar paths, which will added to the session| +|38|lens.query.result.size.format.threshold|10737418240|The maximum allowed size of the query result. If exceeds, no server side formatting would be done.| *--+--+---+--+ -|39|lens.session.cluster.user| |Session level config which will determine which cluster user will access hdfs| +|39|lens.query.result.split.multiple|false|Whether to split the result into multiple files. If enabled, each file will be restricted to max rows configured. All the files will be available as zip.| *--+--+---+--+ -|40|lens.session.loggedin.user| |The username used to log in to lens. e.g. LDAP user| +|40|lens.query.result.split.multiple.maxrows|100000|The maximum number of rows allowed in each file, when splitting the result into multiple files is enabled.| *--+--+---+--+ -|41|lens.session.metastore.exclude.cubetables.from.nativetables|true|Exclude cube related tables when fetching native tables| +|41|lens.session.aux.jars| |List of comma separated jar paths, which will added to the session| +*--+--+---+--+ +|42|lens.session.cluster.user| |Session level config which will determine which cluster user will access hdfs| +*--+--+---+--+ +|43|lens.session.loggedin.user| |The username used to log in to lens. e.g. LDAP user| +*--+--+---+--+ +|44|lens.session.metastore.exclude.cubetables.from.nativetables|true|Exclude cube related tables when fetching native tables| *--+--+---+--+ The configuration parameters and their default values
