Repository: lens Updated Branches: refs/heads/master 4e822031f -> d264c9a44
LENS-1029 : Fix exception handling in lens server snapshotter Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d264c9a4 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d264c9a4 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d264c9a4 Branch: refs/heads/master Commit: d264c9a44b3f0b889e87241a6eb9ec1051f7c6a5 Parents: 4e82203 Author: Puneet Gupta <puneet.k.gu...@gmail.com> Authored: Mon May 9 16:43:42 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Mon May 9 16:43:42 2016 +0530 ---------------------------------------------------------------------- .../lens/lib/query/AbstractFileFormatter.java | 12 +- .../lens/server/api/LensConfConstants.java | 27 +-- .../org/apache/lens/server/LensServices.java | 237 ++++++++++--------- .../src/main/resources/lensserver-default.xml | 20 +- .../apache/lens/server/TestServerRestart.java | 16 +- src/site/apt/admin/config.apt | 106 ++++----- src/site/apt/admin/deployment.apt | 10 +- 7 files changed, 220 insertions(+), 208 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/d264c9a4/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java ---------------------------------------------------------------------- diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java index 8c06621..2cd6b3f 100644 --- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java +++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java @@ -18,9 +18,7 @@ */ package org.apache.lens.lib.query; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; +import java.io.*; import org.apache.lens.server.api.driver.LensResultSetMetadata; import org.apache.lens.server.api.query.QueryContext; @@ -97,15 +95,15 @@ public abstract class AbstractFileFormatter extends AbstractOutputFormatter impl public void writeExternal(ObjectOutput out) throws IOException { out.writeUTF(metadata.toJson()); out.writeUTF(finalPath.toString()); - out.writeInt(numRows); - out.writeLong(fileSize); + out.writeObject(numRows); // writeObject and readObject have handling for null objects + out.writeObject(fileSize); } @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { metadata = LensResultSetMetadata.fromJson(in.readUTF()); finalPath = new Path(in.readUTF()); - numRows = in.readInt(); - fileSize = in.readLong(); + numRows = (Integer)in.readObject(); + fileSize = (Long)in.readObject(); } } http://git-wip-us.apache.org/repos/asf/lens/blob/d264c9a4/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 23537cb..7cf6449 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -141,14 +141,14 @@ public final class LensConfConstants { public static final String DEFAULT_SERVER_BASE_URL = "http://0.0.0.0:9999/lensapi";; /** - * The Constant SERVER_RESTART_ENABLED. + * The Constant SERVER_STATE_PERSISTENCE_ENABLED. */ - public static final String SERVER_RESTART_ENABLED = SERVER_PFX + "restart.enabled"; + public static final String SERVER_STATE_PERSISTENCE_ENABLED = SERVER_PFX + "state.persistence.enabled"; /** - * The Constant DEFAULT_SERVER_RESTART_ENABLED. + * The Constant DEFAULT_SERVER_STATE_PERSISTENCE_ENABLED. */ - public static final boolean DEFAULT_SERVER_RESTART_ENABLED = true; + public static final boolean DEFAULT_SERVER_STATE_PERSISTENCE_ENABLED = true; /** * The Constant SERVER_STATE_PERSIST_LOCATION. @@ -161,16 +161,6 @@ public final class LensConfConstants { public static final String DEFAULT_SERVER_STATE_PERSIST_LOCATION = "file:///tmp/lensserver"; /** - * The Constant SERVER_RECOVER_ON_RESTART. - */ - public static final String SERVER_RECOVER_ON_RESTART = SERVER_PFX + "recover.onrestart"; - - /** - * The Constant DEFAULT_SERVER_RECOVER_ON_RESTART. - */ - public static final boolean DEFAULT_SERVER_RECOVER_ON_RESTART = true; - - /** * The Constant SESSION_TIMEOUT_SECONDS. */ public static final String SESSION_TIMEOUT_SECONDS = SERVER_PFX + "session.timeout.seconds"; @@ -221,14 +211,15 @@ public final class LensConfConstants { public static final boolean DEFAULT_SERVER_UI_ENABLE_CACHING = true; /** - * The Constant SERVER_SNAPSHOT_INTERVAL. + * The Constant SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS. */ - public static final String SERVER_SNAPSHOT_INTERVAL = SERVER_PFX + "snapshot.interval"; + public static final String SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS = + SERVER_PFX + "state.persistence.interval.millis"; /** - * The Constant DEFAULT_SERVER_SNAPSHOT_INTERVAL. + * The Constant DEFAULT_SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS. */ - public static final long DEFAULT_SERVER_SNAPSHOT_INTERVAL = 5 * 60 * 1000; + public static final long DEFAULT_SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS = 5 * 60 * 1000; // Email related configurations /** http://git-wip-us.apache.org/repos/asf/lens/blob/d264c9a4/lens-server/src/main/java/org/apache/lens/server/LensServices.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java index 48b3e00..a2d037a 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java @@ -23,6 +23,7 @@ import static org.apache.lens.server.api.LensConfConstants.*; import java.io.*; import java.lang.reflect.Constructor; import java.util.*; +import java.util.concurrent.*; import org.apache.lens.api.error.ErrorCollection; import org.apache.lens.api.error.ErrorCollectionFactory; @@ -37,6 +38,7 @@ import org.apache.lens.server.stats.StatisticsService; import org.apache.lens.server.user.UserConfigLoaderFactory; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -88,9 +90,6 @@ public class LensServices extends CompositeService implements ServiceProvider { /** The stopping. */ private boolean stopping = false; - /** The snap shot interval. */ - private long snapShotInterval; - /** * The metrics service. */ @@ -106,8 +105,8 @@ public class LensServices extends CompositeService implements ServiceProvider { @Setter private SERVICE_MODE serviceMode; - /** The timer. */ - private Timer timer; + /** Scheduled Executor which persists the server state periodically*/ + private ScheduledExecutorService serverSnapshotScheduler; /* Lock for synchronizing persistence of LensServices state */ private final Object statePersistenceLock = new Object(); @@ -115,6 +114,11 @@ public class LensServices extends CompositeService implements ServiceProvider { @Getter private ErrorCollection errorCollection; + private boolean isServerStatePersistenceEnabled; + + private long serverStatePersistenceInterval; + + @Getter private final LogSegregationContext logSegregationContext; @@ -235,27 +239,30 @@ public class LensServices extends CompositeService implements ServiceProvider { super.init(conf); // setup persisted state - String persistPathStr = conf.get(SERVER_STATE_PERSIST_LOCATION, - DEFAULT_SERVER_STATE_PERSIST_LOCATION); - persistDir = new Path(persistPathStr); - try { - Configuration configuration = new Configuration(conf); - configuration.setBoolean(FS_AUTOMATIC_CLOSE, false); - - int outStreamBufferSize = conf.getInt(STATE_PERSIST_OUT_STREAM_BUFF_SIZE, - DEFAULT_STATE_PERSIST_OUT_STREAM_BUFF_SIZE); - configuration.setInt(FS_IO_FILE_BUFFER_SIZE, outStreamBufferSize); - log.info("STATE_PERSIST_OUT_STREAM_BUFF_SIZE IN BYTES:{}", outStreamBufferSize); - persistenceFS = FileSystem.newInstance(persistDir.toUri(), configuration); - setupPersistedState(); - } catch (Exception e) { - log.error("Could not recover from persisted state", e); - throw new RuntimeException("Could not recover from persisted state", e); + isServerStatePersistenceEnabled = conf.getBoolean(SERVER_STATE_PERSISTENCE_ENABLED, + DEFAULT_SERVER_STATE_PERSISTENCE_ENABLED); + if (isServerStatePersistenceEnabled) { + String persistPathStr = conf.get(SERVER_STATE_PERSIST_LOCATION, + DEFAULT_SERVER_STATE_PERSIST_LOCATION); + persistDir = new Path(persistPathStr); + try { + Configuration configuration = new Configuration(conf); + configuration.setBoolean(FS_AUTOMATIC_CLOSE, false); + + int outStreamBufferSize = conf.getInt(STATE_PERSIST_OUT_STREAM_BUFF_SIZE, + DEFAULT_STATE_PERSIST_OUT_STREAM_BUFF_SIZE); + configuration.setInt(FS_IO_FILE_BUFFER_SIZE, outStreamBufferSize); + log.info("STATE_PERSIST_OUT_STREAM_BUFF_SIZE IN BYTES:{}", outStreamBufferSize); + persistenceFS = FileSystem.newInstance(persistDir.toUri(), configuration); + setupPersistedState(); + } catch (Exception e) { + log.error("Could not recover from persisted state", e); + throw new RuntimeException("Could not recover from persisted state", e); + } + serverStatePersistenceInterval = conf.getLong(SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS, + DEFAULT_SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS); } - snapShotInterval = conf.getLong(SERVER_SNAPSHOT_INTERVAL, - DEFAULT_SERVER_SNAPSHOT_INTERVAL); log.info("Initialized services: {}", services.keySet().toString()); - timer = new Timer("lens-server-snapshotter", true); } } @@ -268,20 +275,33 @@ public class LensServices extends CompositeService implements ServiceProvider { if (getServiceState() != STATE.STARTED) { super.start(); } - timer.schedule(new TimerTask() { - @Override - public void run() { - try { - final String runId = UUID.randomUUID().toString(); - logSegregationContext.setLogSegregationId(runId); - persistLensServiceState(); - log.info("SnapShot of Lens Services created"); - } catch (IOException e) { - incrCounter(SERVER_STATE_PERSISTENCE_ERRORS); - log.warn("Unable to persist lens server state", e); + + if (!isServerStatePersistenceEnabled) { + log.info("Server restart is not enabled. Not persisting lens server state"); + } else { + ThreadFactory factory = new BasicThreadFactory.Builder() + .namingPattern("Lens-server-snapshotter-Thread-%d") + .daemon(true) + .priority(Thread.NORM_PRIORITY) + .build(); + serverSnapshotScheduler = Executors.newSingleThreadScheduledExecutor(factory); + serverSnapshotScheduler.scheduleWithFixedDelay(new Runnable() { + @Override + public void run() { + try { + final String runId = UUID.randomUUID().toString(); + logSegregationContext.setLogSegregationId(runId); + persistLensServiceState(); + log.info("SnapShot of Lens Services created"); + } catch (Exception e) { + incrCounter(SERVER_STATE_PERSISTENCE_ERRORS); + log.error("Unable to persist lens server state", e); + } } - } - }, snapShotInterval, snapShotInterval); + }, serverStatePersistenceInterval, serverStatePersistenceInterval, TimeUnit.MILLISECONDS); + log.info("Enabled periodic persistence of lens server state at {} millis interval", + serverStatePersistenceInterval); + } } /** @@ -291,24 +311,21 @@ public class LensServices extends CompositeService implements ServiceProvider { * @throws ClassNotFoundException the class not found exception */ private void setupPersistedState() throws IOException, ClassNotFoundException { - if (conf.getBoolean(SERVER_RECOVER_ON_RESTART, - DEFAULT_SERVER_RECOVER_ON_RESTART)) { - - for (BaseLensService service : lensServices) { - ObjectInputStream in = null; + for (BaseLensService service : lensServices) { + ObjectInputStream in = null; + Path path = getServicePersistPath(service); + try { try { - try { - in = new ObjectInputStream(persistenceFS.open(getServicePersistPath(service))); - } catch (FileNotFoundException fe) { - log.warn("No persist path available for service:{}", service.getName()); - continue; - } - service.readExternal(in); - log.info("Recovered service {} from persisted state", service.getName()); - } finally { - if (in != null) { - in.close(); - } + in = new ObjectInputStream(persistenceFS.open(path)); + } catch (FileNotFoundException fe) { + log.warn("Persisted state not available for service: {} at: {}", service.getName(), path); + continue; + } + service.readExternal(in); + log.info("Recovered service {} from persisted state {}", service.getName(), path); + } finally { + if (in != null) { + in.close(); } } } @@ -316,51 +333,47 @@ public class LensServices extends CompositeService implements ServiceProvider { /** * Persist lens service state. - * - * @throws IOException Signals that an I/O exception has occurred. */ - private void persistLensServiceState() throws IOException { - + private void persistLensServiceState() { synchronized (statePersistenceLock) { - if (conf.getBoolean(SERVER_RESTART_ENABLED, DEFAULT_SERVER_RESTART_ENABLED)) { - if (persistDir != null) { - log.info("Persisting server state in {}", persistDir); - - long now = System.currentTimeMillis(); - - for (BaseLensService service : lensServices) { - log.info("Persisting state of service: {}", service.getName()); - Path serviceWritePath = new Path(persistDir, service.getName() + ".out" + "." + now); - ObjectOutputStream out = null; - try { - out = new ObjectOutputStream(persistenceFS.create(serviceWritePath)); - service.writeExternal(out); - } finally { - if (out != null) { - out.close(); - } - } - Path servicePath = getServicePersistPath(service); - if (persistenceFS.exists(servicePath)) { - // delete the destination first, because rename is no-op in HDFS, if destination exists - if (!persistenceFS.delete(servicePath, true)) { - log.error("Failed to delete [{}]", servicePath); - } - } - if (!persistenceFS.rename(serviceWritePath, servicePath)) { - incrCounter(SERVER_STATE_PERSISTENCE_ERRORS); - log.error("Failed to persist {} to [{}]", service.getName(), servicePath); - } else { - log.info("Persisted service {} to [{}]", service.getName(), servicePath); - } - } - } else { - log.info("Server restart is not enabled. Not persisting the server state"); + log.info("Persisting server state in {}", persistDir); + String now = "" + System.currentTimeMillis(); + for (BaseLensService service : lensServices) { + try { + persistState(service, now); + } catch (Exception e) { + incrCounter(SERVER_STATE_PERSISTENCE_ERRORS); + log.error("Error while persisting state for service {}", service.getName(), e); } } } } + private void persistState(BaseLensService service, String time) throws IOException { + log.info("Persisting state of service: {}", service.getName()); + Path serviceWritePath = new Path(persistDir, service.getName() + ".out" + "." + time); + ObjectOutputStream out = null; + try { + out = new ObjectOutputStream(persistenceFS.create(serviceWritePath)); + service.writeExternal(out); + } finally { + if (out != null) { + out.close(); + } + } + Path servicePath = getServicePersistPath(service); + if (persistenceFS.exists(servicePath)) { + // delete the destination first, because rename is no-op in HDFS, if destination exists + if (!persistenceFS.delete(servicePath, true)) { + throw new IOException("Failed to delete " + servicePath); + } + } + if (!persistenceFS.rename(serviceWritePath, servicePath)) { + throw new IOException("Failed to rename " + serviceWritePath + " to " + servicePath); + } + log.info("Persisted service {} to [{}]", service.getName(), servicePath); + } + /** * Gets the service persist path. * @@ -384,23 +397,33 @@ public class LensServices extends CompositeService implements ServiceProvider { service.prepareStopping(); } - if (timer != null) { - timer.cancel(); - } + if (isServerStatePersistenceEnabled) { + try { + //1. shutdown serverSnapshotScheduler gracefully by allowing already triggered task (if any) to finish + serverSnapshotScheduler.shutdown(); + try { //Wait for shutdown. Shutdown should be immediate in case no task is running at this point + while (!serverSnapshotScheduler.awaitTermination(1, TimeUnit.MINUTES)) { + log.info("Waiting for Lens-server-snapshotter to shutdown gracefully..."); + } + } catch (InterruptedException e) { + log.error("Lens-server-snapshotter interrupted while shutting down" , e); + } + log.info("Lens-server-snapshotter was shutdown"); - try { - // persist all the services - persistLensServiceState(); - - persistenceFS.close(); - log.info("Persistence File system object close complete"); - } catch (IOException e) { - incrCounter(SERVER_STATE_PERSISTENCE_ERRORS); - log.error("Could not persist server state", e); - throw new IllegalStateException(e); - } finally { - super.stop(); + //2. persist the latest state of all the services + persistLensServiceState(); + }finally { + try { + persistenceFS.close(); + log.info("Persistence File system object close complete"); + } catch (Exception e) { + log.error("Error while closing Persistence File system object", e); + } + } } + + super.stop(); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/d264c9a4/lens-server/src/main/resources/lensserver-default.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml index 1a15658..9580397 100644 --- a/lens-server/src/main/resources/lensserver-default.xml +++ b/lens-server/src/main/resources/lensserver-default.xml @@ -209,19 +209,18 @@ </property> <property> - <name>lens.server.restart.enabled</name> + <name>lens.server.state.persistence.enabled</name> <value>true</value> - <description>If flag is enabled, all the services will be persisted to persistent - location passed. + <description>If flag is enabled, state of all the services will be persisted periodically to a location + specified by lens.server.persist.location and on server restart all the services will be started + from last saved state. </description> </property> <property> - <name>lens.server.recover.onrestart</name> - <value>true</value> - <description>If the flag is enabled, all the services will be started from last - saved state, if disabled all the services will start afresh - </description> + <name>lens.server.state.persistence.interval.millis</name> + <value>300000</value> + <description>Lens server state persistence time interval in milliseconds</description> </property> <property> @@ -359,11 +358,6 @@ {{{http://stackoverflow.com/questions/10684244/dbcp-validationquery-for-different-databases}validationquery for different databases}} \ </description> </property> - <property> - <name>lens.server.snapshot.interval</name> - <value>300000</value> - <description>Snapshot interval time in miliseconds for saving lens server state.</description> - </property> <property> <name>lens.server.mail.from.address</name> http://git-wip-us.apache.org/repos/asf/lens/blob/d264c9a4/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java index 1fa61ef..c610ae4 100644 --- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java +++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java @@ -50,6 +50,7 @@ import org.apache.lens.server.session.HiveSessionService; import org.apache.lens.server.session.LensSessionImpl; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hive.service.Service; import org.glassfish.jersey.media.multipart.FormDataBodyPart; @@ -88,11 +89,18 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { super.setUp(); } + @Override + public HiveConf getServerConf() { + HiveConf conf = super.getServerConf(); + conf.set("lens.server.state.persistence.interval.millis", "1000"); + return conf; + } + /* - * (non-Javadoc) - * - * @see org.glassfish.jersey.test.JerseyTest#tearDown() - */ + * (non-Javadoc) + * + * @see org.glassfish.jersey.test.JerseyTest#tearDown() + */ @AfterTest public void tearDown() throws Exception { super.tearDown(); http://git-wip-us.apache.org/repos/asf/lens/blob/d264c9a4/src/site/apt/admin/config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt index 5466e7a..4bf1e7c 100644 --- a/src/site/apt/admin/config.apt +++ b/src/site/apt/admin/config.apt @@ -155,110 +155,108 @@ Lens server configuration *--+--+---+--+ |63|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource| *--+--+---+--+ -|64|lens.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh| +|64|lens.server.resultset.purge.enabled|false|Whether to purge the query results| *--+--+---+--+ -|65|lens.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed.| +|65|lens.server.resultsetpurger.sleep.interval.secs|3600|Periodicity for Query result purger runs. Default 1 hour.| *--+--+---+--+ -|66|lens.server.resultset.purge.enabled|false|Whether to purge the query results| +|66|lens.server.savedquery.jdbc.dialectclass|org.apache.lens.server.query.save.SavedQueryDao$HSQLDialect|Dialect of the target DB, Default is HSQL. Override with the target DB used.| *--+--+---+--+ -|67|lens.server.resultsetpurger.sleep.interval.secs|3600|Periodicity for Query result purger runs. Default 1 hour.| +|67|lens.server.savedquery.list.default.count|20|Key denoting the default fetch value of saved query list api.| *--+--+---+--+ -|68|lens.server.savedquery.jdbc.dialectclass|org.apache.lens.server.query.save.SavedQueryDao$HSQLDialect|Dialect of the target DB, Default is HSQL. Override with the target DB used.| +|68|lens.server.savedquery.list.default.offset|0|Key denoting the default start value of saved query list api.| *--+--+---+--+ -|69|lens.server.savedquery.list.default.count|20|Key denoting the default fetch value of saved query list api.| +|69|lens.server.savedquery.service.impl|org.apache.lens.server.query.save.SavedQueryServiceImpl|Implementation class for saved query service| *--+--+---+--+ -|70|lens.server.savedquery.list.default.offset|0|Key denoting the default start value of saved query list api.| +|70|lens.server.savedquery.ws.resource.impl|org.apache.lens.server.query.save.SavedQueryResource|Implementation class for Saved query Resource| *--+--+---+--+ -|71|lens.server.savedquery.service.impl|org.apache.lens.server.query.save.SavedQueryServiceImpl|Implementation class for saved query service| +|71|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service| *--+--+---+--+ -|72|lens.server.savedquery.ws.resource.impl|org.apache.lens.server.query.save.SavedQueryResource|Implementation class for Saved query Resource| +|72|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource| *--+--+---+--+ -|73|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service| +|73|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.| *--+--+---+--+ -|74|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource| +|74|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter| *--+--+---+--+ -|75|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.| +|75|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.| *--+--+---+--+ -|76|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter| +|76|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up| *--+--+---+--+ -|77|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.| +|77|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs| *--+--+---+--+ -|78|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up| +|78|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service| *--+--+---+--+ -|79|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs| +|79|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.| *--+--+---+--+ -|80|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service| +|80|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource| *--+--+---+--+ -|81|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.| +|81|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.| *--+--+---+--+ -|82|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource| +|82|lens.server.state.persistence.enabled|true|If flag is enabled, state of all the services will be persisted periodically to a location specified by lens.server.persist.location and on server restart all the services will be started from last saved state.| *--+--+---+--+ -|83|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.| +|83|lens.server.state.persistence.interval.millis|300000|Lens server state persistence time interval in milliseconds| *--+--+---+--+ -|84|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.| +|84|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.| *--+--+---+--+ -|85|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.| +|85|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.| *--+--+---+--+ -|86|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.| +|86|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.| *--+--+---+--+ -|87|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.| +|87|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.| *--+--+---+--+ -|88|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.| +|88|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| *--+--+---+--+ -|89|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| +|89|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| *--+--+---+--+ -|90|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| +|90|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.| *--+--+---+--+ -|91|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.| +|91|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.| *--+--+---+--+ -|92|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.| +|92|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server| *--+--+---+--+ -|93|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server| +|93|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.| *--+--+---+--+ -|94|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.| +|94|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server| *--+--+---+--+ -|95|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server| +|95|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from| *--+--+---+--+ -|96|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from| +|96|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| *--+--+---+--+ -|97|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| +|97|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| *--+--+---+--+ -|98|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| +|98|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| *--+--+---+--+ -|99|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| +|99|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| *--+--+---+--+ -|100|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| +|100|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| *--+--+---+--+ -|101|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| +|101|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| *--+--+---+--+ -|102|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| +|102|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| *--+--+---+--+ -|103|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| +|103|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| *--+--+---+--+ -|104|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| +|104|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| *--+--+---+--+ -|105|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| +|105|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| *--+--+---+--+ -|106|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| +|106|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| *--+--+---+--+ -|107|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| +|107|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| *--+--+---+--+ -|108|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| +|108|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| *--+--+---+--+ -|109|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| +|109|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| *--+--+---+--+ -|110|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| +|110|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| *--+--+---+--+ -|111|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| +|111|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| *--+--+---+--+ -|112|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| +|112|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| *--+--+---+--+ -|113|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| +|113|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| *--+--+---+--+ -|114|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| +|114|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| *--+--+---+--+ -|115|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| -*--+--+---+--+ -|116|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| +|115|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| *--+--+---+--+ The configuration parameters and their default values http://git-wip-us.apache.org/repos/asf/lens/blob/d264c9a4/src/site/apt/admin/deployment.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/deployment.apt b/src/site/apt/admin/deployment.apt index b4f4d0a..0e1d775 100644 --- a/src/site/apt/admin/deployment.apt +++ b/src/site/apt/admin/deployment.apt @@ -47,17 +47,17 @@ Lens server deployment * Restart and recovery Lens server should not have problems with Hive Metastore server, Hive Server2 or Hadoop restarts. Lens server can - persist its state before going and recover from previous state upon restart. + persist its state before going down and recover from previous state upon restart. * Recoverability - ** To enable lens server with recoverability, i.e. to allow users access queries before restart, lens server should be - started with lens.server.restart.enabled set to true. Admin can set this to false if no recoverability is required. + * To enable lens server with recoverability, i.e. to allow users access queries after restart, lens server should be + started with lens.server.state.persistence.enabled set to true. Admin can set this to false if no recoverability is + required. * Restarting - ** To start the lens server a fresh lens.server.recover.onrestart should be disabled. If it is true, the server will - load its persisted state before restart. + * If lens server recoverability is enabled, the server will load its persisted state on restart. * Server modes
