Repository: lens Updated Branches: refs/heads/master 6995962fd -> 9b7541bcb
LENS-1013 : Add postDriverSelection to DriverQueryHook Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9b7541bc Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9b7541bc Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9b7541bc Branch: refs/heads/master Commit: 9b7541bcbcc33faf3cbde71452f334783795ea27 Parents: 6995962 Author: Puneet Gupta <puneet.k.gu...@gmail.com> Authored: Thu Apr 28 09:36:38 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Thu Apr 28 09:36:38 2016 +0530 ---------------------------------------------------------------------- .../org/apache/lens/driver/hive/HiveDriver.java | 6 ++- .../apache/lens/driver/hive/TestHiveDriver.java | 3 -- .../lens/driver/hive/TestRemoteHiveDriver.java | 3 -- .../org/apache/lens/driver/jdbc/JDBCDriver.java | 6 ++- .../apache/lens/driver/jdbc/TestJdbcDriver.java | 3 -- .../server/api/driver/AbstractLensDriver.java | 7 +++ .../lens/server/api/driver/DriverQueryHook.java | 39 ++++++++------ .../lens/server/api/driver/LensDriver.java | 6 +++ .../server/api/driver/NoOpDriverQueryHook.java | 14 +++-- .../server/api/user/MockDriverQueryHook.java | 34 +++++++++--- .../server/query/QueryExecutionServiceImpl.java | 18 ++++--- .../org/apache/lens/server/LensJerseyTest.java | 11 +++- .../apache/lens/server/TestServerRestart.java | 55 +++++++++++++++++--- .../lens/server/query/TestQueryService.java | 7 ++- .../drivers/hive/hive1/hivedriver-site.xml | 7 +++ .../drivers/hive/hive2/hivedriver-site.xml | 7 +++ 16 files changed, 176 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index 6eec3f7..19a010e 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -559,7 +559,6 @@ public class HiveDriver extends AbstractLensDriver { Configuration qdconf = ctx.getDriverConf(this); qdconf.set("mapred.job.name", ctx.getQueryHandle().toString()); decidePriority(ctx); - queryHook.preLaunch(ctx); SessionHandle sessionHandle = getSession(ctx); OperationHandle op = getClient().executeStatementAsync(sessionHandle, ctx.getSelectedDriverQuery(), qdconf.getValByRegex(".*")); @@ -1370,4 +1369,9 @@ public class HiveDriver extends AbstractLensDriver { public boolean hasLensSession(LensSessionHandle session) { return lensToHiveSession.containsKey(session.getPublicId().toString()); } + + @Override + public DriverQueryHook getQueryHook() { + return queryHook; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java index 3f77b0c..cf51909 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java @@ -39,7 +39,6 @@ import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.lens.server.api.query.priority.CostRangePriorityDecider; import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf; -import org.apache.lens.server.api.user.MockDriverQueryHook; import org.apache.lens.server.api.util.LensUtil; import org.apache.hadoop.conf.Configuration; @@ -121,7 +120,6 @@ public class TestHiveDriver { conf = new HiveConf(); conf.addResource("drivers/hive/hive1/hivedriver-site.xml"); conf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class); - conf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class); conf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true); driver = new HiveDriver(); @@ -484,7 +482,6 @@ public class TestHiveDriver { */ protected void validateExecuteAsync(QueryContext ctx, DriverQueryState finalState, boolean isPersistent, boolean formatNulls) throws Exception { - assertEquals(ctx.getSelectedDriverConf().get(MockDriverQueryHook.KEY), MockDriverQueryHook.VALUE); validateExecuteAsync(ctx, finalState, isPersistent, formatNulls, driver); } http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java index 4f18c24..765bb45 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java @@ -28,13 +28,11 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.server.api.LensConfConstants; -import org.apache.lens.server.api.driver.DriverQueryHook; import org.apache.lens.server.api.driver.DriverQueryPlan; import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState; import org.apache.lens.server.api.driver.LensDriver; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.QueryContext; -import org.apache.lens.server.api.user.MockDriverQueryHook; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; @@ -136,7 +134,6 @@ public class TestRemoteHiveDriver extends TestHiveDriver { conf.addResource("drivers/hive/hive1/hivedriver-site.xml"); driver = new HiveDriver(); conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true); - conf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class); driver.configure(conf, "hive", "hive1"); drivers = Lists.<LensDriver>newArrayList(driver); System.out.println("TestRemoteHiveDriver created"); http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index eef4464..044a19c 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -935,7 +935,6 @@ public class JDBCDriver extends AbstractLensDriver { String rewrittenQuery = rewriteQuery(context); JdbcQueryContext jdbcCtx = new JdbcQueryContext(context, logSegregationContext); jdbcCtx.setRewrittenQuery(rewrittenQuery); - queryHook.preLaunch(context); try { Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx, logSegregationContext)); jdbcCtx.setResultFuture(future); @@ -1145,4 +1144,9 @@ public class JDBCDriver extends AbstractLensDriver { public void writeExternal(ObjectOutput arg0) throws IOException { // TODO Auto-generated method stub } + + @Override + public DriverQueryHook getQueryHook() { + return queryHook; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java index 81a9552..67f6c1f 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java @@ -40,7 +40,6 @@ import org.apache.lens.server.api.query.ExplainQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.query.cost.QueryCost; -import org.apache.lens.server.api.user.MockDriverQueryHook; import org.apache.lens.server.api.util.LensUtil; import org.apache.hadoop.conf.Configuration; @@ -86,7 +85,6 @@ public class TestJdbcDriver { baseConf.set(JDBC_USER, "SA"); baseConf.set(JDBC_PASSWORD, ""); baseConf.set(JDBC_EXPLAIN_KEYWORD_PARAM, "explain plan for "); - baseConf.setClass(JDBC_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class); hConf = new HiveConf(baseConf, this.getClass()); driver = new JDBCDriver(); @@ -746,7 +744,6 @@ public class TestJdbcDriver { private void executeAsync(QueryContext ctx) throws LensException { driver.executeAsync(ctx); - assertEquals(ctx.getSelectedDriverConf().get(MockDriverQueryHook.KEY), MockDriverQueryHook.VALUE); } /** http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java index 883ad9d..959a5b2 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java @@ -47,6 +47,8 @@ public abstract class AbstractLensDriver implements LensDriver { @Getter private String fullyQualifiedName = null; + private DriverQueryHook noOpDriverQueryHook = new NoOpDriverQueryHook(); + @Override public void configure(Configuration conf, String driverType, String driverName) throws LensException { if (StringUtils.isBlank(driverType) || StringUtils.isBlank(driverName)) { @@ -103,6 +105,11 @@ public abstract class AbstractLensDriver implements LensDriver { } @Override + public DriverQueryHook getQueryHook() { + return noOpDriverQueryHook; + } + + @Override public String toString() { return getFullyQualifiedName(); } http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java index ecac6be..da125bd 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java @@ -23,30 +23,39 @@ package org.apache.lens.server.api.driver; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.QueryContext; /** - * Drivers can choose to initialize a DriverQueryHook object in their - * initialization and use that wherever they want. + * Drivers should initialize a DriverQueryHook object in their initialization and expose it + * via {@link LensDriver#getQueryHook}. Lens Server will invoke the driver hook at relevant points during + * query execution. By default each driver exposes a {@link NoOpDriverQueryHook} which does nothing when invoked. * - * The only use case I see right now is to provide a hook just before query is - * launched on the driver. + * The only use case I see right now is to provide a hook just after driver has been selected for a query and + * before query is launched on the driver. One example usage for hive driver would be to add dynamic configuration or + * stall execution of a query by looking at the final translated query itself (based on table involved, filters + * involved, etc in the query). * - * This interface is meant to unify drivers' needs of having hooks. Each driver - * can use the methods in their own way. Each driver can pose its own restrictions - * or guidelines on methods for its hooks. - * e.g. some driver may choose to not allow any hooks - * another driver may allow hooks but for restricted usage - * Some drivers may want their hooks to be initialized with some constructor params - * Currently, Hivedriver and Jdbcdriver only require their hook implementations to have a default constructor - * - * This interface is expected to evolve for some time as more needs of hooks are discovered + * This interface is expected to evolve for some time as more needs for hook are discovered * + */ public interface DriverQueryHook { /** - * Should be Called before launch on the driver + * Called just before launching the query on the selected driver. + * @param ctx + * @throws LensException + */ + void preLaunch(QueryContext ctx) throws LensException; + + /** + * Called just after driver has been selected to execute a query. + * + * Note: Note if this method updates any configuration, same should be reflected in QueryContext + * via {@link AbstractQueryContext#updateConf(Map)} to ensure the modified configration is persisted and is available + * on server restarts and other bookkeeping needs. + * * @param ctx * @throws LensException */ - void preLaunch(AbstractQueryContext ctx) throws LensException; + void postDriverSelection(AbstractQueryContext ctx) throws LensException; } http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java index 69295d9..95ea360 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java @@ -219,4 +219,10 @@ public interface LensDriver extends Externalizable { * @param queryContext */ Priority decidePriority(AbstractQueryContext queryContext); + + /** + * @return the DriverQueryHook implementation for the driver. + * @see DriverQueryHook for more details. + */ + DriverQueryHook getQueryHook(); } http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java index a5475e4..8cd03cb 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java @@ -21,15 +21,23 @@ */ package org.apache.lens.server.api.driver; +import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.QueryContext; import lombok.extern.slf4j.Slf4j; @Slf4j public class NoOpDriverQueryHook implements DriverQueryHook { @Override - public void preLaunch(AbstractQueryContext ctx) { - log.info("Pre launch for {}, user query: {}, driver query: {}", ctx.getSubmittedUser(), ctx.getUserQuery(), - ctx.getSelectedDriverQuery()); + public void preLaunch(QueryContext ctx) { + log.debug("Pre launch for {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(), + ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery()); + } + + @Override + public void postDriverSelection(AbstractQueryContext ctx) throws LensException { + log.debug("Post driver selection for {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(), + ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery()); } } http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java index 2505ef4..0c4a3f0 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java @@ -18,15 +18,37 @@ */ package org.apache.lens.server.api.user; -import org.apache.lens.server.api.driver.DriverQueryHook; +import java.util.HashMap; + +import org.apache.lens.server.api.driver.NoOpDriverQueryHook; +import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.QueryContext; + +public class MockDriverQueryHook extends NoOpDriverQueryHook { + public static final String KEY_PRE_LAUNCH = "TEST_KEY_PRE_LAUNCH"; + public static final String VALUE_PRE_LAUNCH = "TEST_VALUE_PRE_LAUNCH"; -public class MockDriverQueryHook implements DriverQueryHook { - public static final String KEY = "TEST_KEY"; - public static final String VALUE = "TEST_VALUE"; + public static final String KEY_POST_SELECT = "TEST_KEY_POST_SELECT"; + public static final String VALUE_POST_SELECT = "TEST_VALUE_POST_SELECT"; + public static final String UNSAVED_KEY_POST_SELECT = "TEST_UNSAVED__KEY_POST_SELECT"; + public static final String UNSAVED_VALUE_POST_SELECT = "TEST_UNSAVED_VALUE_POST_SELECT"; @Override - public void preLaunch(AbstractQueryContext ctx) { - ctx.getSelectedDriverConf().set(KEY, VALUE); + public void preLaunch(QueryContext ctx) { + super.preLaunch(ctx); + ctx.getSelectedDriverConf().set(KEY_PRE_LAUNCH, VALUE_PRE_LAUNCH); + } + + @Override + public void postDriverSelection(AbstractQueryContext ctx) throws LensException { + super.postDriverSelection(ctx); + + //Updated both in driver config and LensConf(which gets persisted) + ctx.getSelectedDriverConf().set(KEY_POST_SELECT, VALUE_POST_SELECT); + ctx.updateConf(new HashMap<String, String>(1){{put(KEY_POST_SELECT, VALUE_POST_SELECT); }}); + + //Updated only in driver conf. + ctx.getSelectedDriverConf().set(UNSAVED_KEY_POST_SELECT, UNSAVED_VALUE_POST_SELECT); } } http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index c72a890..f5552dc 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -658,8 +658,12 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE @Override public void run() { log.info("Starting QuerySubmitter thread"); - while (!pausedForTest && !stopped && !querySubmitter.isInterrupted()) { + while (!stopped && !querySubmitter.isInterrupted()) { try { + if (pausedForTest) { + Thread.sleep(100); + continue; + } QueryContext query = queuedQueries.take(); synchronized (query) { @@ -725,6 +729,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE private void launchQuery(final QueryContext query) throws LensException { checkEstimatedQueriesState(query); + query.getSelectedDriver().getQueryHook().preLaunch(query); QueryStatus oldStatus = query.getStatus(); QueryStatus newStatus = new QueryStatus(query.getStatus().getProgress(), null, QueryStatus.Status.LAUNCHED, "Query is launched on driver", false, null, null, null); @@ -757,13 +762,13 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE } } - // used in tests /** * Pause query submitter. + * note : used in tests only */ - public void pauseQuerySubmitter() { - querySubmitterRunnable.pausedForTest = true; + public void pauseQuerySubmitter(boolean pause) { + querySubmitterRunnable.pausedForTest = pause; } /** @@ -1422,6 +1427,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE ctx.setSelectedDriverQueryCost(selectedDriverQueryCost); Priority priority = driver.decidePriority(ctx); ctx.setPriority(priority == null ? Priority.NORMAL : priority); + driver.getQueryHook().postDriverSelection(ctx); selectGauge.markSuccess(); } finally { parallelCallGauge.markSuccess(); @@ -1927,11 +1933,11 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE /** * Gets the query context. - * + * note: this method is made public to expose it to test cases * @param queryHandle the query handle * @return the query context */ - QueryContext getQueryContext(QueryHandle queryHandle) { + public QueryContext getQueryContext(QueryHandle queryHandle) { return allQueries.get(queryHandle); } http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java index fb757e4..b5d5482 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java @@ -40,6 +40,7 @@ import org.apache.lens.driver.hive.TestRemoteHiveDriver; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.metrics.LensMetricsUtil; import org.apache.lens.server.api.metrics.MetricsService; +import org.apache.lens.server.api.query.QueryExecutionService; import org.apache.lens.server.model.LogSegregationContext; import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; import org.apache.lens.server.query.QueryExecutionServiceImpl; @@ -213,20 +214,26 @@ public abstract class LensJerseyTest extends JerseyTest { */ public void restartLensServer() { HiveConf h = getServerConf(); - restartLensServer(h); + restartLensServer(h, false); } /** * Restart lens server. * * @param conf the conf + * @param pauseQuerySubmitter whether to pause query submitter while starting lens server */ - public void restartLensServer(HiveConf conf) { + public void restartLensServer(HiveConf conf, boolean pauseQuerySubmitter) { LensServices.get().stop(); LensMetricsUtil.clearRegistry(); System.out.println("Lens services stopped!"); LensServices.setInstance(new LensServices(LensServices.LENS_SERVICES_NAME, this.logSegregationContext)); LensServices.get().init(conf); + if (pauseQuerySubmitter) { + QueryExecutionServiceImpl queryService = LensServices.get().getService(QueryExecutionService.NAME); + queryService.pauseQuerySubmitter(true); + System.out.println("Paused Query Submitter"); + } LensServices.get().start(); System.out.println("Lens services restarted!"); } http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java index 0f55d9e..1fa61ef 100644 --- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java +++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java @@ -20,9 +20,10 @@ package org.apache.lens.server; import static org.apache.lens.server.LensServerTestUtil.createTable; import static org.apache.lens.server.LensServerTestUtil.loadData; +import static org.apache.lens.server.api.user.MockDriverQueryHook.*; import static org.apache.lens.server.common.RestAPITestUtil.execute; -import static org.testng.Assert.assertEquals; +import static org.testng.Assert.*; import java.io.*; import java.util.*; @@ -39,6 +40,7 @@ import org.apache.lens.api.query.*; import org.apache.lens.api.result.LensAPIResult; import org.apache.lens.driver.hive.TestRemoteHiveDriver; import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.query.QueryExecutionService; import org.apache.lens.server.api.session.SessionService; import org.apache.lens.server.common.TestResourceFile; @@ -47,6 +49,7 @@ import org.apache.lens.server.query.TestQueryService; import org.apache.lens.server.session.HiveSessionService; import org.apache.lens.server.session.LensSessionImpl; +import org.apache.hadoop.conf.Configuration; import org.apache.hive.service.Service; import org.glassfish.jersey.media.multipart.FormDataBodyPart; @@ -152,18 +155,19 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { List<QueryHandle> launchedQueries = new ArrayList<>(); final int NUM_QUERIES = 10; - boolean killed = false; + boolean isQuerySubmitterPaused = false; + QueryHandle handleForMockDriverQueryHookTest = null; for (int i = 0; i < NUM_QUERIES; i++) { - if (!killed && i > NUM_QUERIES / 3) { + if (!isQuerySubmitterPaused && i > NUM_QUERIES / 3) { // Kill the query submitter thread to make sure some queries stay in accepted queue try { - queryService.pauseQuerySubmitter(); + queryService.pauseQuerySubmitter(true); log.info("Stopped query submitter"); Assert.assertFalse(queryService.getHealthStatus().isHealthy()); } catch (Exception exc) { log.error("Could not kill query submitter", exc); } - killed = true; + isQuerySubmitterPaused = true; } final FormDataMultiPart mp = new FormDataMultiPart(); @@ -182,13 +186,23 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { .get(LensQuery.class); log.info("{} submitted query {} state: {}", i, handle, ctx.getStatus().getStatus()); launchedQueries.add(handle); + if (i == (NUM_QUERIES-1)) { + //checking this only for one of the queued queries. A queued query has all the config information available in + // server memory. (Some of the information is lost after query is purged) + testMockDriverQueryHookPostDriverSelection(queryService, handle, false); + handleForMockDriverQueryHookTest = handle; + log.info("Testing query {} for MockDriverQueryHook", handleForMockDriverQueryHookTest); + } } // Restart the server log.info("Restarting lens server!"); - restartLensServer(); + restartLensServer(getServerConf(), true); log.info("Restarted lens server!"); queryService = LensServices.get().getService(QueryExecutionService.NAME); + Assert.assertFalse(queryService.getHealthStatus().isHealthy()); + testMockDriverQueryHookPostDriverSelection(queryService, handleForMockDriverQueryHookTest, true); + queryService.pauseQuerySubmitter(false); Assert.assertTrue(queryService.getHealthStatus().isHealthy()); // All queries should complete after server restart @@ -223,6 +237,35 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { } /** + * Tests whether the driver configuration updated by mock query driver hook is + * 1. updated in LensConf wherever applicable and + * 2. is persisted and available even after server startup. + * + * @param queryService + * @param handle + * @param afterRestart + */ + private void testMockDriverQueryHookPostDriverSelection(QueryExecutionServiceImpl queryService, QueryHandle handle, + boolean afterRestart){ + QueryContext ctx = queryService.getQueryContext(handle); + assertNotNull(ctx, "Make sure that the query has not been purged"); + assertTrue(ctx.getStatus().queued(), "Make sure query is still in QUEUED state"); + LensConf lensQueryConf = queryService.getQueryContext(handle).getLensConf(); + Configuration driverConf = queryService.getQueryContext(handle).getSelectedDriverConf(); + + assertEquals(driverConf.get(KEY_POST_SELECT), VALUE_POST_SELECT); + assertEquals(lensQueryConf.getProperty(KEY_POST_SELECT), VALUE_POST_SELECT); + + if (afterRestart) { + //This will be unavailable since if was not updated in LensConf by MockDriverQueryHook + assertNull(driverConf.get(UNSAVED_KEY_POST_SELECT)); + } else { + assertEquals(driverConf.get(UNSAVED_KEY_POST_SELECT), UNSAVED_VALUE_POST_SELECT); + } + assertNull(lensQueryConf.getProperty(UNSAVED_KEY_POST_SELECT)); + } + + /** * Test hive server restart. * * @throws Exception the exception http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index 6bf077d..fd6bae3 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -23,6 +23,7 @@ import static javax.ws.rs.core.Response.Status.*; import static org.apache.lens.server.LensServerTestUtil.DB_WITH_JARS; import static org.apache.lens.server.LensServerTestUtil.DB_WITH_JARS_2; import static org.apache.lens.server.api.LensServerAPITestUtil.getLensConf; +import static org.apache.lens.server.api.user.MockDriverQueryHook.*; import static org.apache.lens.server.common.RestAPITestUtil.*; import static org.testng.Assert.*; @@ -668,6 +669,8 @@ public class TestQueryService extends LensJerseyTest { new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); assertNotNull(handle); + QueryContext ctx = queryService.getUpdatedQueryContext(lensSessionId, handle); + assertEquals(ctx.getSelectedDriverConf().get(KEY_POST_SELECT), VALUE_POST_SELECT); // Get query LensQuery lensQuery = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) @@ -696,12 +699,14 @@ public class TestQueryService extends LensJerseyTest { }*/ Thread.sleep(1000); } + assertEquals(ctx.getSelectedDriverConf().get(KEY_PRE_LAUNCH), VALUE_PRE_LAUNCH); assertTrue(lensQuery.getSubmissionTime() > 0); assertTrue(lensQuery.getLaunchTime() > 0); assertTrue(lensQuery.getDriverStartTime() > 0); assertTrue(lensQuery.getDriverFinishTime() > 0); assertTrue(lensQuery.getFinishTime() > 0); - QueryContext ctx = queryService.getUpdatedQueryContext(lensSessionId, lensQuery.getQueryHandle()); + ctx = queryService.getUpdatedQueryContext(lensSessionId, lensQuery.getQueryHandle()); + assertNotNull(ctx.getPhase1RewrittenQuery()); assertEquals(ctx.getPhase1RewrittenQuery(), ctx.getUserQuery()); //Since there is no rewriter in this test assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml index 6362473..4d2bc9a 100644 --- a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml +++ b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml @@ -82,4 +82,11 @@ <value>hive/hive1</value> </property> + <property> + <name>lens.driver.hive.query.hook.class</name> + <value>org.apache.lens.server.api.user.MockDriverQueryHook</value> + <description>The query hook class for hive driver. + </description> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml index d5be96e..f9f345e 100644 --- a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml +++ b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml @@ -82,4 +82,11 @@ <value>hive/hive2</value> </property> + <property> + <name>lens.driver.hive.query.hook.class</name> + <value>org.apache.lens.server.api.user.MockDriverQueryHook</value> + <description>The query hook class for hive driver. + </description> + </property> + </configuration>