LENS-555 : Add presubmit hook in UserConfigLoader and call from drivers
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/558ac1c7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/558ac1c7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/558ac1c7 Branch: refs/heads/current-release-line Commit: 558ac1c75c154a1b483900174c32e62d3ea2350c Parents: 022b589 Author: Rajat Khandelwal <[email protected]> Authored: Fri Jun 12 16:01:34 2015 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Fri Jun 12 16:01:34 2015 +0530 ---------------------------------------------------------------------- .../org/apache/lens/driver/hive/HiveDriver.java | 27 ++++---- .../apache/lens/driver/hive/TestHiveDriver.java | 5 ++ .../lens/driver/hive/TestRemoteHiveDriver.java | 2 + lens-driver-jdbc/pom.xml | 7 ++ .../org/apache/lens/driver/jdbc/JDBCDriver.java | 26 ++++--- .../apache/lens/driver/jdbc/TestJdbcDriver.java | 19 ++++-- .../lens/server/api/driver/LensDriver.java | 7 ++ .../lens/server/api/user/UserConfigLoader.java | 64 +++++++++++++++++ .../api/user/UserConfigLoaderException.java | 60 ++++++++++++++++ .../lens/server/api/driver/MockDriver.java | 4 ++ .../server/api/user/MockUserConfigLoader.java | 47 +++++++++++++ .../org/apache/lens/server/LensServices.java | 3 +- .../server/query/QueryExecutionServiceImpl.java | 6 +- .../server/user/CustomUserConfigLoader.java | 72 -------------------- .../server/user/DatabaseUserConfigLoader.java | 2 + .../lens/server/user/FixedUserConfigLoader.java | 1 + .../LDAPBackedDatabaseUserConfigLoader.java | 1 + .../user/PropertyBasedUserConfigLoader.java | 2 + .../lens/server/user/UserConfigLoader.java | 50 -------------- .../server/user/UserConfigLoaderException.java | 60 ---------------- .../server/user/UserConfigLoaderFactory.java | 30 ++++++-- .../lens/server/user/FooBarConfigLoader.java | 10 +++ .../lens/server/user/TestUserConfigLoader.java | 1 + 23 files changed, 288 insertions(+), 218 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index 052d0e2..b7a3be7 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -18,18 +18,9 @@ */ package org.apache.lens.driver.hive; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.DelayQueue; -import java.util.concurrent.Delayed; -import java.util.concurrent.TimeUnit; +import java.io.*; +import java.util.*; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -49,6 +40,7 @@ import org.apache.lens.server.api.priority.QueryPriorityDecider; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.user.UserConfigLoader; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -135,6 +127,7 @@ public class HiveDriver implements LensDriver { // package-local. Test case can change. boolean whetherCalculatePriority; + private UserConfigLoader userConfigLoader; private String sessionDbKey(String sessionHandle, String database) { @@ -511,6 +504,9 @@ public class HiveDriver implements LensDriver { + "User query: " + ctx.getUserQuery(), e); } } + if (userConfigLoader != null) { + userConfigLoader.preSubmit(ctx); + } OperationHandle op = getClient().executeStatementAsync(getSession(ctx), ctx.getSelectedDriverQuery(), qdconf.getValByRegex(".*")); ctx.setDriverOpHandle(op.toString()); @@ -727,7 +723,7 @@ public class HiveDriver implements LensDriver { } catch (Exception e) { checkInvalidSession(e); log.warn("Error closing session for lens session: " + lensSessionDbKey + ", hive session: " - + lensToHiveSession.get(lensSessionDbKey), e); + + lensToHiveSession.get(lensSessionDbKey), e); } } lensToHiveSession.clear(); @@ -746,6 +742,11 @@ public class HiveDriver implements LensDriver { driverListeners.add(driverEventListener); } + @Override + public void registerUserConfigLoader(UserConfigLoader userConfigLoader) { + this.userConfigLoader = userConfigLoader; + } + protected CLIServiceClient getClient() throws LensException { if (isEmbedded) { if (embeddedConnection == null) { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java index b21258a..8355f29 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java @@ -18,6 +18,8 @@ */ package org.apache.lens.driver.hive; +import static org.apache.lens.server.api.user.MockUserConfigLoader.*; + import static org.testng.Assert.*; import java.io.*; @@ -42,6 +44,7 @@ import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.ExplainQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.user.MockUserConfigLoader; import org.apache.lens.server.api.util.LensUtil; import org.apache.hadoop.conf.Configuration; @@ -126,6 +129,7 @@ public class TestHiveDriver { conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true); driver = new HiveDriver(); driver.configure(conf); + driver.registerUserConfigLoader(new MockUserConfigLoader(conf)); drivers = new ArrayList<LensDriver>() { { add(driver); @@ -483,6 +487,7 @@ public class TestHiveDriver { */ protected void validateExecuteAsync(QueryContext ctx, DriverQueryState finalState, boolean isPersistent, boolean formatNulls) throws Exception { + assertEquals(ctx.getSelectedDriverConf().get(KEY), VALUE); validateExecuteAsync(ctx, finalState, isPersistent, formatNulls, driver); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java index b4086b8..c144fd8 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java @@ -33,6 +33,7 @@ import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState; import org.apache.lens.server.api.driver.LensDriver; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.user.MockUserConfigLoader; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; @@ -134,6 +135,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver { driver = new HiveDriver(); conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true); driver.configure(conf); + driver.registerUserConfigLoader(new MockUserConfigLoader(conf)); drivers = new ArrayList<LensDriver>() { { add(driver); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/pom.xml b/lens-driver-jdbc/pom.xml index a7994c5..615410b 100644 --- a/lens-driver-jdbc/pom.xml +++ b/lens-driver-jdbc/pom.xml @@ -81,5 +81,12 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> </dependency> + <dependency> + <groupId>org.apache.lens</groupId> + <artifactId>lens-server-api</artifactId> + <type>test-jar</type> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 70deaa4..bda6572 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -43,10 +43,8 @@ import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.events.LensEventListener; import org.apache.lens.server.api.metrics.MethodMetricsContext; import org.apache.lens.server.api.metrics.MethodMetricsFactory; -import org.apache.lens.server.api.query.AbstractQueryContext; -import org.apache.lens.server.api.query.PreparedQueryContext; -import org.apache.lens.server.api.query.QueryContext; -import org.apache.lens.server.api.query.QueryRewriter; +import org.apache.lens.server.api.query.*; +import org.apache.lens.server.api.user.UserConfigLoader; import org.apache.lens.server.model.LogSegregationContext; import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; @@ -93,6 +91,7 @@ public class JDBCDriver implements LensDriver { private ConnectionProvider estimateConnectionProvider; private LogSegregationContext logSegregationContext; + private UserConfigLoader userConfigLoader; /** * Data related to a query submitted to JDBCDriver. @@ -545,6 +544,7 @@ public class JDBCDriver implements LensDriver { } private static final QueryCost JDBC_DRIVER_COST = new QueryCost(0, 0); + /** * Dummy JDBC query Plan class to get min cost selector working. */ @@ -741,9 +741,9 @@ public class JDBCDriver implements LensDriver { * @throws LensException */ private PreparedStatement prepareInternal(AbstractQueryContext pContext, - boolean calledForEstimate, - boolean checkConfigured, - String metricCallStack) throws LensException { + boolean calledForEstimate, + boolean checkConfigured, + String metricCallStack) throws LensException { // Caller might have already verified configured status and driver query, so we don't have // to do this check twice. Caller must set checkConfigured to false in that case. if (checkConfigured) { @@ -888,6 +888,9 @@ public class JDBCDriver implements LensDriver { String rewrittenQuery = rewriteQuery(context); JdbcQueryContext jdbcCtx = new JdbcQueryContext(context, logSegregationContext); jdbcCtx.setRewrittenQuery(rewrittenQuery); + if (userConfigLoader != null) { + userConfigLoader.preSubmit(context); + } try { Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx, logSegregationContext)); jdbcCtx.setResultFuture(future); @@ -1057,7 +1060,7 @@ public class JDBCDriver implements LensDriver { throw new LensException(); } } catch (LensException e) { - LOG.warn("Error closing prapared query : " + query , e); + LOG.warn("Error closing prapared query : " + query, e); } } } finally { @@ -1075,6 +1078,13 @@ public class JDBCDriver implements LensDriver { } + + @Override + public void registerUserConfigLoader(UserConfigLoader userConfigLoader) { + this.userConfigLoader = userConfigLoader; + } + + /* * (non-Javadoc) * http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java index 2e3dcc7..be34164 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java @@ -18,6 +18,8 @@ */ package org.apache.lens.driver.jdbc; +import static org.apache.lens.server.api.user.MockUserConfigLoader.*; + import static org.testng.Assert.*; import java.sql.Connection; @@ -41,6 +43,7 @@ import org.apache.lens.server.api.metrics.LensMetricsRegistry; import org.apache.lens.server.api.query.ExplainQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.user.MockUserConfigLoader; import org.apache.lens.server.api.util.LensUtil; import org.apache.hadoop.conf.Configuration; @@ -91,6 +94,7 @@ public class TestJdbcDriver { driver = new JDBCDriver(); driver.configure(baseConf); + driver.registerUserConfigLoader(new MockUserConfigLoader(hConf)); assertNotNull(driver); assertTrue(driver.configured); @@ -556,7 +560,7 @@ public class TestJdbcDriver { } }; - driver.executeAsync(context); + executeAsync(context); QueryHandle handle = context.getQueryHandle(); driver.registerForCompletionNotification(handle, 0, listener); @@ -633,7 +637,7 @@ public class TestJdbcDriver { QueryContext ctx = new QueryContext(query, "SA", new LensConf(), baseConf, drivers); for (int i = 0; i < JDBCDriverConfConstants.JDBC_POOL_MAX_SIZE_DEFAULT; i++) { - driver.executeAsync(ctx); + executeAsync(ctx); driver.updateStatus(ctx); System.out.println("@@@@ QUERY " + (i + 1)); } @@ -641,7 +645,7 @@ public class TestJdbcDriver { String validQuery = "SELECT * FROM invalid_conn_close"; QueryContext validCtx = createQueryContext(validQuery); System.out.println("@@@ Submitting valid query"); - driver.executeAsync(validCtx); + executeAsync(validCtx); // Wait for query to finish while (true) { @@ -655,6 +659,11 @@ public class TestJdbcDriver { driver.closeQuery(validCtx.getQueryHandle()); } + private void executeAsync(QueryContext ctx) throws LensException { + driver.executeAsync(ctx); + assertEquals(ctx.getSelectedDriverConf().get(KEY), VALUE); + } + /** * Test connection close for successful queries. * @@ -706,7 +715,7 @@ public class TestJdbcDriver { final String query = "SELECT * FROM cancel_query_test"; QueryContext context = createQueryContext(query); System.out.println("@@@ test_cancel:" + context.getQueryHandle()); - driver.executeAsync(context); + executeAsync(context); QueryHandle handle = context.getQueryHandle(); boolean isCancelled = driver.cancelQuery(handle); driver.updateStatus(context); @@ -753,7 +762,7 @@ public class TestJdbcDriver { } }; - driver.executeAsync(ctx); + executeAsync(ctx); QueryHandle handle = ctx.getQueryHandle(); driver.registerForCompletionNotification(handle, 0, listener); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java index 8511ce6..0c5ee0a 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java @@ -28,6 +28,7 @@ import org.apache.lens.server.api.events.LensEventListener; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.user.UserConfigLoader; import org.apache.hadoop.conf.Configuration; @@ -182,4 +183,10 @@ public interface LensDriver extends Externalizable { * @param driverEventListener the driver event listener */ void registerDriverEventListener(LensEventListener<DriverEvent> driverEventListener); + + /** + * Add the user config loader to driver for use + * @param userConfigLoader + */ + void registerUserConfigLoader(UserConfigLoader userConfigLoader); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java new file mode 100644 index 0000000..93e11a2 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.user; + +import java.util.Map; + +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.QueryContext; + +import org.apache.hadoop.hive.conf.HiveConf; + +import lombok.extern.slf4j.Slf4j; + +/** + * The Class UserConfigLoader. It's initialized once in the server lifetime. After that, t: + * 1. Gets session configs for the user on each session open. This config applies to the particular session + * and is forwarded for all actions. One Use case is to decide driver specific details e.g. priority/queue of + * all queries of the user. + * 2. Provides a pre-submit hook. Just before submission + */ +@Slf4j +public abstract class UserConfigLoader { + + /** The hive conf. */ + protected final HiveConf hiveConf; + + /** + * Instantiates a new user config loader. + * + * @param conf the conf + */ + protected UserConfigLoader(HiveConf conf) { + this.hiveConf = conf; + } + + /** + * Gets the user config. + * + * @param loggedInUser the logged in user + * @return the user config + * @throws UserConfigLoaderException the user config loader exception + */ + public abstract Map<String, String> getUserConfig(String loggedInUser) throws UserConfigLoaderException; + + public void preSubmit(QueryContext ctx) throws LensException { + log.debug("Pre submit " + ctx + " on " + ctx.getSelectedDriver()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoaderException.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoaderException.java b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoaderException.java new file mode 100644 index 0000000..c1cbb24 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoaderException.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.user; + +/** + * The Class UserConfigLoaderException. + */ +public class UserConfigLoaderException extends RuntimeException { + + /** + * Instantiates a new user config loader exception. + */ + public UserConfigLoaderException() { + super(); + } + + /** + * Instantiates a new user config loader exception. + * + * @param s the s + */ + public UserConfigLoaderException(String s) { + super(s); + } + + /** + * Instantiates a new user config loader exception. + * + * @param e the e + */ + public UserConfigLoaderException(Throwable e) { + super(e); + } + + /** + * Instantiates a new user config loader exception. + * + * @param message the message + * @param cause the cause + */ + public UserConfigLoaderException(String message, Throwable cause) { + super(message, cause); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java index 30c1a71..4b80918 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java @@ -35,6 +35,7 @@ import org.apache.lens.server.api.events.LensEventListener; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.user.UserConfigLoader; import org.apache.hadoop.conf.Configuration; import org.apache.hive.service.cli.ColumnDescriptor; @@ -182,7 +183,10 @@ public class MockDriver implements LensDriver { */ @Override public void registerDriverEventListener(LensEventListener<DriverEvent> driverEventListener) { + } + @Override + public void registerUserConfigLoader(UserConfigLoader userConfigLoader) { } /* http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java new file mode 100644 index 0000000..f6ef0d4 --- /dev/null +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.user; + +import java.util.Map; + +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.QueryContext; + +import org.apache.hadoop.hive.conf.HiveConf; + +import com.beust.jcommander.internal.Maps; + +public class MockUserConfigLoader extends UserConfigLoader { + public static final String KEY = "TEST_KEY"; + public static final String VALUE = "TEST_VALUE"; + + public MockUserConfigLoader(HiveConf conf) { + super(conf); + } + + @Override + public Map<String, String> getUserConfig(String loggedInUser) throws UserConfigLoaderException { + return Maps.newHashMap(); + } + + @Override + public void preSubmit(QueryContext ctx) throws LensException { + ctx.getSelectedDriverConf().set(KEY, VALUE); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/LensServices.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java index c27717f..5fe3935 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java @@ -191,7 +191,7 @@ public class LensServices extends CompositeService implements ServiceProvider { serviceMode = conf.getEnum(SERVER_MODE, SERVICE_MODE.valueOf(DEFAULT_SERVER_MODE)); cliService = new CLIService(); - + UserConfigLoaderFactory.init(conf); // Add default services addService(cliService); addService(new EventServiceImpl(LensEventService.NAME)); @@ -262,7 +262,6 @@ public class LensServices extends CompositeService implements ServiceProvider { snapShotInterval = conf.getLong(SERVER_SNAPSHOT_INTERVAL, DEFAULT_SERVER_SNAPSHOT_INTERVAL); LOG.info("Initialized services: " + services.keySet().toString()); - UserConfigLoaderFactory.init(conf); timer = new Timer("lens-server-snapshotter", true); } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 7623743..e41a0d2 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -55,6 +55,7 @@ import org.apache.lens.server.model.LogSegregationContext; import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; import org.apache.lens.server.session.LensSessionImpl; import org.apache.lens.server.stats.StatisticsService; +import org.apache.lens.server.user.UserConfigLoaderFactory; import org.apache.lens.server.util.UtilityMethods; import org.apache.commons.collections.CollectionUtils; @@ -307,7 +308,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu Class<?> clazz = Class.forName(driverClass); LensDriver driver = (LensDriver) clazz.newInstance(); driver.configure(LensServerConf.getConf()); - + driver.registerUserConfigLoader(UserConfigLoaderFactory.getUserConfigLoader()); if (driver instanceof HiveDriver) { driver.registerDriverEventListener(driverEventListener); } @@ -895,7 +896,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu try { loadDriversAndSelector(); } catch (LensException e) { - throw new IllegalStateException("Could not load drivers"); + LOG.error(e); + throw new IllegalStateException("Could not load drivers", e); } maxFinishedQueries = conf.getInt(LensConfConstants.MAX_NUMBER_OF_FINISHED_QUERY, LensConfConstants.DEFAULT_FINISHED_QUERIES); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/CustomUserConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/user/CustomUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/CustomUserConfigLoader.java deleted file mode 100644 index 9a6a350..0000000 --- a/lens-server/src/main/java/org/apache/lens/server/user/CustomUserConfigLoader.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.server.user; - -import java.lang.reflect.InvocationTargetException; -import java.util.Map; - -import org.apache.lens.server.api.LensConfConstants; - -import org.apache.hadoop.hive.conf.HiveConf; - -/** - * The Class CustomUserConfigLoader. - */ -public class CustomUserConfigLoader extends UserConfigLoader { - - /** The custom handler class. */ - Class<? extends UserConfigLoader> customHandlerClass; - - /** The custom provider. */ - UserConfigLoader customProvider; - - /** - * Instantiates a new custom user config loader. - * - * @param conf the conf - */ - public CustomUserConfigLoader(HiveConf conf) { - super(conf); - this.customHandlerClass = (Class<? extends UserConfigLoader>) hiveConf.getClass( - LensConfConstants.USER_RESOLVER_CUSTOM_CLASS, UserConfigLoader.class); - try { - this.customProvider = customHandlerClass.getConstructor(HiveConf.class).newInstance(conf); - // in java6, these four extend directly from Exception. So have to handle separately. In java7, - // the common subclass is ReflectiveOperationException - } catch (InvocationTargetException e) { - throw new UserConfigLoaderException(e); - } catch (NoSuchMethodException e) { - throw new UserConfigLoaderException(e); - } catch (InstantiationException e) { - throw new UserConfigLoaderException(e); - } catch (IllegalAccessException e) { - throw new UserConfigLoaderException(e); - } - } - - /* - * (non-Javadoc) - * - * @see org.apache.lens.server.user.UserConfigLoader#getUserConfig(java.lang.String) - */ - @Override - public Map<String, String> getUserConfig(String loggedInUser) throws UserConfigLoaderException { - return customProvider.getUserConfig(loggedInUser); - } -} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java index d267948..ce0d112 100644 --- a/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java +++ b/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java @@ -26,6 +26,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.user.UserConfigLoader; +import org.apache.lens.server.api.user.UserConfigLoaderException; import org.apache.lens.server.util.UtilityMethods; import org.apache.commons.dbcp.BasicDataSource; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java index b81b994..5813ac0 100644 --- a/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java +++ b/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.user.UserConfigLoader; import org.apache.hadoop.hive.conf.HiveConf; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/LDAPBackedDatabaseUserConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/user/LDAPBackedDatabaseUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/LDAPBackedDatabaseUserConfigLoader.java index 108a012..82b76c9 100644 --- a/lens-server/src/main/java/org/apache/lens/server/user/LDAPBackedDatabaseUserConfigLoader.java +++ b/lens-server/src/main/java/org/apache/lens/server/user/LDAPBackedDatabaseUserConfigLoader.java @@ -34,6 +34,7 @@ import javax.naming.directory.SearchResult; import javax.naming.ldap.InitialLdapContext; import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.user.UserConfigLoaderException; import org.apache.commons.dbutils.QueryRunner; import org.apache.hadoop.hive.conf.HiveConf; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java index 16a9a70..009e34a 100644 --- a/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java +++ b/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java @@ -27,6 +27,8 @@ import java.util.Map; import java.util.Properties; import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.user.UserConfigLoader; +import org.apache.lens.server.api.user.UserConfigLoaderException; import org.apache.hadoop.hive.conf.HiveConf; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoader.java deleted file mode 100644 index 21eea3b..0000000 --- a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoader.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.server.user; - -import java.util.Map; - -import org.apache.hadoop.hive.conf.HiveConf; - -/** - * The Class UserConfigLoader. - */ -public abstract class UserConfigLoader { - - /** The hive conf. */ - protected final HiveConf hiveConf; - - /** - * Instantiates a new user config loader. - * - * @param conf the conf - */ - protected UserConfigLoader(HiveConf conf) { - this.hiveConf = conf; - } - - /** - * Gets the user config. - * - * @param loggedInUser the logged in user - * @return the user config - * @throws UserConfigLoaderException the user config loader exception - */ - public abstract Map<String, String> getUserConfig(String loggedInUser) throws UserConfigLoaderException; -} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderException.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderException.java b/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderException.java deleted file mode 100644 index 0899746..0000000 --- a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderException.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.server.user; - -/** - * The Class UserConfigLoaderException. - */ -public class UserConfigLoaderException extends RuntimeException { - - /** - * Instantiates a new user config loader exception. - */ - public UserConfigLoaderException() { - super(); - } - - /** - * Instantiates a new user config loader exception. - * - * @param s the s - */ - public UserConfigLoaderException(String s) { - super(s); - } - - /** - * Instantiates a new user config loader exception. - * - * @param e the e - */ - public UserConfigLoaderException(Throwable e) { - super(e); - } - - /** - * Instantiates a new user config loader exception. - * - * @param message the message - * @param cause the cause - */ - public UserConfigLoaderException(String message, Throwable cause) { - super(message, cause); - } -} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java b/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java index 383aaf5..f92584b 100644 --- a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java +++ b/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java @@ -18,9 +18,14 @@ */ package org.apache.lens.server.user; +import static org.apache.lens.server.api.LensConfConstants.USER_RESOLVER_CUSTOM_CLASS; + +import java.lang.reflect.InvocationTargetException; import java.util.Map; import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.user.UserConfigLoader; +import org.apache.lens.server.api.user.UserConfigLoaderException; import org.apache.hadoop.hive.conf.HiveConf; @@ -31,6 +36,7 @@ public final class UserConfigLoaderFactory { private UserConfigLoaderFactory() { } + /** The conf. */ private static HiveConf conf; @@ -44,13 +50,13 @@ public final class UserConfigLoaderFactory { */ public static void init(HiveConf c) { conf = c; - userConfigLoader = initializeUserConfigLoader(); + userConfigLoader = null; } /** * The Enum RESOLVER_TYPE. */ - public static enum RESOLVER_TYPE { + public enum RESOLVER_TYPE { /** The fixed. */ FIXED, @@ -68,6 +74,13 @@ public final class UserConfigLoaderFactory { CUSTOM } + public static UserConfigLoader getUserConfigLoader() { + if (userConfigLoader == null) { + userConfigLoader = initializeUserConfigLoader(); + } + return userConfigLoader; + } + /** * Initialize user config loader. * @@ -80,7 +93,7 @@ public final class UserConfigLoaderFactory { } for (RESOLVER_TYPE type : RESOLVER_TYPE.values()) { if (type.name().equals(resolverType)) { - return getQueryUserResolver(type); + return createUserConfigLoader(type); } } throw new UserConfigLoaderException("user resolver type not determined. provided value: " + resolverType); @@ -92,7 +105,7 @@ public final class UserConfigLoaderFactory { * @param resolverType the resolver type * @return the query user resolver */ - public static UserConfigLoader getQueryUserResolver(RESOLVER_TYPE resolverType) { + public static UserConfigLoader createUserConfigLoader(RESOLVER_TYPE resolverType) { switch (resolverType) { case PROPERTYBASED: return new PropertyBasedUserConfigLoader(conf); @@ -101,7 +114,12 @@ public final class UserConfigLoaderFactory { case LDAP_BACKED_DATABASE: return new LDAPBackedDatabaseUserConfigLoader(conf); case CUSTOM: - return new CustomUserConfigLoader(conf); + try { + return (conf.getClass(USER_RESOLVER_CUSTOM_CLASS, UserConfigLoader.class, UserConfigLoader.class)) + .getConstructor(HiveConf.class).newInstance(conf); + } catch (InvocationTargetException | NoSuchMethodException | IllegalAccessException | InstantiationException e) { + throw new UserConfigLoaderException(e); + } case FIXED: default: return new FixedUserConfigLoader(conf); @@ -115,6 +133,6 @@ public final class UserConfigLoaderFactory { * @return the user config */ public static Map<String, String> getUserConfig(String loggedInUser) { - return userConfigLoader.getUserConfig(loggedInUser); + return getUserConfigLoader().getUserConfig(loggedInUser); } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java b/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java index 5b33440..0a319ef 100644 --- a/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java +++ b/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java @@ -21,6 +21,10 @@ package org.apache.lens.server.user; import java.util.HashMap; import java.util.Map; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.user.UserConfigLoader; + import org.apache.hadoop.hive.conf.HiveConf; /** @@ -53,4 +57,10 @@ public class FooBarConfigLoader extends UserConfigLoader { public Map<String, String> getUserConfig(String loggedInUser) { return CONST_HASH_MAP; } + + @Override + public void preSubmit(QueryContext ctx) throws LensException { + super.preSubmit(ctx); + ctx.getSelectedDriverConf().set("random.conf.key", "random.conf.value"); + } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java b/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java index 4f399c2..a122d1e 100644 --- a/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java +++ b/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java @@ -25,6 +25,7 @@ import java.util.HashMap; import org.apache.lens.server.LensServerConf; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.user.UserConfigLoader; import org.apache.lens.server.util.UtilityMethods; import org.apache.commons.dbcp.BasicDataSource;
