LENS-446: Remove server params from session and driver configurations
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7080b76c Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7080b76c Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7080b76c Branch: refs/heads/current-release-line Commit: 7080b76c63a82e09b5d42c2d2b7458f80b1085f8 Parents: 02760a3 Author: Amareshwari Sriramadasu <[email protected]> Authored: Thu Oct 8 15:14:57 2015 +0530 Committer: Raju Bairishetti <[email protected]> Committed: Thu Oct 8 15:14:57 2015 +0530 ---------------------------------------------------------------------- .../org/apache/lens/server/LensApplication.java | 2 +- .../org/apache/lens/server/LensServerConf.java | 24 ++++++++++++++++---- .../server/query/QueryExecutionServiceImpl.java | 2 +- .../lens/server/session/LensSessionImpl.java | 23 ++++++++++++++----- .../lens/server/query/TestQueryService.java | 4 ++++ .../server/session/TestSessionResource.java | 9 +++++++- 6 files changed, 50 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/main/java/org/apache/lens/server/LensApplication.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java index a446ab6..71ff9ea 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java @@ -38,7 +38,7 @@ import lombok.extern.slf4j.Slf4j; public class LensApplication extends Application { /** The conf. */ - public static final Configuration CONF = LensServerConf.getConf(); + public static final Configuration CONF = LensServerConf.getHiveConf(); @Override public Set<Class<?>> getClasses() { http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java index 4d333c6..6db720d 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java @@ -18,6 +18,11 @@ */ package org.apache.lens.server; +import java.util.Iterator; +import java.util.Map; + +import org.apache.lens.server.api.LensConfConstants; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; @@ -32,13 +37,22 @@ public final class LensServerConf { private static final class ConfHolder { public static final HiveConf HIVE_CONF = new HiveConf(); // configuration object which does not load defaults and loads only lens*.xml files. - public static final Configuration CONF = new Configuration(false); + // and removes any server specific configuration, that should not be passed to drivers + private static final Configuration OVERRIDING_CONF_FOR_DRIVER = new Configuration(false); static { HIVE_CONF.addResource("lensserver-default.xml"); HIVE_CONF.addResource("lens-site.xml"); - CONF.addResource("lensserver-default.xml"); - CONF.addResource("lens-site.xml"); + Configuration conf = new Configuration(false); + conf.addResource("lens-site.xml"); + Iterator<Map.Entry<String, String>> confItr = conf.iterator(); + while (confItr.hasNext()) { + Map.Entry<String, String> prop = confItr.next(); + if (!prop.getKey().startsWith(LensConfConstants.SERVER_PFX)) { + OVERRIDING_CONF_FOR_DRIVER.set(prop.getKey(), prop.getValue()); + } + } + } } @@ -57,8 +71,8 @@ public final class LensServerConf { * * @return the conf */ - public static Configuration getConf() { - return ConfHolder.CONF; + public static Configuration getConfForDrivers() { + return ConfHolder.OVERRIDING_CONF_FOR_DRIVER; } /** http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 73179e1..d57812f 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -338,7 +338,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE for (Class<?> driverClass : driverClasses) { try { LensDriver driver = (LensDriver) driverClass.newInstance(); - driver.configure(LensServerConf.getConf()); + driver.configure(LensServerConf.getConfForDrivers()); if (driver instanceof HiveDriver) { driver.registerDriverEventListener(driverEventListener); } http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java index d0677cc..9d8f198 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java @@ -65,7 +65,7 @@ public class LensSessionImpl extends HiveSessionImpl { private long sessionTimeout; /** The conf. */ - private Configuration conf = createDefaultConf(); + private Configuration conf = new Configuration(createDefaultConf()); /** * Keep track of DB static resources which failed to be added to this session @@ -102,16 +102,27 @@ public class LensSessionImpl extends HiveSessionImpl { persistInfo.setSessionConf(sessionConf); } + private static Configuration sessionDefaultConfig; /** * Creates the default conf. * * @return the configuration */ - public static Configuration createDefaultConf() { - Configuration conf = new Configuration(false); - conf.addResource("lenssession-default.xml"); - conf.addResource("lens-site.xml"); - return conf; + public static synchronized Configuration createDefaultConf() { + if (sessionDefaultConfig == null) { + Configuration conf = new Configuration(false); + conf.addResource("lenssession-default.xml"); + conf.addResource("lens-site.xml"); + sessionDefaultConfig = new Configuration(false); + Iterator<Map.Entry<String, String>> confItr = conf.iterator(); + while (confItr.hasNext()) { + Map.Entry<String, String> prop = confItr.next(); + if (!prop.getKey().startsWith(LensConfConstants.SERVER_PFX)) { + sessionDefaultConfig.set(prop.getKey(), prop.getValue()); + } + } + } + return sessionDefaultConfig; } /** The default hive session conf. */ http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index ea75ffb..6218882 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -1167,6 +1167,8 @@ public class TestQueryService extends LensJerseyTest { assertNull(conf.get("hive.metastore.warehouse.dir")); // core default should not be loaded assertNull(conf.get("fs.default.name")); + // server configuration should not set + assertNull(conf.get("lens.server.persist.location")); // Test server config. Hive configs overriden should be set assertFalse(Boolean.parseBoolean(queryService.getHiveConf().get("hive.server2.log.redirection.enabled"))); @@ -1202,6 +1204,8 @@ public class TestQueryService extends LensJerseyTest { assertEquals(dconf.get("lens.driver.test.key"), "set"); // core default should not be loaded assertNull(dconf.get("fs.default.name")); + // server configuration should not set + assertNull(dconf.get("lens.server.persist.location")); } } http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java index 8fffbc9..3055ce5 100644 --- a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java +++ b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java @@ -169,7 +169,14 @@ public class TestSessionResource extends LensJerseyTest { System.out.println("Session params:" + sessionParams.getElements()); Assert.assertEquals(sessionParams.getElements().size(), 1); Assert.assertTrue(sessionParams.getElements().contains("my.conf=myvalue")); - + // get server params on session + try { + paramtarget.queryParam("sessionid", handle).queryParam("key", "lens.server.persist.location").request() + .get(StringList.class); + Assert.fail("Expected 404"); + } catch (Exception ne) { + Assert.assertTrue(ne instanceof NotFoundException); + } // get all params verbose sessionParams = paramtarget.queryParam("sessionid", handle).queryParam("verbose", true).request() .get(StringList.class);
