Repository: hive Updated Branches: refs/heads/branch-2.0 abe594ec7 -> b277f8ae7 refs/heads/master fad307719 -> c278860a9
HIVE-12813 : LLAP: issues in setup, shutdown (Sergey Shelukhin, reviewed by Gunther Hagleitner) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c278860a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c278860a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c278860a Branch: refs/heads/master Commit: c278860a941b1014eea6d3a85cfa0c5eb16d49e5 Parents: fad3077 Author: Sergey Shelukhin <[email protected]> Authored: Mon Jan 11 10:45:44 2016 -0800 Committer: Sergey Shelukhin <[email protected]> Committed: Mon Jan 11 10:45:44 2016 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 7 ++- .../hadoop/hive/llap/cli/LlapServiceDriver.java | 60 ++++++++++++-------- .../hive/llap/daemon/impl/LlapDaemon.java | 6 +- .../impl/LlapDaemonProtocolServerImpl.java | 1 + .../daemon/services/impl/LlapWebServices.java | 2 - 5 files changed, 47 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c278860a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 479fa46..eedd205 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2409,9 +2409,10 @@ public class HiveConf extends Configuration { "ZooKeeper for ZooKeeper SecretManager."), LLAP_ZKSM_ZK_CONNECTION_STRING("hive.llap.zk.sm.connectionString", "", "ZooKeeper connection string for ZooKeeper SecretManager."), - LLAP_SECURITY_ACL("hive.llap.daemon.service.acl", "*", "The ACL for LLAP daemon."), - LLAP_MANAGEMENT_ACL("hive.llap.management.service.acl", "*", - "The ACL for LLAP daemon management."), + // Note: do not rename to ..service.acl; Hadoop generates .hosts setting name from this, + // resulting in a collision with existing hive.llap.daemon.service.hosts and bizarre errors. + LLAP_SECURITY_ACL("hive.llap.daemon.acl", "*", "The ACL for LLAP daemon."), + LLAP_MANAGEMENT_ACL("hive.llap.management.acl", "*", "The ACL for LLAP daemon management."), // Hadoop DelegationTokenManager default is 1 week. LLAP_DELEGATION_TOKEN_LIFETIME("hive.llap.daemon.delegation.token.lifetime", "14d", new TimeValidator(TimeUnit.SECONDS), http://git-wip-us.apache.org/repos/asf/hive/blob/c278860a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index 4b330f4..0d54558 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -18,10 +18,9 @@ package org.apache.hadoop.hive.llap.cli; +import java.io.IOException; import java.io.OutputStreamWriter; import java.net.URL; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -49,6 +48,10 @@ public class LlapServiceDriver { protected static final Logger LOG = LoggerFactory.getLogger(LlapServiceDriver.class.getName()); private static final String[] DEFAULT_AUX_CLASSES = new String[] { "org.apache.hive.hcatalog.data.JsonSerDe", "org.apache.hadoop.hive.hbase.HBaseSerDe" }; + private static final String[] NEEDED_CONFIGS = { + "tez-site.xml", "hive-site.xml", "llap-daemon-site.xml", "core-site.xml" }; + private static final String[] OPTIONAL_CONFIGS = { "ssl-server.xml" }; + private final Configuration conf; @@ -115,17 +118,16 @@ public class LlapServiceDriver { FileSystem fs = FileSystem.get(conf); FileSystem lfs = FileSystem.getLocal(conf).getRawFileSystem(); - String[] neededConfig = - { "tez-site.xml", "hive-site.xml", "llap-daemon-site.xml", "core-site.xml" }; - // needed so that the file is actually loaded into configuration. - for (String f : neededConfig) { + for (String f : NEEDED_CONFIGS) { conf.addResource(f); if (conf.getResource(f) == null) { throw new Exception("Unable to find required config file: " + f); } } - + for (String f : OPTIONAL_CONFIGS) { + conf.addResource(f); + } conf.reloadConfiguration(); if (options.getName() != null) { @@ -251,22 +253,14 @@ public class LlapServiceDriver { Path confPath = new Path(tmpDir, "conf"); lfs.mkdirs(confPath); - for (String f : neededConfig) { - if (f.equals("llap-daemon-site.xml")) { - FSDataOutputStream confStream = lfs.create(new Path(confPath, f)); - - Configuration copy = resolve(conf, "llap-daemon-site.xml"); - - for (Entry<Object, Object> props : options.getConfig().entrySet()) { - // overrides - copy.set((String) props.getKey(), (String) props.getValue()); - } - - copy.writeXml(confStream); - confStream.close(); - } else { - // they will be file:// URLs - lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath); + for (String f : NEEDED_CONFIGS) { + copyConfig(options, lfs, confPath, f); + } + for (String f : OPTIONAL_CONFIGS) { + try { + copyConfig(options, lfs, confPath, f); + } catch (Throwable t) { + LOG.info("Error getting an optional config " + f + "; ignoring: " + t.getMessage()); } } @@ -312,4 +306,24 @@ public class LlapServiceDriver { LOG.debug("Exiting successfully"); } } + + private void copyConfig( + LlapOptions options, FileSystem lfs, Path confPath, String f) throws IOException { + if (f.equals("llap-daemon-site.xml")) { + FSDataOutputStream confStream = lfs.create(new Path(confPath, f)); + + Configuration copy = resolve(conf, "llap-daemon-site.xml"); + + for (Entry<Object, Object> props : options.getConfig().entrySet()) { + // overrides + copy.set((String) props.getKey(), (String) props.getValue()); + } + + copy.writeXml(confStream); + confStream.close(); + } else { + // they will be file:// URLs + lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath); + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/c278860a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index b3057c3..917ff32 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -258,7 +258,11 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla public void shutdown() { LOG.info("LlapDaemon shutdown invoked"); if (llapDaemonInfoBean != null) { - MBeans.unregister(llapDaemonInfoBean); + try { + MBeans.unregister(llapDaemonInfoBean); + } catch (Throwable ex) { + LOG.info("Error unregistering the bean; ignoring", ex); + } } if (pauseMonitor != null) { http://git-wip-us.apache.org/repos/asf/hive/blob/c278860a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java index f87fffe..45ca906 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java @@ -215,6 +215,7 @@ public class LlapDaemonProtocolServerImpl extends AbstractService return server; } + @Override public GetTokenResponseProto getDelegationToken(RpcController controller, GetTokenRequestProto request) throws ServiceException { http://git-wip-us.apache.org/repos/asf/hive/blob/c278860a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java index ed51f3c..e233b41 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java @@ -43,7 +43,6 @@ public class LlapWebServices extends AbstractService { @Override public void serviceInit(Configuration conf) { - this.conf = new Configuration(conf); this.conf.addResource(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); @@ -55,7 +54,6 @@ public class LlapWebServices extends AbstractService { @Override public void serviceStart() throws Exception { String bindAddress = "0.0.0.0"; - Configuration conf = getConfig(); if (UserGroupInformation.isSecurityEnabled() && HiveConf.getBoolVar(conf, ConfVars.LLAP_WEB_AUTO_AUTH)) { conf.set("hadoop.http.authentication.type", "kerberos");
