HIVE-11766. LLAP: Remove MiniLlapCluster from shim layer after hadoop-1 removal. (Siddharth Seth, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/184e0e1d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/184e0e1d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/184e0e1d Branch: refs/heads/llap Commit: 184e0e1da55c576cd8766c52423bcbcd4a2be69b Parents: f41cbea Author: Siddharth Seth <ss...@apache.org> Authored: Thu Mar 31 14:53:59 2016 -0700 Committer: Siddharth Seth <ss...@apache.org> Committed: Thu Mar 31 14:53:59 2016 -0700 ---------------------------------------------------------------------- data/conf/llap/hive-site.xml | 44 ++++++++++ data/conf/llap/llap-daemon-site.xml | 61 -------------- itests/hive-unit/pom.xml | 11 ++- .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 12 ++- itests/util/pom.xml | 11 +++ .../apache/hadoop/hive/llap/LlapItUtils.java | 84 ++++++++++++++++++++ .../org/apache/hadoop/hive/ql/QTestUtil.java | 17 ++-- .../configuration/LlapDaemonConfiguration.java | 8 ++ .../apache/hadoop/hive/shims/Hadoop23Shims.java | 66 +-------------- .../apache/hadoop/hive/shims/HadoopShims.java | 2 +- 10 files changed, 175 insertions(+), 141 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/184e0e1d/data/conf/llap/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml index c2bef58..72bdcfb 100644 --- a/data/conf/llap/hive-site.xml +++ b/data/conf/llap/hive-site.xml @@ -273,4 +273,48 @@ <value>false</value> </property> +<!-- llap properties --> + +<property> + <name>hive.llap.daemon.service.hosts</name> + <value>localhost</value> +</property> + +<property> + <name>hive.llap.daemon.service.port</name> + <value>0</value> +</property> + +<property> + <name>hive.llap.daemon.num.executors</name> + <value>4</value> +</property> + +<property> + <name>hive.llap.daemon.task.scheduler.wait.queue.size</name> + <value>4</value> +</property> + +<property> + <name>hive.llap.cache.allow.synthetic.fileid</name> + <value>true</value> +</property> + +<!-- hadoop IPC options --> +<property> + <name>ipc.client.low-latency</name> + <value>true</value> + <description> </description> +</property> +<property> + <name>ipc.client.tcpnodelay</name> + <value>true</value> + <description> </description> +</property> +<property> + <name>ipc.clients-per-factory</name> + <value>4</value> + <description> </description> +</property> + </configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/184e0e1d/data/conf/llap/llap-daemon-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/llap/llap-daemon-site.xml b/data/conf/llap/llap-daemon-site.xml deleted file mode 100644 index 98c0f2b..0000000 --- a/data/conf/llap/llap-daemon-site.xml +++ /dev/null @@ -1,61 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<configuration> -<property> - <name>hive.llap.daemon.service.hosts</name> - <value>localhost</value> -</property> - -<property> - <name>hive.llap.daemon.service.port</name> - <value>0</value> -</property> - -<property> - <name>hive.llap.daemon.num.executors</name> - <value>4</value> -</property> - -<property> - <name>hive.llap.daemon.task.scheduler.wait.queue.size</name> - <value>4</value> -</property> - -<property> - <name>hive.llap.cache.allow.synthetic.fileid</name> - <value>true</value> -</property> - -<!-- hadoop IPC options --> -<property> - <name>ipc.client.low-latency</name> - <value>true</value> - <description> </description> -</property> -<property> - <name>ipc.client.tcpnodelay</name> - <value>true</value> - <description> </description> -</property> -<property> - <name>ipc.clients-per-factory</name> - <value>4</value> - <description> </description> -</property> -</configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/184e0e1d/itests/hive-unit/pom.xml ---------------------------------------------------------------------- diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml index 97786d9..7219f1d 100644 --- a/itests/hive-unit/pom.xml +++ b/itests/hive-unit/pom.xml @@ -75,6 +75,11 @@ <artifactId>hive-hcatalog-streaming</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-it-util</artifactId> + <version>${project.version}</version> + </dependency> <!-- inter-project --> <dependency> <groupId>org.apache.hadoop</groupId> @@ -123,12 +128,6 @@ </dependency> <dependency> <groupId>org.apache.hive</groupId> - <artifactId>hive-it-util</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> <artifactId>hive-jdbc</artifactId> <version>${project.version}</version> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/hive/blob/184e0e1d/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index a9d9c76..eca2317 100644 --- a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -31,6 +31,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.llap.LlapItUtils; +import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -60,6 +62,7 @@ public class MiniHS2 extends AbstractHiveService { private static final AtomicLong hs2Counter = new AtomicLong(); private MiniMrShim mr; private MiniDFSShim dfs; + private MiniLlapCluster llapCluster = null; private final FileSystem localFS; private boolean useMiniKdc = false; private final String serverPrincipal; @@ -187,13 +190,15 @@ public class MiniHS2 extends AbstractHiveService { // Initialize the execution engine based on cluster type switch (miniClusterType) { case TEZ: - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString, false); + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); break; case LLAP: if (usePortsFromConf) { hiveConf.setBoolean("minillap.usePortsFromConf", true); } - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString, true); + llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null); + + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); break; case MR: mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1); @@ -285,6 +290,9 @@ public class MiniHS2 extends AbstractHiveService { hiveServer2.stop(); setStarted(false); try { + if (llapCluster != null) { + llapCluster.stop(); + } if (mr != null) { mr.shutdown(); mr = null; http://git-wip-us.apache.org/repos/asf/hive/blob/184e0e1d/itests/util/pom.xml ---------------------------------------------------------------------- diff --git a/itests/util/pom.xml b/itests/util/pom.xml index aaafc0a..4789586 100644 --- a/itests/util/pom.xml +++ b/itests/util/pom.xml @@ -97,6 +97,17 @@ <classifier>tests</classifier> </dependency> <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-llap-server</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-llap-server</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> http://git-wip-us.apache.org/repos/asf/hive/blob/184e0e1d/itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java b/itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java new file mode 100644 index 0000000..cb4aba5 --- /dev/null +++ b/itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.llap; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.Iterator; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; +import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; +import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LlapItUtils { + + private static final Logger LOG = LoggerFactory.getLogger(LlapItUtils.class); + + public static MiniLlapCluster startAndGetMiniLlapCluster(Configuration conf, String confDir) throws + IOException { + MiniLlapCluster llapCluster; + LOG.info("Using conf dir: {}", confDir); + if (confDir != null && !confDir.isEmpty()) { + conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + + "/tez-site.xml")); + } + + Configuration daemonConf = new LlapDaemonConfiguration(conf); + final String clusterName = "llap"; + final long maxMemory = LlapDaemon.getTotalHeapSize(); + // 15% for io cache + final long memoryForCache = (long) (0.15f * maxMemory); + // 75% for 4 executors + final long totalExecutorMemory = (long) (0.75f * maxMemory); + final int numExecutors = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_NUM_EXECUTORS); + final boolean asyncIOEnabled = true; + // enabling this will cause test failures in Mac OS X + final boolean directMemoryEnabled = false; + final int numLocalDirs = 1; + LOG.info("MiniLlap Configs - maxMemory: " + maxMemory + " memoryForCache: " + memoryForCache + + " totalExecutorMemory: " + totalExecutorMemory + " numExecutors: " + numExecutors + + " asyncIOEnabled: " + asyncIOEnabled + " directMemoryEnabled: " + directMemoryEnabled + + " numLocalDirs: " + numLocalDirs); + llapCluster = MiniLlapCluster.create(clusterName, + numExecutors, + totalExecutorMemory, + asyncIOEnabled, + directMemoryEnabled, + memoryForCache, + numLocalDirs); + llapCluster.init(daemonConf); + llapCluster.start(); + + // Augment conf with the settings from the started llap configuration. + Configuration llapConf = llapCluster.getClusterSpecificConfiguration(); + Iterator<Map.Entry<String, String>> confIter = llapConf.iterator(); + while (confIter.hasNext()) { + Map.Entry<String, String> entry = confIter.next(); + conf.set(entry.getKey(), entry.getValue()); + } + return llapCluster; + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/184e0e1d/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 5e81e98..5ccbcba 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -51,6 +51,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.Deque; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -82,6 +83,10 @@ import org.apache.hadoop.hive.common.io.SortAndDigestPrintStream; import org.apache.hadoop.hive.common.io.SortPrintStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.llap.LlapItUtils; +import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; +import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; +import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon; import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Index; @@ -166,6 +171,7 @@ public class QTestUtil { private HadoopShims.MiniMrShim mr = null; private HadoopShims.MiniDFSShim dfs = null; private HadoopShims.HdfsEncryptionShim hes = null; + private MiniLlapCluster llapCluster = null; private String hadoopVer = null; private QTestSetup setup = null; private TezSessionState tezSessionState = null; @@ -429,15 +435,10 @@ public class QTestUtil { conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml")); } - mr = shims.getMiniTezCluster(conf, 4, uriString, false); + mr = shims.getMiniTezCluster(conf, 4, uriString); } else if (clusterType == MiniClusterType.llap) { - if (confDir != null && !confDir.isEmpty()) { - conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() - + "/tez-site.xml")); - conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() - + "/llap-daemon-site.xml")); - } - mr = shims.getMiniTezCluster(conf, 2, uriString, true); + llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, confDir); + mr = shims.getMiniTezCluster(conf, 2, uriString); } else if (clusterType == MiniClusterType.miniSparkOnYarn) { mr = shims.getMiniSparkCluster(conf, 4, uriString, 1); } else { http://git-wip-us.apache.org/repos/asf/hive/blob/184e0e1d/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java b/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java index 51e8509..88f3b19 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/configuration/LlapDaemonConfiguration.java @@ -14,6 +14,7 @@ package org.apache.hadoop.hive.llap.configuration; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.conf.HiveConf; @@ -21,6 +22,7 @@ import org.apache.hadoop.hive.conf.HiveConf; /** * Configuration for LLAP daemon processes only. This should not be used by any clients. */ +@InterfaceAudience.Private public class LlapDaemonConfiguration extends Configuration { @InterfaceAudience.Private @@ -46,4 +48,10 @@ public class LlapDaemonConfiguration extends Configuration { } addResource(LLAP_DAEMON_SITE); } + + @VisibleForTesting + public LlapDaemonConfiguration(Configuration conf) { + this(); + addResource(conf); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/184e0e1d/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java ---------------------------------------------------------------------- diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 9a3a31c..e028212 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -370,8 +370,8 @@ public class Hadoop23Shims extends HadoopShimsSecure { */ @Override public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, boolean isLlap) throws IOException { - return new MiniTezShim(conf, numberOfTaskTrackers, nameNode, isLlap); + String nameNode) throws IOException { + return new MiniTezShim(conf, numberOfTaskTrackers, nameNode); } /** @@ -381,11 +381,8 @@ public class Hadoop23Shims extends HadoopShimsSecure { private final MiniTezCluster mr; private final Configuration conf; - private Class<?> miniLlapKlass; - private Object miniLlapCluster; - public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, - boolean isLlap) throws IOException { + public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); @@ -393,54 +390,6 @@ public class Hadoop23Shims extends HadoopShimsSecure { mr.init(conf); mr.start(); this.conf = mr.getConfig(); - if (isLlap) { - createAndLaunchLlapDaemon(this.conf); - } else { - miniLlapCluster = null; - } - } - - private void createAndLaunchLlapDaemon(final Configuration conf) - throws IOException { - try { - final String clusterName = "llap"; - Class<?> llapDaemonKlass = - Class.forName("org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon", - false, ShimLoader.class.getClassLoader()); - Method totalMemMethod = llapDaemonKlass.getMethod("getTotalHeapSize"); - final long maxMemory = (long) totalMemMethod.invoke(null); - // 15% for io cache - final long memoryForCache = (long) (0.15f * maxMemory); - // 75% for executors - final long totalExecutorMemory = (long) (0.75f * maxMemory); - final int numExecutors = conf.getInt("llap.daemon.num.executors", 4); - final boolean asyncIOEnabled = true; - // enabling this will cause test failures in Mac OS X - final boolean directMemoryEnabled = false; - final int numLocalDirs = 1; - LOG.info("MiniLlap Configs - maxMemory: " + maxMemory + " memoryForCache: " + memoryForCache - + " totalExecutorMemory: " + totalExecutorMemory + " numExecutors: " + numExecutors - + " asyncIOEnabled: " + asyncIOEnabled + " directMemoryEnabled: " + directMemoryEnabled - + " numLocalDirs: " + numLocalDirs); - - miniLlapKlass = Class.forName("org.apache.hadoop.hive.llap.daemon.MiniLlapCluster", - false, ShimLoader.class.getClassLoader()); - Method create = miniLlapKlass.getMethod("createAndLaunch", new Class[]{Configuration.class, - String.class, Integer.TYPE, Long.TYPE, Boolean.TYPE, Boolean.TYPE, - Long.TYPE, Integer.TYPE}); - miniLlapCluster = create.invoke(null, - conf, - clusterName, - numExecutors, - totalExecutorMemory, - asyncIOEnabled, - directMemoryEnabled, - memoryForCache, - numLocalDirs); - } catch (Exception e) { - LOG.error("Unable to create MiniLlapCluster. Exception: " + e.getMessage()); - throw new IOException(e); - } } @Override @@ -458,15 +407,6 @@ public class Hadoop23Shims extends HadoopShimsSecure { @Override public void shutdown() throws IOException { mr.stop(); - - if (miniLlapKlass != null && miniLlapCluster != null) { - try { - Method stop = miniLlapKlass.getMethod("stop", new Class[]{}); - stop.invoke(miniLlapCluster); - } catch (Exception e) { - LOG.error("Unable to stop llap daemon. Exception: " + e.getMessage()); - } - } } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/184e0e1d/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java ---------------------------------------------------------------------- diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 7a5a9b5..a44d0c0 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -90,7 +90,7 @@ public interface HadoopShims { String nameNode, int numDir) throws IOException; public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, boolean isLlap) throws IOException; + String nameNode) throws IOException; public MiniMrShim getMiniSparkCluster(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException;
