Author: gunther
Date: Tue Feb 24 03:23:23 2015
New Revision: 1661834
URL: http://svn.apache.org/r1661834
Log:
HIVE-9761: LLAP: Misc fixes to launch scripts, startup error handling
(Siddharth Seth via Gunther Hagleitner)
Modified:
hive/branches/llap/llap-server/bin/llap-daemon-env.sh
hive/branches/llap/llap-server/bin/llapDaemon.sh
hive/branches/llap/llap-server/bin/runLlapDaemon.sh
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
hive/branches/llap/llap-server/src/test/resources/llap-daemon-site.xml
Modified: hive/branches/llap/llap-server/bin/llap-daemon-env.sh
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-server/bin/llap-daemon-env.sh?rev=1661834&r1=1661833&r2=1661834&view=diff
==============================================================================
--- hive/branches/llap/llap-server/bin/llap-daemon-env.sh (original)
+++ hive/branches/llap/llap-server/bin/llap-daemon-env.sh Tue Feb 24 03:23:23
2015
@@ -1,11 +1,26 @@
-# Required
+# Path to the directory which containts hive-exec and hive-llap-server
#export LLAP_DAEMON_HOME=
-# TODO Remove this.
+
+#Path to the directory containing llap configs. Eventually, merge with
hive-site.
+#export LLAP_DAEMON_CONF_DIR=
+
+# Heap size in MB for the llap-daemon. Determined by #executors,
#memory-per-executor setup in llap-daemon-configuration.
+#export LLAP_DAEMON_HEAPSIZE=
+
+# Path to the BIN scripts. Ideally this should be the same as the hive bin
directories.
#export LLAP_DAEMON_BIN_HOME=
-# Optional
+# Set this to a path containing tez jars
+#export LLAP_DAEMON_USER_CLASSPATH=
+
+# Logger setup for LLAP daemon
+#export LLAP_DAEMON_LOGGER=INFO,RFA
+
+# Directory to which logs will be generated
#export LLAP_DAEMON_LOG_DIR=
+
+# Directory in which the pid file will be generated
#export LLAP_DAEMON_PID_DIR=
-#export LLAP_DAEMON_USER_CLASSPATH=
+
+# Additional JAVA_OPTS for the daemon process
#export LLAP_DAEMON_OPTS=
-#export LLAP_DAEMON_LOGGER=INFO,console
Modified: hive/branches/llap/llap-server/bin/llapDaemon.sh
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-server/bin/llapDaemon.sh?rev=1661834&r1=1661833&r2=1661834&view=diff
==============================================================================
--- hive/branches/llap/llap-server/bin/llapDaemon.sh (original)
+++ hive/branches/llap/llap-server/bin/llapDaemon.sh Tue Feb 24 03:23:23 2015
@@ -116,7 +116,8 @@ case $startStop in
#rotate_log $logLog
#rotate_log $logOut
echo starting llapdaemon, logging to $logLog and $logOut
- nohup nice -n $LLAP_DAEMON_NICENESS
"$LLAP_DAEMON_BIN_HOME"/bin/runLlapDaemon.sh run > "$logOut" 2>&1 < /dev/null &
+ export LLAP_DAEMON_LOGFILE=${LLAP_DAEMON_LOG_BASE}.log
+ nohup nice -n $LLAP_DAEMON_NICENESS
"$LLAP_DAEMON_BIN_HOME"/runLlapDaemon.sh run > "$logOut" 2>&1 < /dev/null &
echo $! > $pid
;;
Modified: hive/branches/llap/llap-server/bin/runLlapDaemon.sh
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-server/bin/runLlapDaemon.sh?rev=1661834&r1=1661833&r2=1661834&view=diff
==============================================================================
--- hive/branches/llap/llap-server/bin/runLlapDaemon.sh (original)
+++ hive/branches/llap/llap-server/bin/runLlapDaemon.sh Tue Feb 24 03:23:23 2015
@@ -72,7 +72,7 @@ if [ ! -n "${LLAP_DAEMON_LOGGER}" ]; the
LLAP_DAEMON_LOGGER=${LOG_LEVEL_DEFAULT}
fi
-CLASSPATH=${LLAP_DAEMON_CONF_DIR}:${LLAP_DAEMON_HOME}/*:${LLAP_DAEMON_HOME}/lib/*:`${HADOOP_PREFIX}/bin/hadoop
classpath`:.
+CLASSPATH=${LLAP_DAEMON_CONF_DIR}:${LLAP_DAEMON_HOME}/hive-llap-server-1.2.0-SNAPSHOT.jar:${LLAP_DAEMON_HOME}/hive-exec-1.2.0-SNAPSHOT.jar:`${HADOOP_PREFIX}/bin/hadoop
classpath`:.
if [ -n "LLAP_DAEMON_USER_CLASSPATH" ]; then
CLASSPATH=${CLASSPATH}:${LLAP_DAEMON_USER_CLASSPATH}
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java?rev=1661834&r1=1661833&r2=1661834&view=diff
==============================================================================
---
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java
(original)
+++
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java
Tue Feb 24 03:23:23 2015
@@ -30,7 +30,6 @@ public class LlapDaemonConfiguration ext
private static final String LLAP_DAEMON_PREFIX = "llap.daemon.";
- private static final String LLAP_DAEMON_AM_PREFIX = LLAP_DAEMON_PREFIX +
"am.";
private static final String LLAP_DAEMON_SITE = "llap-daemon-site.xml";
@@ -46,17 +45,20 @@ public class LlapDaemonConfiguration ext
// Section for configs used in AM and executors
public static final String LLAP_DAEMON_NUM_EXECUTORS = LLAP_DAEMON_PREFIX +
"num.executors";
public static final int LLAP_DAEMON_NUM_EXECUTORS_DEFAULT = 4;
+
public static final String LLAP_DAEMON_RPC_PORT = LLAP_DAEMON_PREFIX +
"rpc.port";
public static final int LLAP_DAEMON_RPC_PORT_DEFAULT = 15001;
- public static final String LLAP_DAEMON_MEMORY_PER_INSTANCE_MB =
LLAP_DAEMON_AM_PREFIX + "memory.per.instance.mb";
+
+ public static final String LLAP_DAEMON_MEMORY_PER_INSTANCE_MB =
LLAP_DAEMON_PREFIX + "memory.per.instance.mb";
public static final int LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT = 4096;
- public static final String LLAP_DAEMON_VCPUS_PER_INSTANCE =
LLAP_DAEMON_AM_PREFIX + "vcpus.per.instance";
+
+ public static final String LLAP_DAEMON_VCPUS_PER_INSTANCE =
LLAP_DAEMON_PREFIX + "vcpus.per.instance";
public static final int LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT = 4;
// Section for configs used in the AM //
- public static final String LLAP_DAEMON_AM_SERVICE_HOSTS =
LLAP_DAEMON_AM_PREFIX + "service.hosts";
- public static final String LLAP_DAEMON_AM_COMMUNICATOR_NUM_THREADS =
LLAP_DAEMON_AM_PREFIX + "communicator.num.threads";
+ public static final String LLAP_DAEMON_AM_SERVICE_HOSTS = LLAP_DAEMON_PREFIX
+ "service.hosts";
+ public static final String LLAP_DAEMON_AM_COMMUNICATOR_NUM_THREADS =
LLAP_DAEMON_PREFIX + "communicator.num.threads";
public static final int LLAP_DAEMON_AM_COMMUNICATOR_NUM_THREADS_DEFAULT = 5;
}
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java?rev=1661834&r1=1661833&r2=1661834&view=diff
==============================================================================
---
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
(original)
+++
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
Tue Feb 24 03:23:23 2015
@@ -64,7 +64,9 @@ public class LlapDaemon extends Abstract
"numExecutors=" + numExecutors +
", rpcListenerPort=" + rpcPort +
", workDirs=" + Arrays.toString(localDirs) +
- ", shufflePort=" + shufflePort);
+ ", shufflePort=" + shufflePort +
+ ", memoryConfigured=" + memoryAvailableBytes +
+ ", jvmAvailableMemory=" + jvmMax);
Preconditions.checkArgument(this.numExecutors > 0);
Preconditions.checkArgument(this.rpcPort > 1024 && this.rpcPort < 65536,
@@ -102,20 +104,24 @@ public class LlapDaemon extends Abstract
public static void main(String[] args) throws Exception {
- LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration();
+ try {
+ LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration();
- Configuration shuffleHandlerConf = new Configuration(daemonConf);
- shuffleHandlerConf.set(ShuffleHandler.SHUFFLE_HANDLER_LOCAL_DIRS,
- daemonConf.get(LlapDaemonConfiguration.LLAP_DAEMON_WORK_DIRS));
- ShuffleHandler.initializeAndStart(shuffleHandlerConf);
-
- LlapDaemon llapDaemon = new LlapDaemon(daemonConf);
- // TODO Get the PID - FWIW
-
- llapDaemon.init(new Configuration());
- llapDaemon.start();
- LOG.info("Started LlapDaemon");
- // Relying on the RPC threads to keep the service alive.
+ Configuration shuffleHandlerConf = new Configuration(daemonConf);
+ shuffleHandlerConf.set(ShuffleHandler.SHUFFLE_HANDLER_LOCAL_DIRS,
+ daemonConf.get(LlapDaemonConfiguration.LLAP_DAEMON_WORK_DIRS));
+ ShuffleHandler.initializeAndStart(shuffleHandlerConf);
+
+ LlapDaemon llapDaemon = new LlapDaemon(daemonConf);
+ llapDaemon.init(new Configuration());
+ llapDaemon.start();
+ LOG.info("Started LlapDaemon");
+ // Relying on the RPC threads to keep the service alive.
+ } catch (Throwable t) {
+ // TODO Replace this with a ExceptionHandler / ShutdownHook
+ LOG.warn("Failed to start LLAP Daemon with exception", t);
+ System.exit(-1);
+ }
}
Modified: hive/branches/llap/llap-server/src/test/resources/llap-daemon-site.xml
URL:
http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/test/resources/llap-daemon-site.xml?rev=1661834&r1=1661833&r2=1661834&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/test/resources/llap-daemon-site.xml
(original)
+++ hive/branches/llap/llap-server/src/test/resources/llap-daemon-site.xml Tue
Feb 24 03:23:23 2015
@@ -19,7 +19,17 @@
</property>
<property>
- <name>llap.daemon.am.service.hosts</name>
+ <name>llap.daemon.num.executors</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <name>llap.daemon.memory.per.instance.mb</name>
+ <value>3072</value>
+ </property>
+
+ <property>
+ <name>llap.daemon.service.hosts</name>
<value>localhost</value>
<description>Comma separate list of nodes running daemons</description>
</property>
