Repository: zeppelin Updated Branches: refs/heads/branch-0.6 144c2fb6f -> c327a7ca8
ZEPPELIN-1175. AM log is not available for yarn-client mode ### What is this PR for? For now, we share the same class path for zeppelin server and remote interpreter process. The cause the issue that AM log is not available for yarn-client mode because the yarn app also use the `ZEPPELIN_HOME/conf/log4j.properties` which is only for zeppelin server. So this PR just distinguish the CLASSPATH of zeppelin server and remote interpreter process. I use `ZEPPELIN_INTP_CLASSPATH` to represent the classpath of remote interpreter process and won't include `ZEPPELIN_HOME/conf/log4j.properties` in `ZEPPELIN_INTP_CLASSPATH`. ### What type of PR is it? [Improvement] ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-1175 ### How should this be tested? Tested manually. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? Yes, if user put custom config file (hive-site.xml) under ZEPPELIN_HOME/conf, it won't take effect after this PR * Does this needs documentation? Yes Author: Jeff Zhang <[email protected]> Closes #1228 from zjffdu/ZEPPELIN-1175 and squashes the following commits: 0973477 [Jeff Zhang] ZEPPELIN-1175. AM log is not available for yarn-client mode (cherry picked from commit 1e478b2293ba29f77b03eed81bfb2f88028d8fa9) Signed-off-by: Mina Lee <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/c327a7ca Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/c327a7ca Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/c327a7ca Branch: refs/heads/branch-0.6 Commit: c327a7ca8f5f9481d7389b7f29d93fd6542db80f Parents: 144c2fb Author: Jeff Zhang <[email protected]> Authored: Tue Jul 26 10:41:50 2016 +0800 Committer: Mina Lee <[email protected]> Committed: Thu Aug 11 06:43:32 2016 +0200 ---------------------------------------------------------------------- bin/common.sh | 13 +++++++++++++ bin/interpreter.sh | 36 ++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c327a7ca/bin/common.sh ---------------------------------------------------------------------- diff --git a/bin/common.sh b/bin/common.sh index a70be7c..592aa1c 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -74,6 +74,13 @@ function addEachJarInDirRecursive(){ fi } +function addEachJarInDirRecursiveForIntp(){ + if [[ -d "${1}" ]]; then + for jar in $(find -L "${1}" -type f -name '*jar'); do + ZEPPELIN_INTP_CLASSPATH="$jar:$ZEPPELIN_INTP_CLASSPATH" + done + fi +} function addJarInDir(){ if [[ -d "${1}" ]]; then @@ -81,6 +88,12 @@ function addJarInDir(){ fi } +function addJarInDirForIntp() { + if [[ -d "${1}" ]]; then + ZEPPELIN_INTP_CLASSPATH="${1}/*:${ZEPPELIN_INTP_CLASSPATH}" + fi +} + ZEPPELIN_COMMANDLINE_MAIN=org.apache.zeppelin.utils.CommandLineUtils function getZeppelinVersion(){ http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c327a7ca/bin/interpreter.sh ---------------------------------------------------------------------- diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 69c94f6..38d0f69 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -53,18 +53,18 @@ fi . "${bin}/common.sh" -ZEPPELIN_CLASSPATH+=":${ZEPPELIN_CONF_DIR}" +ZEPPELIN_INTP_CLASSPATH="" # construct classpath if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then - ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" + ZEPPELIN_INTP_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" else ZEPPELIN_INTERPRETER_JAR="$(ls ${ZEPPELIN_HOME}/lib/zeppelin-interpreter*.jar)" - ZEPPELIN_CLASSPATH+=":${ZEPPELIN_INTERPRETER_JAR}" + ZEPPELIN_INTP_CLASSPATH+=":${ZEPPELIN_INTERPRETER_JAR}" fi -addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" -addJarInDir "${INTERPRETER_DIR}" +addJarInDirForIntp "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" +addJarInDirForIntp "${INTERPRETER_DIR}" HOSTNAME=$(hostname) ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer @@ -85,7 +85,7 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then export SPARK_SUBMIT="${SPARK_HOME}/bin/spark-submit" SPARK_APP_JAR="$(ls ${ZEPPELIN_HOME}/interpreter/spark/zeppelin-spark*.jar)" # This will evantually passes SPARK_APP_JAR to classpath of SparkIMain - ZEPPELIN_CLASSPATH+=${SPARK_APP_JAR} + ZEPPELIN_INTP_CLASSPATH+=":${SPARK_APP_JAR}" pattern="$SPARK_HOME/python/lib/py4j-*-src.zip" py4j=($pattern) @@ -96,14 +96,14 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then # add Hadoop jars into classpath if [[ -n "${HADOOP_HOME}" ]]; then # Apache - addEachJarInDirRecursive "${HADOOP_HOME}/share" + addEachJarInDirRecursiveForIntp "${HADOOP_HOME}/share" # CDH - addJarInDir "${HADOOP_HOME}" - addJarInDir "${HADOOP_HOME}/lib" + addJarInDirForIntp "${HADOOP_HOME}" + addJarInDirForIntp "${HADOOP_HOME}/lib" fi - addJarInDir "${INTERPRETER_DIR}/dep" + addJarInDirForIntp "${INTERPRETER_DIR}/dep" pattern="${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-*-src.zip" py4j=($pattern) @@ -127,29 +127,29 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then fi if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then - ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}" + ZEPPELIN_INTP_CLASSPATH+=":${HADOOP_CONF_DIR}" fi - export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}" + export SPARK_CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}" fi elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then if [[ -n "${HBASE_CONF_DIR}" ]]; then - ZEPPELIN_CLASSPATH+=":${HBASE_CONF_DIR}" + ZEPPELIN_INTP_CLASSPATH+=":${HBASE_CONF_DIR}" elif [[ -n "${HBASE_HOME}" ]]; then - ZEPPELIN_CLASSPATH+=":${HBASE_HOME}/conf" + ZEPPELIN_INTP_CLASSPATH+=":${HBASE_HOME}/conf" else echo "HBASE_HOME and HBASE_CONF_DIR are not set, configuration might not be loaded" fi fi -addJarInDir "${LOCAL_INTERPRETER_REPO}" +addJarInDirForIntp "${LOCAL_INTERPRETER_REPO}" -CLASSPATH+=":${ZEPPELIN_CLASSPATH}" +CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}" if [[ -n "${SPARK_SUBMIT}" ]]; then - ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & + ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & else - ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} & + ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} & fi pid=$!
