Repository: zeppelin Updated Branches: refs/heads/master a2a229d16 -> 1e478b229
ZEPPELIN-1175. AM log is not available for yarn-client mode ### What is this PR for? For now, we share the same class path for zeppelin server and remote interpreter process. The cause the issue that AM log is not available for yarn-client mode because the yarn app also use the `ZEPPELIN_HOME/conf/log4j.properties` which is only for zeppelin server. So this PR just distinguish the CLASSPATH of zeppelin server and remote interpreter process. I use `ZEPPELIN_INTP_CLASSPATH` to represent the classpath of remote interpreter process and won't include `ZEPPELIN_HOME/conf/log4j.properties` in `ZEPPELIN_INTP_CLASSPATH`. ### What type of PR is it? [Improvement] ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-1175 ### How should this be tested? Tested manually. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? Yes, if user put custom config file (hive-site.xml) under ZEPPELIN_HOME/conf, it won't take effect after this PR * Does this needs documentation? Yes Author: Jeff Zhang <[email protected]> Closes #1228 from zjffdu/ZEPPELIN-1175 and squashes the following commits: 0973477 [Jeff Zhang] ZEPPELIN-1175. AM log is not available for yarn-client mode Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/1e478b22 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/1e478b22 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/1e478b22 Branch: refs/heads/master Commit: 1e478b2293ba29f77b03eed81bfb2f88028d8fa9 Parents: a2a229d Author: Jeff Zhang <[email protected]> Authored: Tue Jul 26 10:41:50 2016 +0800 Committer: Prabhjyot Singh <[email protected]> Committed: Mon Aug 1 11:42:03 2016 +0530 ---------------------------------------------------------------------- bin/common.sh | 13 +++++++++++++ bin/interpreter.sh | 36 ++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e478b22/bin/common.sh ---------------------------------------------------------------------- diff --git a/bin/common.sh b/bin/common.sh index a70be7c..592aa1c 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -74,6 +74,13 @@ function addEachJarInDirRecursive(){ fi } +function addEachJarInDirRecursiveForIntp(){ + if [[ -d "${1}" ]]; then + for jar in $(find -L "${1}" -type f -name '*jar'); do + ZEPPELIN_INTP_CLASSPATH="$jar:$ZEPPELIN_INTP_CLASSPATH" + done + fi +} function addJarInDir(){ if [[ -d "${1}" ]]; then @@ -81,6 +88,12 @@ function addJarInDir(){ fi } +function addJarInDirForIntp() { + if [[ -d "${1}" ]]; then + ZEPPELIN_INTP_CLASSPATH="${1}/*:${ZEPPELIN_INTP_CLASSPATH}" + fi +} + ZEPPELIN_COMMANDLINE_MAIN=org.apache.zeppelin.utils.CommandLineUtils function getZeppelinVersion(){ http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e478b22/bin/interpreter.sh ---------------------------------------------------------------------- diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 69c94f6..38d0f69 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -53,18 +53,18 @@ fi . "${bin}/common.sh" -ZEPPELIN_CLASSPATH+=":${ZEPPELIN_CONF_DIR}" +ZEPPELIN_INTP_CLASSPATH="" # construct classpath if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then - ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" + ZEPPELIN_INTP_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" else ZEPPELIN_INTERPRETER_JAR="$(ls ${ZEPPELIN_HOME}/lib/zeppelin-interpreter*.jar)" - ZEPPELIN_CLASSPATH+=":${ZEPPELIN_INTERPRETER_JAR}" + ZEPPELIN_INTP_CLASSPATH+=":${ZEPPELIN_INTERPRETER_JAR}" fi -addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" -addJarInDir "${INTERPRETER_DIR}" +addJarInDirForIntp "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" +addJarInDirForIntp "${INTERPRETER_DIR}" HOSTNAME=$(hostname) ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer @@ -85,7 +85,7 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then export SPARK_SUBMIT="${SPARK_HOME}/bin/spark-submit" SPARK_APP_JAR="$(ls ${ZEPPELIN_HOME}/interpreter/spark/zeppelin-spark*.jar)" # This will evantually passes SPARK_APP_JAR to classpath of SparkIMain - ZEPPELIN_CLASSPATH+=${SPARK_APP_JAR} + ZEPPELIN_INTP_CLASSPATH+=":${SPARK_APP_JAR}" pattern="$SPARK_HOME/python/lib/py4j-*-src.zip" py4j=($pattern) @@ -96,14 +96,14 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then # add Hadoop jars into classpath if [[ -n "${HADOOP_HOME}" ]]; then # Apache - addEachJarInDirRecursive "${HADOOP_HOME}/share" + addEachJarInDirRecursiveForIntp "${HADOOP_HOME}/share" # CDH - addJarInDir "${HADOOP_HOME}" - addJarInDir "${HADOOP_HOME}/lib" + addJarInDirForIntp "${HADOOP_HOME}" + addJarInDirForIntp "${HADOOP_HOME}/lib" fi - addJarInDir "${INTERPRETER_DIR}/dep" + addJarInDirForIntp "${INTERPRETER_DIR}/dep" pattern="${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-*-src.zip" py4j=($pattern) @@ -127,29 +127,29 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then fi if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then - ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}" + ZEPPELIN_INTP_CLASSPATH+=":${HADOOP_CONF_DIR}" fi - export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}" + export SPARK_CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}" fi elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then if [[ -n "${HBASE_CONF_DIR}" ]]; then - ZEPPELIN_CLASSPATH+=":${HBASE_CONF_DIR}" + ZEPPELIN_INTP_CLASSPATH+=":${HBASE_CONF_DIR}" elif [[ -n "${HBASE_HOME}" ]]; then - ZEPPELIN_CLASSPATH+=":${HBASE_HOME}/conf" + ZEPPELIN_INTP_CLASSPATH+=":${HBASE_HOME}/conf" else echo "HBASE_HOME and HBASE_CONF_DIR are not set, configuration might not be loaded" fi fi -addJarInDir "${LOCAL_INTERPRETER_REPO}" +addJarInDirForIntp "${LOCAL_INTERPRETER_REPO}" -CLASSPATH+=":${ZEPPELIN_CLASSPATH}" +CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}" if [[ -n "${SPARK_SUBMIT}" ]]; then - ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & + ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & else - ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} & + ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} & fi pid=$!
