Repository: incubator-zeppelin Updated Branches: refs/heads/master aaee71977 -> dcb03a9fe
Simplify classpath. When Zeppelin constructs JVM CLASSPATH, it adds path of every single jar. That creates really long CLASSPATH and it may cause some problem like https://issues.apache.org/jira/browse/ZEPPELIN-68. This PR solves it by constructing CLASSPATH with wildcard. Another problem is, Zeppelin constructs CLASSPATH with duplicated entry or not related entry. This PR solves it by constructing CLASSPATH not in common.sh but in each script zeppelin-daemon.sh, interpreter.sh, zeppelin.sh while they need different entry in CLASSPATH. Author: Lee moon soo <[email protected]> Closes #65 from Leemoonsoo/ZEPPELIN-68 and squashes the following commits: ae3ea5a [Lee moon soo] Print rat.txt when build failed 4eee837 [Lee moon soo] ZEPPELIN-68 simplify classpath Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/dcb03a9f Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/dcb03a9f Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/dcb03a9f Branch: refs/heads/master Commit: dcb03a9fe66d753348154c41f5b40927125723aa Parents: aaee719 Author: Lee moon soo <[email protected]> Authored: Wed May 13 01:45:30 2015 +0200 Committer: Lee moon soo <[email protected]> Committed: Wed May 13 14:16:19 2015 +0900 ---------------------------------------------------------------------- .travis.yml | 1 + bin/common.sh | 23 ++++++----------------- bin/interpreter.sh | 9 +++++++-- bin/zeppelin-daemon.sh | 22 ++++++++++++++++++++++ bin/zeppelin.sh | 22 ++++++++++++++++++++++ 5 files changed, 58 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/dcb03a9f/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 7eb5935..b09e228 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,7 @@ script: - mvn verify -Pusing-packaged-distr -B after_failure: + - cat target/rat.txt - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/dcb03a9f/bin/common.sh ---------------------------------------------------------------------- diff --git a/bin/common.sh b/bin/common.sh index a825b76..8087e9d 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -70,28 +70,19 @@ fi ZEPPELIN_CLASSPATH+=":${ZEPPELIN_CONF_DIR}" -function addJarInDir(){ +function addEachJarInDir(){ if [[ -d "${1}" ]]; then for jar in $(find -L "${1}" -maxdepth 1 -name '*jar'); do ZEPPELIN_CLASSPATH="$jar:$ZEPPELIN_CLASSPATH" done fi } - -addJarInDir "${ZEPPELIN_HOME}" -addJarInDir "${ZEPPELIN_HOME}/lib" -addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" -addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib" -addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib" -addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib" - -if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then - ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" -fi -if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then - ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes" -fi +function addJarInDir(){ + if [[ -d "${1}" ]]; then + export ZEPPELIN_CLASSPATH="${1}/*:${ZEPPELIN_CLASSPATH}" + fi +} if [[ ! -z "${SPARK_HOME}" ]] && [[ -d "${SPARK_HOME}" ]]; then addJarInDir "${SPARK_HOME}" @@ -106,8 +97,6 @@ if [[ ! -z "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then fi export ZEPPELIN_CLASSPATH -export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}" -export CLASSPATH+=":${ZEPPELIN_CLASSPATH}" # Text encoding for # read/write job into files, http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/dcb03a9f/bin/interpreter.sh ---------------------------------------------------------------------- diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 025df36..c214c30 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -49,10 +49,16 @@ fi ZEPPELIN_CLASSPATH+=":${ZEPPELIN_CONF_DIR}" +# construct classpath +if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" +fi + addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" addJarInDir "${INTERPRETER_DIR}" -export CLASSPATH+=":${ZEPPELIN_CLASSPATH}" +export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}" +CLASSPATH+=":${ZEPPELIN_CLASSPATH}" HOSTNAME=$(hostname) ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer @@ -68,7 +74,6 @@ if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then fi - ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} -cp ${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} & pid=$! if [[ -z "${pid}" ]]; then http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/dcb03a9f/bin/zeppelin-daemon.sh ---------------------------------------------------------------------- diff --git a/bin/zeppelin-daemon.sh b/bin/zeppelin-daemon.sh index 56e6c88..2440c12 100755 --- a/bin/zeppelin-daemon.sh +++ b/bin/zeppelin-daemon.sh @@ -54,6 +54,28 @@ ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}. ZEPPELIN_MAIN=org.apache.zeppelin.server.ZeppelinServer JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}" +# construct classpath +if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" +fi + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" +fi + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes" +fi + +addJarInDir "${ZEPPELIN_HOME}" +addJarInDir "${ZEPPELIN_HOME}/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib" + +CLASSPATH+=":${ZEPPELIN_CLASSPATH}" + if [[ "${ZEPPELIN_NICENESS}" = "" ]]; then export ZEPPELIN_NICENESS=0 fi http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/dcb03a9f/bin/zeppelin.sh ---------------------------------------------------------------------- diff --git a/bin/zeppelin.sh b/bin/zeppelin.sh index d81556c..eb2f691 100755 --- a/bin/zeppelin.sh +++ b/bin/zeppelin.sh @@ -51,6 +51,28 @@ LOG="${ZEPPELIN_LOG_DIR}/zeppelin-cli-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.out" ZEPPELIN_SERVER=org.apache.zeppelin.server.ZeppelinServer JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}" +# construct classpath +if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" +fi + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" +fi + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes" +fi + +addJarInDir "${ZEPPELIN_HOME}" +addJarInDir "${ZEPPELIN_HOME}/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib" + +CLASSPATH+=":${ZEPPELIN_CLASSPATH}" + if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then echo "Log dir doesn't exist, create ${ZEPPELIN_LOG_DIR}" $(mkdir -p "${ZEPPELIN_LOG_DIR}")
