Repository: incubator-predictionio Updated Branches: refs/heads/develop cc4e2e0a2 -> a81302171
[PIO-72] Fix class loading for pio-shell Closes #401 Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/d894b72a Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/d894b72a Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/d894b72a Branch: refs/heads/develop Commit: d894b72af8975aef872aae5311f3c7f75aa00961 Parents: cc4e2e0 Author: Chan Lee <[email protected]> Authored: Thu Jul 27 14:54:37 2017 -0700 Committer: Donald Szeto <[email protected]> Committed: Thu Jul 27 14:54:37 2017 -0700 ---------------------------------------------------------------------- bin/compute-classpath.sh | 23 +++++++++----------- bin/pio-class | 2 -- bin/pio-shell | 5 ++--- tests/build_docker.sh | 13 ++++++----- .../org/apache/predictionio/tools/Common.scala | 19 +++++++++------- 5 files changed, 31 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/compute-classpath.sh ---------------------------------------------------------------------- diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index 032d111..cd0527f 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -17,20 +17,11 @@ # limitations under the License. # -SCALA_VERSION=2.10 - # Figure out where PredictionIO is installed FWDIR="$(cd `dirname $0`/..; pwd)" . ${FWDIR}/bin/load-pio-env.sh -# Build up classpath -CLASSPATH="${PIO_CONF_DIR}" - -CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*:${FWDIR}/lib/spark/*" - -ASSEMBLY_DIR="${FWDIR}/assembly" - if [ -n "$JAVA_HOME" ]; then JAR_CMD="$JAVA_HOME/bin/jar" else @@ -41,12 +32,19 @@ fi if [ -f "${FWDIR}/RELEASE" ]; then assembly_folder="${FWDIR}"/lib else - assembly_folder="${ASSEMBLY_DIR}" + assembly_folder="${FWDIR}"/assembly/src/universal/lib fi -ASSEMBLY_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null) +MAIN_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null) +DATA_JARS=$(ls "${assembly_folder}"/spark/pio-data-*assembly*.jar 2>/dev/null) +# Comma-separated list of assembly jars for submitting to spark-shell +ASSEMBLY_JARS=$(printf "${MAIN_JAR}\n${DATA_JARS}" | paste -sd "," -) -CLASSPATH="$CLASSPATH:${ASSEMBLY_JAR}" +# Build up classpath +CLASSPATH="${PIO_CONF_DIR}" +CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*" +CLASSPATH="$CLASSPATH:${assembly_folder}/spark/*" +CLASSPATH="$CLASSPATH:${MAIN_JAR}" # Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail ! Note, this # assumes that there is either a HADOOP_CONF_DIR or YARN_CONF_DIR which hosts @@ -70,5 +68,4 @@ if [ -n "$MYSQL_JDBC_DRIVER" ]; then CLASSPATH="$CLASSPATH:$MYSQL_JDBC_DRIVER" fi - echo "$CLASSPATH" http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/pio-class ---------------------------------------------------------------------- diff --git a/bin/pio-class b/bin/pio-class index 8bde407..94cc520 100755 --- a/bin/pio-class +++ b/bin/pio-class @@ -22,8 +22,6 @@ case "`uname`" in CYGWIN*) cygwin=true;; esac -SCALA_VERSION=2.10 - # Figure out where PredictionIO is installed FWDIR="$(cd `dirname $0`/..; pwd)" http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/pio-shell ---------------------------------------------------------------------- diff --git a/bin/pio-shell b/bin/pio-shell index d47008f..5a5745a 100755 --- a/bin/pio-shell +++ b/bin/pio-shell @@ -55,11 +55,10 @@ fi if [[ "$1" == "--with-spark" ]] then echo "Starting the PIO shell with the Apache Spark Shell." - # compute the $ASSEMPLY_JAR, the location of the assemply jar, with - # bin/compute-classpath.sh + # Get paths of assembly jars to pass to spark-shell . ${PIO_HOME}/bin/compute-classpath.sh shift - $SPARK_HOME/bin/spark-shell --jars $ASSEMBLY_JAR $@ + ${SPARK_HOME}/bin/spark-shell --jars ${ASSEMBLY_JARS} $@ else echo -e "\033[0;33mStarting the PIO shell without Apache Spark.\033[0m" echo -e "\033[0;33mIf you need the Apache Spark library, run 'pio-shell --with-spark [spark-submit arguments...]'.\033[0m" http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/tests/build_docker.sh ---------------------------------------------------------------------- diff --git a/tests/build_docker.sh b/tests/build_docker.sh index d467e97..b17a304 100755 --- a/tests/build_docker.sh +++ b/tests/build_docker.sh @@ -38,11 +38,14 @@ fi -Dhadoop.version=$PIO_HADOOP_VERSION \ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION sbt/sbt clean storage/clean -rm -rf assembly/*.jar -cp dist/lib/*.jar assembly/ -rm -rf lib/spark -mkdir -p lib/spark -cp dist/lib/spark/*.jar lib/spark + +assembly_folder=assembly/src/universal/lib +rm -rf ${assembly_folder}/*.jar +rm -rf ${assembly_folder}/spark +mkdir -p ${assembly_folder}/spark + +cp dist/lib/*.jar ${assembly_folder} +cp dist/lib/spark/*.jar ${assembly_folder}/spark rm *.tar.gz docker build -t predictionio/pio . popd http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala ---------------------------------------------------------------------- diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala index 0ddd644..047b8dd 100644 --- a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala +++ b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala @@ -65,6 +65,15 @@ object Common extends EitherLogging { } } + def getCoreDir(pioHome: String): String = { + if (new File(pioHome + File.separator + "RELEASE").exists) { + pioHome + File.separator + "lib" + } else { + Array(pioHome, "assembly", "src", "universal", "lib") + .mkString(File.separator) + } + } + def getEngineDirPath(directory: Option[String]): String = { new File(directory.getOrElse(".")).getCanonicalPath } @@ -97,18 +106,12 @@ object Common extends EitherLogging { def jarFilesAt(path: File): Array[File] = path.listFiles filter { _.getName.toLowerCase.endsWith(".jar") } - jarFilesAt(new File(pioHome, "lib/spark")) + jarFilesAt(new File(getCoreDir(pioHome) + File.separator + "spark")) } def coreAssembly(pioHome: String): Expected[File] = { val core = s"pio-assembly-${BuildInfo.version}.jar" - val coreDir = - if (new File(pioHome + File.separator + "RELEASE").exists) { - new File(pioHome + File.separator + "lib") - } else { - new File(pioHome + File.separator + "assembly") - } - val coreFile = new File(coreDir, core) + val coreFile = new File(getCoreDir(pioHome), core) if (coreFile.exists) { Right(coreFile) } else {
