Repository: bigtop Updated Branches: refs/heads/master ccc1181a0 -> 42ac3a5c1
BIGTOP-1640: Add missing files for Apache Spark Signed-off-by: Evans Ye <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/42ac3a5c Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/42ac3a5c Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/42ac3a5c Branch: refs/heads/master Commit: 42ac3a5c1ceca3605741674a7bda27579734880d Parents: ccc1181 Author: Youngwoo Kim <[email protected]> Authored: Fri Jan 30 19:32:45 2015 +0900 Committer: Evans Ye <[email protected]> Committed: Tue Feb 3 05:42:59 2015 +0000 ---------------------------------------------------------------------- .../src/common/spark/compute-classpath.sh | 6 +-- .../src/common/spark/do-component-build | 2 +- .../src/common/spark/install_spark.sh | 43 ++++++++------------ .../src/deb/spark/spark-core.install | 8 ++-- bigtop-packages/src/rpm/spark/SPECS/spark.spec | 3 ++ 5 files changed, 26 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/42ac3a5c/bigtop-packages/src/common/spark/compute-classpath.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/compute-classpath.sh b/bigtop-packages/src/common/spark/compute-classpath.sh index 61db280..eb6a807 100644 --- a/bigtop-packages/src/common/spark/compute-classpath.sh +++ b/bigtop-packages/src/common/spark/compute-classpath.sh @@ -28,15 +28,13 @@ if [ -e $FWDIR/conf/spark-env.sh ] ; then fi CORE_DIR="$FWDIR/core" -ASSEMBLY_DIR="$FWDIR/assembly" -EXAMPLES_DIR="$FWDIR/examples" +ASSEMBLY_DIR="$FWDIR/lib" PYSPARK_DIR="$FWDIR/python" # Build up classpath CLASSPATH="$SPARK_CLASSPATH" CLASSPATH="$CLASSPATH:$FWDIR/conf" -CLASSPATH="$CLASSPATH:$ASSEMBLY_DIR/lib/*" -CLASSPATH="$CLASSPATH:$EXAMPLES_DIR/lib/*" +CLASSPATH="$CLASSPATH:$ASSEMBLY_DIR/*" if [ -e "$PYSPARK_DIR" ]; then for jar in `find $PYSPARK_DIR/lib -name '*jar'`; do CLASSPATH="$CLASSPATH:$jar" http://git-wip-us.apache.org/repos/asf/bigtop/blob/42ac3a5c/bigtop-packages/src/common/spark/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/do-component-build b/bigtop-packages/src/common/spark/do-component-build index ac39b03..5327c35 100644 --- a/bigtop-packages/src/common/spark/do-component-build +++ b/bigtop-packages/src/common/spark/do-component-build @@ -37,4 +37,4 @@ BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${ # http://maven.40175.n5.nabble.com/Not-finding-artifact-in-local-repo-td3727753.html export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -XX:PermSize=1024m -XX:MaxPermSize=1024m" -mvn -Pbigtop-dist -Pyarn $BUILD_OPTS install +mvn -Pbigtop-dist -Pyarn -Phive $BUILD_OPTS install http://git-wip-us.apache.org/repos/asf/bigtop/blob/42ac3a5c/bigtop-packages/src/common/spark/install_spark.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/install_spark.sh b/bigtop-packages/src/common/spark/install_spark.sh index f09642b..3dfd2f6 100644 --- a/bigtop-packages/src/common/spark/install_spark.sh +++ b/bigtop-packages/src/common/spark/install_spark.sh @@ -121,6 +121,7 @@ SCALA_HOME=${SCALA_HOME:-/usr/share/scala} PYSPARK_PYTHON=${PYSPARK_PYTHON:-python} install -d -m 0755 $PREFIX/$LIB_DIR +install -d -m 0755 $PREFIX/$LIB_DIR/lib install -d -m 0755 $PREFIX/$LIB_DIR/bin install -d -m 0755 $PREFIX/$LIB_DIR/sbin install -d -m 0755 $PREFIX/$DOC_DIR @@ -130,28 +131,26 @@ install -d -m 0755 $PREFIX/var/log/spark/ install -d -m 0755 $PREFIX/var/run/spark/ install -d -m 0755 $PREFIX/var/run/spark/work/ -for comp in assembly; do - install -d -m 0755 $PREFIX/$LIB_DIR/$comp/lib - tar --wildcards -C $PREFIX/$LIB_DIR/$comp/lib -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz spark-$comp\* -done +tar --wildcards -C $PREFIX/$LIB_DIR/lib -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz spark-assembly\* +tar --wildcards --strip-components=1 -C $PREFIX/$LIB_DIR/lib -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz \*datanucleus\* +tar --wildcards -C $PREFIX/$LIB_DIR/ -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz bin\* +tar --wildcards -C $PREFIX/$LIB_DIR/ -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz sbin\* + +rm -rf $PREFIX/$LIB_DIR/bin/*.cmd + +# Examples jar +cp ${BUILD_DIR}/examples/target/spark-examples*${SPARK_VERSION}.jar $PREFIX/$LIB_DIR/lib +sed -i -e "s|lib/spark-examples-\*hadoop\*.jar|lib/spark-examples_\*.jar|" $PREFIX/$LIB_DIR/bin/run-example -## FIXME: Spark maven assembly needs to include examples into it. -install -d -m 0755 $PREFIX/$LIB_DIR/examples/lib -cp ${BUILD_DIR}/examples/target/spark-examples*${SPARK_VERSION}.jar $PREFIX/$LIB_DIR/examples/lib +# Examples src +install -d -m 0755 $PREFIX/$EXAMPLES_DIR +cp -ra ${BUILD_DIR}/examples/src $PREFIX/$EXAMPLES_DIR/ -cp -a ${BUILD_DIR}/bin/*.sh $PREFIX/$LIB_DIR/bin/ -cp -a ${BUILD_DIR}/sbin/*.sh $PREFIX/$LIB_DIR/sbin/ chmod 755 $PREFIX/$LIB_DIR/bin/* chmod 755 $PREFIX/$LIB_DIR/sbin/* -# FIXME: executor scripts need to reside in bin -cp -a $BUILD_DIR/bin/spark-class $PREFIX/$LIB_DIR/bin/ -cp -a $BUILD_DIR/sbin/spark-executor $PREFIX/$LIB_DIR/sbin/ cp -a ${SOURCE_DIR}/compute-classpath.sh $PREFIX/$LIB_DIR/bin/ -cp -a ${BUILD_DIR}/bin/spark-shell $PREFIX/$LIB_DIR/bin/ -cp -a ${BUILD_DIR}/bin/spark-submit $PREFIX/$LIB_DIR/bin/ - -touch $PREFIX/$LIB_DIR/RELEASE +chmod 755 $PREFIX/$LIB_DIR/bin/compute-classpath.sh # Copy in the configuration files install -d -m 0755 $PREFIX/$CONF_DIR @@ -159,15 +158,6 @@ cp -a ${BUILD_DIR}/conf/* $PREFIX/$CONF_DIR cp $PREFIX/$CONF_DIR/spark-env.sh.template $PREFIX/$CONF_DIR/spark-env.sh ln -s /etc/spark/conf $PREFIX/$LIB_DIR/conf -# Unpack static UI resources into install_dir/spark where it is expected to be -tar --wildcards -C $PREFIX/$LIB_DIR -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz ui-resources/\* - -# set correct permissions for exec. files -for execfile in bin/spark-class bin/spark-shell sbin/spark-executor bin/spark-submit; do - chmod 755 $PREFIX/$LIB_DIR/$execfile -done -chmod 755 $PREFIX/$LIB_DIR/bin/compute-classpath.sh - # Copy in the wrappers install -d -m 0755 $PREFIX/$BIN_DIR for wrap in sbin/spark-executor bin/spark-shell bin/spark-submit; do @@ -211,7 +201,6 @@ EOF ln -s /var/run/spark/work $PREFIX/$LIB_DIR/work cp -r ${BUILD_DIR}/python ${PREFIX}/${INSTALLED_LIB_DIR}/ -cp ${BUILD_DIR}/bin/pyspark ${PREFIX}/${INSTALLED_LIB_DIR}/bin/ cat > $PREFIX/$BIN_DIR/pyspark <<EOF #!/bin/bash @@ -224,4 +213,6 @@ exec $INSTALLED_LIB_DIR/bin/pyspark "\$@" EOF chmod 755 $PREFIX/$BIN_DIR/pyspark +touch $PREFIX/$LIB_DIR/RELEASE + cp ${BUILD_DIR}/{LICENSE,NOTICE} ${PREFIX}/${LIB_DIR}/ http://git-wip-us.apache.org/repos/asf/bigtop/blob/42ac3a5c/bigtop-packages/src/deb/spark/spark-core.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark/spark-core.install b/bigtop-packages/src/deb/spark/spark-core.install index 236e11d..d1efdfd 100644 --- a/bigtop-packages/src/deb/spark/spark-core.install +++ b/bigtop-packages/src/deb/spark/spark-core.install @@ -2,17 +2,15 @@ /usr/bin/spark-executor /usr/bin/spark-submit /usr/bin/spark-shell -/usr/lib/spark/assembly -/usr/lib/spark/bin/compute-classpath.sh -/usr/lib/spark/bin/spark-class -/usr/lib/spark/bin/spark-shell +/usr/lib/spark/bin/ +/usr/lib/spark/lib /usr/lib/spark/sbin -/usr/lib/spark/examples /usr/lib/spark/ui-resources /usr/lib/spark/conf /usr/lib/spark/LICENSE /usr/lib/spark/NOTICE /usr/lib/spark/RELEASE +/usr/share/doc/spark /usr/lib/spark/work /var/lib/spark/ /var/log/spark/ http://git-wip-us.apache.org/repos/asf/bigtop/blob/42ac3a5c/bigtop-packages/src/rpm/spark/SPECS/spark.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec index 1008b1b..81d658e 100644 --- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec +++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec @@ -152,6 +152,9 @@ done %config(noreplace) %{config_spark}.dist %doc %{doc_spark} %{lib_spark} +%{lib_spark}/bin +%{lib_spark}/sbin +%{lib_spark}/lib %exclude %{lib_spark}/bin/pyspark %exclude %{lib_spark}/python %{etc_spark}
