Updated Branches: refs/heads/master 2aa4877c1 -> c909eb1fc
BIGTOP-1056. some changes are introduced into Spark build: needs to be updated here Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c909eb1f Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c909eb1f Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c909eb1f Branch: refs/heads/master Commit: c909eb1fc92e9d3fd2f55325bfc7ebb1be23a164 Parents: 2aa4877 Author: Konstantin Boudnik <[email protected]> Authored: Wed Sep 4 20:56:06 2013 -0700 Committer: Konstantin Boudnik <[email protected]> Committed: Wed Sep 4 20:56:06 2013 -0700 ---------------------------------------------------------------------- bigtop-packages/src/common/spark/do-component-build | 7 ++----- bigtop-packages/src/common/spark/install_spark.sh | 13 ++++++++++--- bigtop-packages/src/common/spark/spark-executor | 7 ------- bigtop-packages/src/common/spark/spark-master.svc | 4 ++-- bigtop-packages/src/common/spark/spark-worker.svc | 4 ++-- bigtop-packages/src/rpm/spark/SPECS/spark.spec | 2 +- bigtop.mk | 2 +- 7 files changed, 18 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/common/spark/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/do-component-build b/bigtop-packages/src/common/spark/do-component-build index f36ab68..0593333 100644 --- a/bigtop-packages/src/common/spark/do-component-build +++ b/bigtop-packages/src/common/spark/do-component-build @@ -26,13 +26,10 @@ fi BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${HOME} \ -Drepo.maven.org=$IVY_MIRROR_PROP \ -Dreactor.repo=file://${HOME}/.m2/repository \ + -Dhadoop.version=$HADOOP_VERSION -DskipTests" ## this might be an issue at times # http://maven.40175.n5.nabble.com/Not-finding-artifact-in-local-repo-td3727753.html export MAVEN_OPTS="-XX:PermSize=1024m -XX:MaxPermSize=1024m" -# FIXME: this is a really unfortunate hack: make sure we are building with -# correct version -sed -ie "s#<yarn.version>2.0.2-alpha#<yarn.version>$HADOOP_VERSION#" pom.xml - -mvn -Phadoop2-yarn,assembly -DnoExpensive $BUILD_OPTS package +mvn -Pbigtop-dist $BUILD_OPTS package http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/common/spark/install_spark.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/install_spark.sh b/bigtop-packages/src/common/spark/install_spark.sh index 870ec09..6918757 100644 --- a/bigtop-packages/src/common/spark/install_spark.sh +++ b/bigtop-packages/src/common/spark/install_spark.sh @@ -101,6 +101,9 @@ if [ -z "${SCALA_HOME}" ]; then echo Missing env. var SCALA_HOME usage fi +if [ -f "$SOURCE_DIR/bigtop.bom" ]; then + . $SOURCE_DIR/bigtop.bom +fi MAN_DIR=${MAN_DIR:-/usr/share/man/man1} DOC_DIR=${DOC_DIR:-/usr/share/doc/spark} @@ -123,12 +126,16 @@ for comp in core repl bagel mllib streaming; do install -d -m 0755 $PREFIX/$LIB_DIR/$comp/lib tar --wildcards -C $PREFIX/$LIB_DIR/$comp/lib -zxf ${BUILD_DIR}/assembly/target/spark-assembly-*-dist.tar.gz spark-$comp\* done +## FIXME: Spark maven assembly needs to include examples into it. +install -d -m 0755 $PREFIX/$LIB_DIR/examples/lib +cp ${BUILD_DIR}/examples/target/spark-examples-${SPARK_VERSION}.jar $PREFIX/$LIB_DIR/examples/lib # FIXME: executor scripts need to reside in bin -cp -a ${SOURCE_DIR}/run $PREFIX/$LIB_DIR -cp -a ${SOURCE_DIR}/spark-executor $PREFIX/$LIB_DIR +cp -a $BUILD_DIR/spark-class $PREFIX/$LIB_DIR +cp -a $BUILD_DIR/spark-executor $PREFIX/$LIB_DIR cp -a ${SOURCE_DIR}/compute-classpath.sh $PREFIX/$SPARK_BIN_DIR cp -a ${BUILD_DIR}/spark-shell $PREFIX/$LIB_DIR +touch $PREFIX/$LIB_DIR/RELEASE # Copy in the configuration files install -d -m 0755 $PREFIX/$CONF_DIR @@ -140,7 +147,7 @@ ln -s /etc/spark/conf $PREFIX/$LIB_DIR/conf tar --wildcards --transform 's,ui-resources/spark,spark,' -C $PREFIX/$LIB_DIR -zxf ${BUILD_DIR}/assembly/target/spark-assembly-*-dist.tar.gz ui-resources/\* # set correct permissions for exec. files -for execfile in run spark-shell spark-executor ; do +for execfile in spark-class spark-shell spark-executor ; do chmod 755 $PREFIX/$LIB_DIR/$execfile done chmod 755 $PREFIX/$SPARK_BIN_DIR/compute-classpath.sh http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/common/spark/spark-executor ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/spark-executor b/bigtop-packages/src/common/spark/spark-executor deleted file mode 100644 index 55a7113..0000000 --- a/bigtop-packages/src/common/spark/spark-executor +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# This is a modification of the standard Spark executor script to run -# with Standalone executor backend -# TODO: we are likely need to add YARN executor later -FWDIR="`dirname $0`" -echo "Running spark-executor with framework dir = $FWDIR" -exec $FWDIR/run spark.executor.StandaloneExecutorBackend http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/common/spark/spark-master.svc ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/spark-master.svc b/bigtop-packages/src/common/spark/spark-master.svc index 0b6d5b3..42e89df 100644 --- a/bigtop-packages/src/common/spark/spark-master.svc +++ b/bigtop-packages/src/common/spark/spark-master.svc @@ -16,7 +16,7 @@ TYPE="master" DAEMON="spark-${TYPE}" DESC="Spark ${TYPE}" -EXEC_PATH="/usr/lib/spark/run" +EXEC_PATH="/usr/lib/spark/spark-class" SVC_USER="spark" WORKING_DIR="/var/lib/spark" DAEMON_FLAGS="" @@ -40,7 +40,7 @@ start() { LOG_FILE=/var/log/spark/${DAEMON}.out su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \ - ${EXEC_PATH} spark.deploy.master.Master $DAEMON_FLAGS \ + ${EXEC_PATH} org.apache.spark.deploy.master.Master $DAEMON_FLAGS \ > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE" sleep 3 http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/common/spark/spark-worker.svc ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/spark-worker.svc b/bigtop-packages/src/common/spark/spark-worker.svc index 1776f33..50c5487 100644 --- a/bigtop-packages/src/common/spark/spark-worker.svc +++ b/bigtop-packages/src/common/spark/spark-worker.svc @@ -16,7 +16,7 @@ TYPE="worker" DAEMON="spark-${TYPE}" DESC="Spark ${TYPE}" -EXEC_PATH="/usr/lib/spark/run" +EXEC_PATH="/usr/lib/spark/spark-class" SVC_USER="spark" WORKING_DIR="/var/lib/spark" DAEMON_FLAGS="" @@ -44,7 +44,7 @@ start() { fi su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \ - ${EXEC_PATH} spark.deploy.worker.Worker spark://$STANDALONE_SPARK_MASTER_HOST:$SPARK_MASTER_PORT $DAEMON_FLAGS \ + ${EXEC_PATH} org.apache.spark.deploy.worker.Worker spark://$STANDALONE_SPARK_MASTER_HOST:$SPARK_MASTER_PORT $DAEMON_FLAGS \ > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE" sleep 3 http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/rpm/spark/SPECS/spark.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec index b0889e4..13301e6 100644 --- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec +++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec @@ -78,7 +78,7 @@ Spark runs on top of the Apache Mesos cluster manager. %prep #%setup -n %{name}-%{spark_base_version} -%setup -n spark-master +%setup -n spark-branch-0.8 %build bash $RPM_SOURCE_DIR/do-component-build http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop.mk ---------------------------------------------------------------------- diff --git a/bigtop.mk b/bigtop.mk index 3fd3c12..f75c412 100644 --- a/bigtop.mk +++ b/bigtop.mk @@ -245,7 +245,7 @@ SPARK_BASE_VERSION=0.8.0-SNAPSHOT SPARK_PKG_VERSION=0.8.0 SPARK_RELEASE_VERSION=1 SPARK_TARBALL_DST=spark-${SPARK_BASE_VERSION}.tar.gz -SPARK_TARBALL_SRC=master.tar.gz +SPARK_TARBALL_SRC=branch-0.8.tar.gz SPARK_SITE=https://github.com/mesos/spark/archive SPARK_ARCHIVE=${SPARK_SITE} $(eval $(call PACKAGE,spark,SPARK))
