Repository: bigtop Updated Branches: refs/heads/master 31ec50fb0 -> 037da0d76
adding spark 1.x family support Signed-off-by: amir sanjar <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/037da0d7 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/037da0d7 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/037da0d7 Branch: refs/heads/master Commit: 037da0d76f19458b8a8ba0fa6f4336c173935e36 Parents: 31ec50f Author: amir sanjar <[email protected]> Authored: Fri Nov 11 12:51:33 2016 -0600 Committer: amir sanjar <[email protected]> Committed: Fri Nov 11 13:12:42 2016 -0600 ---------------------------------------------------------------------- .../src/common/spark1/do-component-build | 43 ++++ .../src/common/spark1/install_spark.sh | 211 +++++++++++++++ bigtop-packages/src/common/spark1/patch0.diff | 11 + bigtop-packages/src/common/spark1/spark-env.sh | 43 ++++ .../src/common/spark1/spark-history-server.svc | 70 +++++ .../src/common/spark1/spark-master.svc | 81 ++++++ .../src/common/spark1/spark-thriftserver.svc | 75 ++++++ .../src/common/spark1/spark-worker.svc | 75 ++++++ bigtop-packages/src/deb/spark1/changelog | 1 + bigtop-packages/src/deb/spark1/compat | 1 + bigtop-packages/src/deb/spark1/control | 77 ++++++ bigtop-packages/src/deb/spark1/copyright | 15 ++ bigtop-packages/src/deb/spark1/rules | 42 +++ bigtop-packages/src/deb/spark1/source/format | 1 + .../src/deb/spark1/spark-core.install | 27 ++ .../src/deb/spark1/spark-core.postinst | 35 +++ .../src/deb/spark1/spark-core.preinst | 62 +++++ bigtop-packages/src/deb/spark1/spark-core.prerm | 38 +++ .../src/deb/spark1/spark-datanucleus.install | 2 + .../src/deb/spark1/spark-extras.install | 1 + .../src/deb/spark1/spark-python.install | 4 + .../src/deb/spark1/spark-yarn-shuffle.install | 2 + bigtop-packages/src/rpm/spark1/BUILD/.gitignore | 0 bigtop-packages/src/rpm/spark1/RPMS/.gitignore | 0 .../src/rpm/spark1/SOURCES/.gitignore | 0 bigtop-packages/src/rpm/spark1/SPECS/spark.spec | 255 +++++++++++++++++++ bigtop-packages/src/rpm/spark1/SRPMS/.gitignore | 0 bigtop.bom | 15 +- 28 files changed, 1185 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/common/spark1/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark1/do-component-build b/bigtop-packages/src/common/spark1/do-component-build new file mode 100644 index 0000000..8d74f4a --- /dev/null +++ b/bigtop-packages/src/common/spark1/do-component-build @@ -0,0 +1,43 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +. `dirname $0`/bigtop.bom + +BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${HOME} \ + -Drepo.maven.org=$IVY_MIRROR_PROP \ + -Dreactor.repo=file://${HOME}/.m2/repository \ + -Dhadoop.version=$HADOOP_VERSION \ + -Dyarn.version=$HADOOP_VERSION \ + -Dprotobuf.version=2.5.0 \ + -DrecompileMode=all \ + -Pbigtop-dist \ + -Pyarn -Phadoop-2.6 \ + -Phive -Phive-thriftserver \ + $SPARK_BUILD_OPTS" + +## this might be an issue at times +# http://maven.40175.n5.nabble.com/Not-finding-artifact-in-local-repo-td3727753.html +export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -XX:PermSize=1024m -XX:MaxPermSize=1024m" + +mvn $BUILD_OPTS -DskipTests install + +# Tests must be run after Spark has already been packaged. +# See http://spark.apache.org/docs/latest/building-spark.html#spark-tests-in-maven +if [ "$SPARK_RUN_TESTS" = "true" ]; then + mvn $BUILD_OPTS test +fi http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/common/spark1/install_spark.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark1/install_spark.sh b/bigtop-packages/src/common/spark1/install_spark.sh new file mode 100644 index 0000000..530b355 --- /dev/null +++ b/bigtop-packages/src/common/spark1/install_spark.sh @@ -0,0 +1,211 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +usage() { + echo " +usage: $0 <options> + Required not-so-options: + --build-dir=DIR path to dist.dir + --source-dir=DIR path to package shared files dir + --prefix=PREFIX path to install into + + Optional options: + --doc-dir=DIR path to install docs into [/usr/share/doc/spark] + --lib-dir=DIR path to install Spark home [/usr/lib/spark] + --installed-lib-dir=DIR path where lib-dir will end up on target system + --bin-dir=DIR path to install bins [/usr/bin] + --examples-dir=DIR path to install examples [doc-dir/examples] + --pyspark-python executable to use for Python interpreter [python] + ... [ see source for more similar options ] + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'doc-dir:' \ + -l 'lib-dir:' \ + -l 'installed-lib-dir:' \ + -l 'bin-dir:' \ + -l 'source-dir:' \ + -l 'examples-dir:' \ + -l 'pyspark-python:' \ + -l 'build-dir:' -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --build-dir) + BUILD_DIR=$2 ; shift 2 + ;; + --source-dir) + SOURCE_DIR=$2 ; shift 2 + ;; + --doc-dir) + DOC_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --installed-lib-dir) + INSTALLED_LIB_DIR=$2 ; shift 2 + ;; + --bin-dir) + BIN_DIR=$2 ; shift 2 + ;; + --examples-dir) + EXAMPLES_DIR=$2 ; shift 2 + ;; + --pyspark-python) + PYSPARK_PYTHON=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX BUILD_DIR SOURCE_DIR; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +if [ -f "$SOURCE_DIR/bigtop.bom" ]; then + . $SOURCE_DIR/bigtop.bom +fi + +MAN_DIR=${MAN_DIR:-/usr/share/man/man1} +DOC_DIR=${DOC_DIR:-/usr/share/doc/spark} +LIB_DIR=${LIB_DIR:-/usr/lib/spark} +INSTALLED_LIB_DIR=${INSTALLED_LIB_DIR:-/usr/lib/spark} +EXAMPLES_DIR=${EXAMPLES_DIR:-$DOC_DIR/examples} +BIN_DIR=${BIN_DIR:-/usr/bin} +CONF_DIR=${CONF_DIR:-/etc/spark/conf.dist} +PYSPARK_PYTHON=${PYSPARK_PYTHON:-python} + +install -d -m 0755 $PREFIX/$LIB_DIR +install -d -m 0755 $PREFIX/$LIB_DIR/lib +install -d -m 0755 $PREFIX/$LIB_DIR/bin +install -d -m 0755 $PREFIX/$LIB_DIR/sbin +install -d -m 0755 $PREFIX/$LIB_DIR/extras +install -d -m 0755 $PREFIX/$LIB_DIR/extras/lib +install -d -m 0755 $PREFIX/$LIB_DIR/yarn +install -d -m 0755 $PREFIX/$LIB_DIR/yarn/lib +install -d -m 0755 $PREFIX/$DOC_DIR +install -d -m 0755 $PREFIX/$EXAMPLES_DIR + +install -d -m 0755 $PREFIX/var/lib/spark/ +install -d -m 0755 $PREFIX/var/log/spark/ +install -d -m 0755 $PREFIX/var/run/spark/ +install -d -m 0755 $PREFIX/var/run/spark/work/ + +tar --wildcards -C $PREFIX/$LIB_DIR/lib -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz spark-assembly\* +tar --wildcards --strip-components=1 -C $PREFIX/$LIB_DIR/lib -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz \*datanucleus\* +tar --wildcards -C $PREFIX/$LIB_DIR/ -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz bin\* +tar --wildcards -C $PREFIX/$LIB_DIR/ -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz sbin\* + +rm -rf $PREFIX/$LIB_DIR/bin/*.cmd + +# External/extra jars +ls ${BUILD_DIR}/{external,extras}/*/target/*${SPARK_VERSION}.jar | grep -v 'original-\|assembly' | xargs -IJARS cp JARS $PREFIX/$LIB_DIR/extras/lib + +# Examples jar +cp ${BUILD_DIR}/examples/target/spark-examples*${SPARK_VERSION}.jar $PREFIX/$LIB_DIR/lib/spark-examples-${SPARK_VERSION}-hadoop${HADOOP_VERSION}.jar + +# Spark YARN Shuffle jar +cp ${BUILD_DIR}/network/yarn/target/*/spark-${SPARK_VERSION}-yarn-shuffle.jar $PREFIX/$LIB_DIR/lib/ + +# Examples src +cp -ra ${BUILD_DIR}/examples/src $PREFIX/$EXAMPLES_DIR/ +ln -s $EXAMPLES_DIR $PREFIX/$LIB_DIR/examples + +# Data +cp -ra ${BUILD_DIR}/data $PREFIX/$LIB_DIR/ + +chmod 755 $PREFIX/$LIB_DIR/bin/* +chmod 755 $PREFIX/$LIB_DIR/sbin/* + +# Copy in the configuration files +install -d -m 0755 $PREFIX/$CONF_DIR +cp -a ${BUILD_DIR}/conf/* $PREFIX/$CONF_DIR +cp $SOURCE_DIR/spark-env.sh $PREFIX/$CONF_DIR +ln -s /etc/spark/conf $PREFIX/$LIB_DIR/conf + +# Copy in the wrappers +install -d -m 0755 $PREFIX/$BIN_DIR +for wrap in bin/spark-class bin/spark-shell bin/spark-sql bin/spark-submit; do + cat > $PREFIX/$BIN_DIR/`basename $wrap` <<EOF +#!/bin/bash + +# Autodetect JAVA_HOME if not defined +. /usr/lib/bigtop-utils/bigtop-detect-javahome + +exec $INSTALLED_LIB_DIR/$wrap "\$@" +EOF + chmod 755 $PREFIX/$BIN_DIR/`basename $wrap` +done + +ln -s /var/run/spark/work $PREFIX/$LIB_DIR/work + +cp -r ${BUILD_DIR}/python ${PREFIX}/${INSTALLED_LIB_DIR}/ +rm -f ${PREFIX}/${INSTALLED_LIB_DIR}/python/.gitignore +cat > $PREFIX/$BIN_DIR/pyspark <<EOF +#!/bin/bash + +# Autodetect JAVA_HOME if not defined +. /usr/lib/bigtop-utils/bigtop-detect-javahome + +export PYSPARK_PYTHON=${PYSPARK_PYTHON} + +exec $INSTALLED_LIB_DIR/bin/pyspark "\$@" +EOF +chmod 755 $PREFIX/$BIN_DIR/pyspark + +touch $PREFIX/$LIB_DIR/RELEASE +cp ${BUILD_DIR}/{LICENSE,NOTICE} ${PREFIX}/${LIB_DIR}/ + +# Version-less symlinks +(cd $PREFIX/$LIB_DIR/lib; ln -s spark-assembly*.jar spark-assembly.jar; ln -s spark-examples*.jar spark-examples.jar) +pushd $PREFIX/$LIB_DIR/yarn/lib +ln -s ../../lib/spark-*-yarn-shuffle.jar spark-yarn-shuffle.jar +ln -s ../../lib/datanucleus-api-jdo*.jar datanucleus-api-jdo.jar +ln -s ../../lib/datanucleus-core*.jar datanucleus-core.jar +ln -s ../../lib/datanucleus-rdbms*.jar datanucleus-rdbms.jar +popd +pushd $PREFIX/$LIB_DIR/extras/lib +for j in $(ls *.jar); do + ln -s $j $(echo $j | sed -n 's/\(.*\)\(_[0-9.]\+-[0-9.]\+\)\(.jar\)/\1\3/p') +done +popd http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/common/spark1/patch0.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark1/patch0.diff b/bigtop-packages/src/common/spark1/patch0.diff new file mode 100644 index 0000000..5e1aefb --- /dev/null +++ b/bigtop-packages/src/common/spark1/patch0.diff @@ -0,0 +1,11 @@ +--- spark-1.6.2/pom.xml 2016-06-21 20:56:39.000000000 -0500 ++++ spark-1.6.2-patched/pom.xml 2016-11-01 21:46:37.629234248 -0500 +@@ -2511,7 +2511,7 @@ + <profile> + <id>hadoop-2.6</id> + <properties> +- <hadoop.version>2.6.0</hadoop.version> ++ <hadoop.version>${hadoop.version}</hadoop.version> + <jets3t.version>0.9.3</jets3t.version> + <zookeeper.version>3.4.6</zookeeper.version> + <curator.version>2.6.0</curator.version> http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/common/spark1/spark-env.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark1/spark-env.sh b/bigtop-packages/src/common/spark1/spark-env.sh new file mode 100644 index 0000000..885aed1 --- /dev/null +++ b/bigtop-packages/src/common/spark1/spark-env.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export SPARK_HOME=${SPARK_HOME:-/usr/lib/spark} +export SPARK_LOG_DIR=${SPARK_LOG_DIR:-/var/log/spark} + +export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop} +export HADOOP_HDFS_HOME=${HADOOP_HDFS_HOME:-${HADOOP_HOME}/../hadoop-hdfs} +export HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME:-${HADOOP_HOME}/../hadoop-mapreduce} +export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-${HADOOP_HOME}/../hadoop-yarn} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf} + +# Let's run everything with JVM runtime, instead of Scala +export SPARK_LAUNCH_WITH_SCALA=0 +export SPARK_LIBRARY_PATH=${SPARK_LIBRARY_PATH:-${SPARK_HOME}/lib} +export SCALA_LIBRARY_PATH=${SCALA_LIBRARY_PATH:-${SPARK_HOME}/lib} + +# Let's make sure that all needed hadoop libs are added properly +export CLASSPATH="$CLASSPATH:$HADOOP_HOME/*:$HADOOP_HDFS_HOME/*:$HADOOP_YARN_HOME/*:$HADOOP_MAPRED_HOME/*" +export SPARK_LIBRARY_PATH=$SPARK_LIBRARY_PATH:${HADOOP_HOME}/lib/native + +export STANDALONE_SPARK_MASTER_HOST=`hostname -f` +export SPARK_MASTER_PORT=7077 +export SPARK_MASTER_WEBUI_PORT=18080 + +export SPARK_WORKER_DIR=${SPARK_WORKER_DIR:-/var/run/spark/work} +export SPARK_WORKER_PORT=7078 +export SPARK_WORKER_WEBUI_PORT=18081 + +export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dspark.history.fs.logDirectory=hdfs:///var/log/spark/apps -Dspark.history.ui.port=18082" http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/common/spark1/spark-history-server.svc ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark1/spark-history-server.svc b/bigtop-packages/src/common/spark1/spark-history-server.svc new file mode 100644 index 0000000..0410d16 --- /dev/null +++ b/bigtop-packages/src/common/spark1/spark-history-server.svc @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TYPE="history-server" +DAEMON="spark-${TYPE}" +DESC="Spark ${TYPE}" +EXEC_PATH="/usr/lib/spark/bin/spark-class" +SVC_USER="spark" +WORKING_DIR="/var/lib/spark" +DAEMON_FLAGS="" +CONF_DIR="/etc/spark/conf" +PIDFILE="/var/run/spark/${DAEMON}.pid" + +generate_start() { + +cat <<'__EOT__' +start() { + [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED + log_success_msg "Starting $DESC (${DAEMON}): " + + checkstatusofproc + status=$? + if [ "$status" -eq "$STATUS_RUNNING" ]; then + log_success_msg "${DESC} is running" + exit 0 + fi + + LOG_FILE=/var/log/spark/${DAEMON}.out + + su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \ + ${EXEC_PATH} org.apache.spark.deploy.history.HistoryServer $DAEMON_FLAGS \ + > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE" + + sleep 3 + + checkstatusofproc + RETVAL=$? + [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE + return $RETVAL +} +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' +stop() { + log_success_msg "Stopping $DESC (${DAEMON}): " + killproc -p $PIDFILE java + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE + return $RETVAL +} +__EOT__ + +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/common/spark1/spark-master.svc ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark1/spark-master.svc b/bigtop-packages/src/common/spark1/spark-master.svc new file mode 100644 index 0000000..1823b99 --- /dev/null +++ b/bigtop-packages/src/common/spark1/spark-master.svc @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TYPE="master" +DAEMON="spark-${TYPE}" +DESC="Spark ${TYPE}" +EXEC_PATH="/usr/lib/spark/bin/spark-class" +SVC_USER="spark" +WORKING_DIR="/var/lib/spark" +DAEMON_FLAGS="" +CONF_DIR="/etc/spark/conf" +PIDFILE="/var/run/spark/${DAEMON}.pid" + +generate_start() { + +cat <<'__EOT__' +start() { + [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED + log_success_msg "Starting $DESC (${DAEMON}): " + + checkstatusofproc + status=$? + if [ "$status" -eq "$STATUS_RUNNING" ]; then + log_success_msg "${DESC} is running" + exit 0 + fi + + LOG_FILE=/var/log/spark/${DAEMON}.out + + if [ -f $CONF_DIR/spark-env.sh ]; then + . $CONF_DIR/spark-env.sh + fi + + if [ "$SPARK_MASTER_IP" = "" ]; then + SPARK_MASTER_IP=`hostname` + fi + + + su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \ + ${EXEC_PATH} org.apache.spark.deploy.master.Master \ + --ip $SPARK_MASTER_IP $DAEMON_FLAGS \ + > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE" + + sleep 3 + + checkstatusofproc + RETVAL=$? + [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE + return $RETVAL +} +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' +stop() { + log_success_msg "Stopping $DESC (${DAEMON}): " + killproc -p $PIDFILE java + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE + return $RETVAL +} +__EOT__ + +} + http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/common/spark1/spark-thriftserver.svc ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark1/spark-thriftserver.svc b/bigtop-packages/src/common/spark1/spark-thriftserver.svc new file mode 100644 index 0000000..39a9748 --- /dev/null +++ b/bigtop-packages/src/common/spark1/spark-thriftserver.svc @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TYPE="thriftserver" +DAEMON="spark-${TYPE}" +DESC="Spark ${TYPE}" +EXEC_PATH="/usr/lib/spark/bin/spark-submit" +SVC_USER="spark" +WORKING_DIR="/var/lib/spark" +DAEMON_FLAGS="" +CONF_DIR="/etc/spark/conf" +PIDFILE="/var/run/spark/${DAEMON}.pid" + +generate_start() { + +cat <<'__EOT__' +start() { + [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED + log_success_msg "Starting $DESC (${DAEMON}): " + + checkstatusofproc + status=$? + if [ "$status" -eq "$STATUS_RUNNING" ]; then + log_success_msg "${DESC} is running" + exit 0 + fi + + function usage { + echo + } + export SUBMIT_USAGE_FUNCTION=usage + + LOG_FILE=/var/log/spark/${DAEMON}.out + + su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \ + ${EXEC_PATH} --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 1 $DAEMON_FLAGS \ + > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE" + + sleep 7 + + checkstatusofproc + RETVAL=$? + [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE + return $RETVAL +} +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' +stop() { + log_success_msg "Stopping $DESC (${DAEMON}): " + killproc -p $PIDFILE java + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE + return $RETVAL +} +__EOT__ + +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/common/spark1/spark-worker.svc ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark1/spark-worker.svc b/bigtop-packages/src/common/spark1/spark-worker.svc new file mode 100644 index 0000000..7c12374 --- /dev/null +++ b/bigtop-packages/src/common/spark1/spark-worker.svc @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TYPE="worker" +DAEMON="spark-${TYPE}" +DESC="Spark ${TYPE}" +EXEC_PATH="/usr/lib/spark/bin/spark-class" +SVC_USER="spark" +WORKING_DIR="/var/lib/spark" +DAEMON_FLAGS="" +CONF_DIR="/etc/spark/conf" +PIDFILE="/var/run/spark/${DAEMON}.pid" + +generate_start() { + +cat <<'__EOT__' +start() { + [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED + log_success_msg "Starting $DESC (${DAEMON}): " + + checkstatusofproc + status=$? + if [ "$status" -eq "$STATUS_RUNNING" ]; then + log_success_msg "${DESC} is running" + exit 0 + fi + + LOG_FILE=/var/log/spark/${DAEMON}.out + + if [ -f $CONF_DIR/spark-env.sh ]; then + . $CONF_DIR/spark-env.sh + fi + + su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \ + ${EXEC_PATH} org.apache.spark.deploy.worker.Worker spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT $DAEMON_FLAGS \ + > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE" + + sleep 3 + + checkstatusofproc + RETVAL=$? + [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE + return $RETVAL +} +__EOT__ + +} + +generate_stop() { + +cat <<'__EOT__' +stop() { + log_success_msg "Stopping $DESC (${DAEMON}): " + killproc -p $PIDFILE java + RETVAL=$? + + [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE + return $RETVAL +} +__EOT__ + +} + http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/changelog ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/changelog b/bigtop-packages/src/deb/spark1/changelog new file mode 100644 index 0000000..547ed02 --- /dev/null +++ b/bigtop-packages/src/deb/spark1/changelog @@ -0,0 +1 @@ +--- This is auto-generated http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/compat ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/compat b/bigtop-packages/src/deb/spark1/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/bigtop-packages/src/deb/spark1/compat @@ -0,0 +1 @@ +9 http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/control b/bigtop-packages/src/deb/spark1/control new file mode 100644 index 0000000..7ebc970 --- /dev/null +++ b/bigtop-packages/src/deb/spark1/control @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Source: spark-core +Section: misc +Priority: extra +Maintainer: Bigtop <[email protected]> +Build-Depends: debhelper (>= 6) +Standards-Version: 3.8.0 +Homepage: http://spark.apache.org/ + +Package: spark-core +Architecture: all +Depends: adduser, bigtop-utils (>= 0.7), hadoop-client +Description: Lightning-Fast Cluster Computing + Spark is a MapReduce-like cluster computing framework designed to support + low-latency iterative jobs and interactive use from an interpreter. It is + written in Scala, a high-level language for the JVM, and exposes a clean + language-integrated syntax that makes it easy to write parallel jobs. + Spark runs on top of the Apache Mesos cluster manager. + +Package: spark-master +Architecture: all +Depends: spark-core (= ${source:Version}) +Description: Server for Spark master + Server for Spark master + +Package: spark-worker +Architecture: all +Depends: spark-core (= ${source:Version}) +Description: Server for Spark worker + Server for Spark worker + +Package: spark-python +Architecture: all +Depends: spark-core (= ${source:Version}), python +Description: Python client for Spark + Includes PySpark, an interactive Python shell for Spark, and related libraries + +Package: spark-history-server +Architecture: all +Depends: spark-core (= ${source:Version}) +Description: History server for Apache Spark + History server for Apache Spark + +Package: spark-thriftserver +Architecture: all +Depends: spark-core (= ${source:Version}) +Description: Thrift server for Spark SQL + Thrift server for Spark SQL + +Package: spark-datanucleus +Architecture: all +Description: DataNucleus libraries for Apache Spark + DataNucleus libraries used by Spark SQL with Hive Support + +Package: spark-extras +Architecture: all +Description: External/extra libraries for Apache Spark + External/extra libraries built for Apache Spark but not included in the main assembly JAR (e.g., external streaming libraries) + +Package: spark-yarn-shuffle +Architecture: all +Description: Spark YARN Shuffle Service + Spark YARN Shuffle Service http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/copyright ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/copyright b/bigtop-packages/src/deb/spark1/copyright new file mode 100644 index 0000000..3255f3d --- /dev/null +++ b/bigtop-packages/src/deb/spark1/copyright @@ -0,0 +1,15 @@ +Format: http://dep.debian.net/deps/dep5 +Source: http://spark.apache.org/ +Upstream-Name: Spark Project + +Files: * +Copyright: 2010-2011, The Spark Project +License: Apache-2.0 + +Files debian/* +Copyright: 2011, The Apache Software Foundation +License: Apache-2.0 + +License: Apache-2.0 + On Debian systems, the complete text of the Apache 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/rules b/bigtop-packages/src/deb/spark1/rules new file mode 100644 index 0000000..5881518 --- /dev/null +++ b/bigtop-packages/src/deb/spark1/rules @@ -0,0 +1,42 @@ +#!/usr/bin/make -f + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -*- makefile -*- + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +# This has to be exported to make some magic below work. +export DH_OPTIONS + +%: + dh $@ + +override_dh_auto_build: + bash debian/do-component-build + +svcs=spark-master spark-worker spark-history-server spark-thriftserver + +$(svcs): debian/init.d.tmpl + bash $< debian/[email protected] deb debian/[email protected] + touch $@ + +override_dh_auto_install: $(svcs) + bash -x debian/install_spark.sh \ + --build-dir=`pwd` \ + --doc-dir=/usr/share/doc/spark \ + --source-dir=debian \ + --prefix=debian/tmp http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/source/format ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/source/format b/bigtop-packages/src/deb/spark1/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/bigtop-packages/src/deb/spark1/source/format @@ -0,0 +1 @@ +3.0 (quilt) http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/spark-core.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/spark-core.install b/bigtop-packages/src/deb/spark1/spark-core.install new file mode 100644 index 0000000..70f588c --- /dev/null +++ b/bigtop-packages/src/deb/spark1/spark-core.install @@ -0,0 +1,27 @@ +/etc/spark +/usr/bin/spark-class +/usr/bin/spark-submit +/usr/bin/spark-shell +/usr/bin/spark-sql +/usr/lib/spark/LICENSE +/usr/lib/spark/RELEASE +/usr/lib/spark/NOTICE +/usr/lib/spark/bin/beeline +/usr/lib/spark/bin/load-spark-env.sh +/usr/lib/spark/bin/run-example +/usr/lib/spark/bin/spark-class +/usr/lib/spark/bin/spark-shell +/usr/lib/spark/bin/spark-sql +/usr/lib/spark/bin/spark-submit +/usr/lib/spark/conf +/usr/lib/spark/data +/usr/lib/spark/examples +/usr/lib/spark/lib/spark-assembly*.jar +/usr/lib/spark/lib/spark-examples*.jar +/usr/lib/spark/sbin +/usr/lib/spark/work +/usr/share/doc/spark* +/var/lib/spark/ +/var/log/spark/ +/var/run/spark/ +/var/run/spark/work/ http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/spark-core.postinst ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/spark-core.postinst b/bigtop-packages/src/deb/spark1/spark-core.postinst new file mode 100644 index 0000000..f6cca0a --- /dev/null +++ b/bigtop-packages/src/deb/spark1/spark-core.postinst @@ -0,0 +1,35 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + configure) + # Install config alternatives + update-alternatives --install /etc/spark/conf spark-conf /etc/spark/conf.dist 30 + chown -R spark:spark /var/log/spark /var/lib/spark /var/run/spark + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/spark-core.preinst ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/spark-core.preinst b/bigtop-packages/src/deb/spark1/spark-core.preinst new file mode 100644 index 0000000..ea83e68 --- /dev/null +++ b/bigtop-packages/src/deb/spark1/spark-core.preinst @@ -0,0 +1,62 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# preinst script for spark +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * <new-preinst> `install' +# * <new-preinst> `install' <old-version> +# * <new-preinst> `upgrade' <old-version> +# * <old-preinst> `abort-upgrade' <new-version> +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + install|upgrade) + if ! getent passwd spark >/dev/null; then + # Adding system user: spark . + adduser \ + --system \ + --group \ + --home /var/lib/spark \ + --gecos "Spark" \ + --shell /bin/false \ + spark >/dev/null + fi + ;; + + abort-upgrade) + ;; + + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 + http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/spark-core.prerm ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/spark-core.prerm b/bigtop-packages/src/deb/spark1/spark-core.prerm new file mode 100644 index 0000000..8c40b7a --- /dev/null +++ b/bigtop-packages/src/deb/spark1/spark-core.prerm @@ -0,0 +1,38 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + remove|upgrade|deconfigure) + update-alternatives --remove spark-conf /etc/spark/conf.dist || : + ;; + + failed-upgrade) + ;; + + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/spark-datanucleus.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/spark-datanucleus.install b/bigtop-packages/src/deb/spark1/spark-datanucleus.install new file mode 100644 index 0000000..69f619b --- /dev/null +++ b/bigtop-packages/src/deb/spark1/spark-datanucleus.install @@ -0,0 +1,2 @@ +/usr/lib/spark/lib/datanucleus-*.jar +/usr/lib/spark/yarn/lib/datanucleus-*.jar http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/spark-extras.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/spark-extras.install b/bigtop-packages/src/deb/spark1/spark-extras.install new file mode 100644 index 0000000..c65fbc0 --- /dev/null +++ b/bigtop-packages/src/deb/spark1/spark-extras.install @@ -0,0 +1 @@ +/usr/lib/spark/extras/lib http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/spark-python.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/spark-python.install b/bigtop-packages/src/deb/spark1/spark-python.install new file mode 100644 index 0000000..8211931 --- /dev/null +++ b/bigtop-packages/src/deb/spark1/spark-python.install @@ -0,0 +1,4 @@ +/usr/bin/pyspark +/usr/lib/spark/bin/pyspark +/usr/lib/spark/python + http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/deb/spark1/spark-yarn-shuffle.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark1/spark-yarn-shuffle.install b/bigtop-packages/src/deb/spark1/spark-yarn-shuffle.install new file mode 100644 index 0000000..79c799a --- /dev/null +++ b/bigtop-packages/src/deb/spark1/spark-yarn-shuffle.install @@ -0,0 +1,2 @@ +/usr/lib/spark/lib/spark-*-yarn-shuffle.jar +/usr/lib/spark/yarn/lib/spark-yarn-shuffle.jar http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/rpm/spark1/BUILD/.gitignore ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/spark1/BUILD/.gitignore b/bigtop-packages/src/rpm/spark1/BUILD/.gitignore new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/rpm/spark1/RPMS/.gitignore ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/spark1/RPMS/.gitignore b/bigtop-packages/src/rpm/spark1/RPMS/.gitignore new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/rpm/spark1/SOURCES/.gitignore ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/spark1/SOURCES/.gitignore b/bigtop-packages/src/rpm/spark1/SOURCES/.gitignore new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/rpm/spark1/SPECS/spark.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/spark1/SPECS/spark.spec b/bigtop-packages/src/rpm/spark1/SPECS/spark.spec new file mode 100644 index 0000000..daecc72 --- /dev/null +++ b/bigtop-packages/src/rpm/spark1/SPECS/spark.spec @@ -0,0 +1,255 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%define spark_name spark +%define lib_spark /usr/lib/%{spark_name} +%define var_lib_spark /var/lib/%{spark_name} +%define var_run_spark /var/run/%{spark_name} +%define var_log_spark /var/log/%{spark_name} +%define bin_spark /usr/lib/%{spark_name}/bin +%define etc_spark /etc/%{spark_name} +%define config_spark %{etc_spark}/conf +%define bin /usr/bin +%define man_dir /usr/share/man +%define spark_services master worker history-server thriftserver + +%if %{?suse_version:1}0 +%define doc_spark %{_docdir}/spark +%define alternatives_cmd update-alternatives +%else +%define doc_spark %{_docdir}/spark-%{spark_version} +%define alternatives_cmd alternatives +%endif + +# disable repacking jars +%define __os_install_post %{nil} + +Name: spark-core +Version: %{spark_version} +Release: %{spark_release} +Summary: Lightning-Fast Cluster Computing +URL: http://spark.apache.org/ +Group: Development/Libraries +BuildArch: noarch +Buildroot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +License: ASL 2.0 +Source0: %{spark_name}-%{spark_base_version}.tar.gz +Source1: do-component-build +Source2: install_%{spark_name}.sh +Source3: spark-master.svc +Source4: spark-worker.svc +Source6: init.d.tmpl +Source7: spark-history-server.svc +Source8: spark-thriftserver.svc +Source9: bigtop.bom +Requires: bigtop-utils >= 0.7, hadoop-client +Requires(preun): /sbin/service + +%global initd_dir %{_sysconfdir}/init.d + +%if %{?suse_version:1}0 +# Required for init scripts +Requires: insserv +%global initd_dir %{_sysconfdir}/rc.d + +%else +# Required for init scripts +Requires: /lib/lsb/init-functions + +%global initd_dir %{_sysconfdir}/rc.d/init.d + +%endif + +%description +Spark is a MapReduce-like cluster computing framework designed to support +low-latency iterative jobs and interactive use from an interpreter. It is +written in Scala, a high-level language for the JVM, and exposes a clean +language-integrated syntax that makes it easy to write parallel jobs. +Spark runs on top of the Apache Mesos cluster manager. + +%package -n spark-master +Summary: Server for Spark master +Group: Development/Libraries +Requires: spark-core = %{version}-%{release} + +%description -n spark-master +Server for Spark master + +%package -n spark-worker +Summary: Server for Spark worker +Group: Development/Libraries +Requires: spark-core = %{version}-%{release} + +%description -n spark-worker +Server for Spark worker + +%package -n spark-python +Summary: Python client for Spark +Group: Development/Libraries +Requires: spark-core = %{version}-%{release}, python + +%description -n spark-python +Includes PySpark, an interactive Python shell for Spark, and related libraries + +%package -n spark-history-server +Summary: History server for Apache Spark +Group: Development/Libraries +Requires: spark-core = %{version}-%{release} + +%description -n spark-history-server +History server for Apache Spark + +%package -n spark-thriftserver +Summary: Thrift server for Spark SQL +Group: Development/Libraries +Requires: spark-core = %{version}-%{release} + +%description -n spark-thriftserver +Thrift server for Spark SQL + +%package -n spark-datanucleus +Summary: DataNucleus libraries for Apache Spark +Group: Development/Libraries + +%description -n spark-datanucleus +DataNucleus libraries used by Spark SQL with Hive Support + +%package -n spark-extras +Summary: External/extra libraries for Apache Spark +Group: Development/Libraries + +%description -n spark-extras +External/extra libraries built for Apache Spark but not included in the main +assembly JAR (e.g., external streaming libraries) + +%package -n spark-yarn-shuffle +Summary: Spark YARN Shuffle Service +Group: Development/Libraries + +%description -n spark-yarn-shuffle +Spark YARN Shuffle Service + +%prep +%setup -n %{spark_name}-%{spark_base_version} + +%build +bash $RPM_SOURCE_DIR/do-component-build + +%install +%__rm -rf $RPM_BUILD_ROOT +%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/ + +bash $RPM_SOURCE_DIR/install_spark.sh \ + --build-dir=`pwd` \ + --source-dir=$RPM_SOURCE_DIR \ + --prefix=$RPM_BUILD_ROOT \ + --doc-dir=%{doc_spark} \ + --pyspark-python=python + +for service in %{spark_services} +do + # Install init script + init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{spark_name}-${service} + bash $RPM_SOURCE_DIR/init.d.tmpl $RPM_SOURCE_DIR/spark-${service}.svc rpm $init_file +done + +%pre +getent group spark >/dev/null || groupadd -r spark +getent passwd spark >/dev/null || useradd -c "Spark" -s /sbin/nologin -g spark -r -d %{var_lib_spark} spark 2> /dev/null || : + +%post +%{alternatives_cmd} --install %{config_spark} %{spark_name}-conf %{config_spark}.dist 30 + +%preun +if [ "$1" = 0 ]; then + %{alternatives_cmd} --remove %{spark_name}-conf %{config_spark}.dist || : +fi + +for service in %{spark_services}; do + /sbin/service %{spark_name}-${service} status > /dev/null 2>&1 + if [ $? -eq 0 ]; then + /sbin/service %{spark_name}-${service} stop > /dev/null 2>&1 + fi +done + +####################### +#### FILES SECTION #### +####################### +%files +%defattr(-,root,root,755) +%config(noreplace) %{config_spark}.dist +%doc %{doc_spark} +%{lib_spark}/conf +%{lib_spark}/LICENSE +%{lib_spark}/RELEASE +%{lib_spark}/NOTICE +%{lib_spark}/bin +%{lib_spark}/lib +%exclude %{lib_spark}/lib/datanucleus-*.jar +%exclude %{lib_spark}/lib/spark-*-yarn-shuffle.jar +%{lib_spark}/sbin +%{lib_spark}/data +%{lib_spark}/examples +%{lib_spark}/work +%exclude %{bin_spark}/pyspark +%exclude %{lib_spark}/python +%{etc_spark} +%attr(0755,spark,spark) %{var_lib_spark} +%attr(0755,spark,spark) %{var_run_spark} +%attr(0755,spark,spark) %{var_log_spark} +%{bin}/spark-class +%{bin}/spark-shell +%{bin}/spark-sql +%{bin}/spark-submit + +%files -n spark-python +%defattr(-,root,root,755) +%attr(0755,root,root) %{bin}/pyspark +%attr(0755,root,root) %{lib_spark}/bin/pyspark +%{lib_spark}/python + +%files -n spark-datanucleus +%defattr(-,root,root,755) +%{lib_spark}/lib/datanucleus-*.jar +%{lib_spark}/yarn/lib/datanucleus-*.jar + +%files -n spark-extras +%defattr(-,root,root,755) +%{lib_spark}/extras + +%files -n spark-yarn-shuffle +%defattr(-,root,root,755) +%{lib_spark}/lib/spark-*-yarn-shuffle.jar +%{lib_spark}/yarn/lib/spark-yarn-shuffle.jar + +%define service_macro() \ +%files -n %1 \ +%attr(0755,root,root)/%{initd_dir}/%1 \ +%post -n %1 \ +chkconfig --add %1 \ +\ +%preun -n %1 \ +if [ $1 = 0 ] ; then \ + service %1 stop > /dev/null 2>&1 \ + chkconfig --del %1 \ +fi \ +%postun -n %1 \ +if [ $1 -ge 1 ]; then \ + service %1 condrestart >/dev/null 2>&1 \ +fi +%service_macro spark-master +%service_macro spark-worker +%service_macro spark-history-server +%service_macro spark-thriftserver http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop-packages/src/rpm/spark1/SRPMS/.gitignore ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/spark1/SRPMS/.gitignore b/bigtop-packages/src/rpm/spark1/SRPMS/.gitignore new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/bigtop/blob/037da0d7/bigtop.bom ---------------------------------------------------------------------- diff --git a/bigtop.bom b/bigtop.bom index e2c6673..3d90283 100644 --- a/bigtop.bom +++ b/bigtop.bom @@ -92,7 +92,7 @@ bigtop { dependencies = [ zookeeper:['hadoop', 'hbase'], hadoop:['ignite-hadoop', 'hbase', 'crunch', 'pig', 'hive', 'tez', 'sqoop', 'sqoop2', - 'oozie', 'mahout', 'flume', 'giraph', 'solr', 'crunch', 'spark', + 'oozie', 'mahout', 'flume', 'giraph', 'solr', 'crunch', 'spark','spark1', 'phoenix', 'alluxio', 'kafka', 'ycsb', 'kite', 'hama', 'zeppelin', 'tajo', 'apex' ], @@ -284,6 +284,17 @@ bigtop { site = "${apache.APACHE_MIRROR}/${download_path}" archive = "${apache.APACHE_ARCHIVE}/${download_path}" } } + 'spark1' { + name = 'spark1' + pkg = 'spark-core' + relNotes = 'Apache Spark' + version { base = '1.6.2'; pkg = base; release = 1 } + tarball { destination = "spark${version.base}.tar.gz" + source = "spark-${version.base}.tgz" } + url { download_path = "/spark/spark-${version.base}" + site = "${apache.APACHE_MIRROR}/${download_path}" + archive = "${apache.APACHE_ARCHIVE}/${download_path}" } + } 'spark' { name = 'spark' pkg = 'spark-core' @@ -291,7 +302,7 @@ bigtop { version { base = '1.6.2'; pkg = base; release = 1 } tarball { destination = "$name-${version.base}.tar.gz" source = "$name-${version.base}.tgz" } - url { download_path = "/$name/$name-${version.base}" + url { download_path = "/spark/spark-${version.base}" site = "${apache.APACHE_MIRROR}/${download_path}" archive = "${apache.APACHE_ARCHIVE}/${download_path}" } }
