Updated Branches: refs/heads/master f19e61efa -> e8addc020
BIGTOP-1193. Update Spark packaging for various conventions Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/e8addc02 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/e8addc02 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/e8addc02 Branch: refs/heads/master Commit: e8addc020013ca526a5038e56ff9d8e6f8f1170e Parents: f19e61e Author: Sean Mackrory <[email protected]> Authored: Thu Jan 23 11:50:41 2014 -0800 Committer: Sean Mackrory <[email protected]> Committed: Tue Jan 28 07:35:20 2014 -0800 ---------------------------------------------------------------------- .../src/common/spark/install_spark.sh | 6 +- bigtop-packages/src/common/spark/run | 148 ------------------- bigtop-packages/src/deb/spark/control | 2 +- bigtop-packages/src/deb/spark/rules | 3 +- bigtop-packages/src/rpm/spark/SPECS/spark.spec | 4 +- 5 files changed, 5 insertions(+), 158 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/e8addc02/bigtop-packages/src/common/spark/install_spark.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/install_spark.sh b/bigtop-packages/src/common/spark/install_spark.sh index d29dfd8..45996db 100644 --- a/bigtop-packages/src/common/spark/install_spark.sh +++ b/bigtop-packages/src/common/spark/install_spark.sh @@ -170,11 +170,7 @@ for wrap in spark-executor spark-shell ; do #!/bin/bash # Autodetect JAVA_HOME if not defined -if [ -e /usr/libexec/bigtop-detect-javahome ]; then - . /usr/libexec/bigtop-detect-javahome -elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then - . /usr/lib/bigtop-utils/bigtop-detect-javahome -fi +. /usr/lib/bigtop-utils/bigtop-detect-javahome exec $INSTALLED_LIB_DIR/$wrap "\$@" EOF http://git-wip-us.apache.org/repos/asf/bigtop/blob/e8addc02/bigtop-packages/src/common/spark/run ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/run b/bigtop-packages/src/common/spark/run deleted file mode 100644 index 50b0e82..0000000 --- a/bigtop-packages/src/common/spark/run +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -SCALA_VERSION=2.9.3 - -# Figure out where the Scala framework is installed -FWDIR="$(cd `dirname $0`; pwd)" - -# Export this as SPARK_HOME -export SPARK_HOME="$FWDIR" - -# Load environment variables from conf/spark-env.sh, if it exists -if [ -e $FWDIR/conf/spark-env.sh ] ; then - . $FWDIR/conf/spark-env.sh -fi - -if [ -z "$1" ]; then - echo "Usage: run <spark-class> [<args>]" >&2 - exit 1 -fi - -# If this is a standalone cluster daemon, reset SPARK_JAVA_OPTS and SPARK_MEM to reasonable -# values for that; it doesn't need a lot -if [ "$1" = "spark.deploy.master.Master" -o "$1" = "spark.deploy.worker.Worker" ]; then - SPARK_MEM=${SPARK_DAEMON_MEMORY:-512m} - SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.akka.logLifecycleEvents=true" - # Do not overwrite SPARK_JAVA_OPTS environment variable in this script - OUR_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS" # Empty by default -else - OUR_JAVA_OPTS="$SPARK_JAVA_OPTS" -fi - - -# Add java opts for master, worker, executor. The opts maybe null -case "$1" in - 'spark.deploy.master.Master') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_MASTER_OPTS" - ;; - 'spark.deploy.worker.Worker') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_WORKER_OPTS" - ;; - 'spark.executor.StandaloneExecutorBackend') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_EXECUTOR_OPTS" - ;; - 'spark.executor.MesosExecutorBackend') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_EXECUTOR_OPTS" - ;; - 'spark.repl.Main') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_REPL_OPTS" - ;; -esac - -# Figure out whether to run our class with java or with the scala launcher. -# In most cases, we'd prefer to execute our process with java because scala -# creates a shell script as the parent of its Java process, which makes it -# hard to kill the child with stuff like Process.destroy(). However, for -# the Spark shell, the wrapper is necessary to properly reset the terminal -# when we exit, so we allow it to set a variable to launch with scala. -if [ "$SPARK_LAUNCH_WITH_SCALA" == "1" ]; then - if [ "$SCALA_HOME" ]; then - RUNNER="${SCALA_HOME}/bin/scala" - else - if [ `command -v scala` ]; then - RUNNER="scala" - else - echo "SCALA_HOME is not set and scala is not in PATH" >&2 - exit 1 - fi - fi -else - if [ -n "${JAVA_HOME}" ]; then - RUNNER="${JAVA_HOME}/bin/java" - else - if [ `command -v java` ]; then - RUNNER="java" - else - echo "JAVA_HOME is not set" >&2 - exit 1 - fi - fi - if [ -z "$SCALA_LIBRARY_PATH" ]; then - if [ -z "$SCALA_HOME" ]; then - echo "SCALA_HOME is not set" >&2 - exit 1 - fi - SCALA_LIBRARY_PATH="$SCALA_HOME/lib" - fi -fi - -# Figure out how much memory to use per executor and set it as an environment -# variable so that our process sees it and can report it to Mesos -if [ -z "$SPARK_MEM" ] ; then - SPARK_MEM="512m" -fi -export SPARK_MEM - -# Set JAVA_OPTS to be able to load native libraries and to set heap size -JAVA_OPTS="$OUR_JAVA_OPTS" -JAVA_OPTS="$JAVA_OPTS -Djava.library.path=$SPARK_LIBRARY_PATH" -JAVA_OPTS="$JAVA_OPTS -Xms$SPARK_MEM -Xmx$SPARK_MEM" -# Load extra JAVA_OPTS from conf/java-opts, if it exists -if [ -e $FWDIR/conf/java-opts ] ; then - JAVA_OPTS="$JAVA_OPTS `cat $FWDIR/conf/java-opts`" -fi -export JAVA_OPTS -# Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in ExecutorRunner.scala! - -CORE_DIR="$FWDIR/core" -EXAMPLES_DIR="$FWDIR/examples" -REPL_DIR="$FWDIR/repl" - -# Compute classpath using external script -CLASSPATH=`$FWDIR/bin/compute-classpath.sh` -export CLASSPATH - -# Figure out the JAR file that our examples were packaged into. This includes a bit of a hack -# to avoid the -sources and -doc packages that are built by publish-local. -if [ -e "$EXAMPLES_DIR/lib/spark-examples"*[0-9T].jar ]; then - # Use the JAR from the SBT build - export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR/lib/spark-examples"*[0-9T].jar` -fi -if [ -e "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar ]; then - # Use the JAR from the Maven build - export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar` -fi - -if [ "$SPARK_LAUNCH_WITH_SCALA" == "1" ]; then - EXTRA_ARGS="" # Java options will be passed to scala as JAVA_OPTS -else - # The JVM doesn't read JAVA_OPTS by default so we need to pass it in - EXTRA_ARGS="$JAVA_OPTS" -fi - -exec "$RUNNER" -cp "$CLASSPATH" $EXTRA_ARGS "$@" http://git-wip-us.apache.org/repos/asf/bigtop/blob/e8addc02/bigtop-packages/src/deb/spark/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark/control b/bigtop-packages/src/deb/spark/control index e57994a..4e36ba8 100644 --- a/bigtop-packages/src/deb/spark/control +++ b/bigtop-packages/src/deb/spark/control @@ -23,7 +23,7 @@ Homepage: http://spark.incubator.apache.org/ Package: spark-core Architecture: all -Depends: bigtop-utils +Depends: bigtop-utils (>= 0.7) Description: Lightning-Fast Cluster Computing Spark is a MapReduce-like cluster computing framework designed to support low-latency iterative jobs and interactive use from an interpreter. It is http://git-wip-us.apache.org/repos/asf/bigtop/blob/e8addc02/bigtop-packages/src/deb/spark/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark/rules b/bigtop-packages/src/deb/spark/rules index f559f49..a21ba82 100644 --- a/bigtop-packages/src/deb/spark/rules +++ b/bigtop-packages/src/deb/spark/rules @@ -26,8 +26,7 @@ export DH_OPTIONS dh $@ override_dh_auto_build: - # we'll just use the build from the tarball. - env FULL_VERSION=${SPARK_PATCHED_VERSION} bash debian/do-component-build -Divy.home=${HOME}/.ivy2 + bash debian/do-component-build svcs=spark-master spark-worker http://git-wip-us.apache.org/repos/asf/bigtop/blob/e8addc02/bigtop-packages/src/rpm/spark/SPECS/spark.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec index e9d26b1..41f1743 100644 --- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec +++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec @@ -52,7 +52,7 @@ Source1: do-component-build Source2: install_%{spark_name}.sh Source3: spark-master.svc Source4: spark-worker.svc -Requires: bigtop-utils +Requires: bigtop-utils >= 0.7 Requires(preun): /sbin/service %global initd_dir %{_sysconfdir}/init.d @@ -111,7 +111,7 @@ bash $RPM_SOURCE_DIR/do-component-build %__rm -rf $RPM_BUILD_ROOT %__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/ -sh $RPM_SOURCE_DIR/install_spark.sh \ +bash $RPM_SOURCE_DIR/install_spark.sh \ --build-dir=`pwd` \ --source-dir=$RPM_SOURCE_DIR \ --prefix=$RPM_BUILD_ROOT \
