Updated Branches: refs/heads/branch-0.3.1 0c8282c23 -> cbacdf9a6
BIGTOP-715. Add Spark packaging Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/cbacdf9a Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/cbacdf9a Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/cbacdf9a Branch: refs/heads/branch-0.3.1 Commit: cbacdf9a6b52a48736210c699a0d147bc9c4ab9a Parents: 0c8282c Author: MTG dev <[email protected]> Authored: Wed Dec 5 17:20:52 2012 -0800 Committer: Konstantin Boudnik <[email protected]> Committed: Wed Dec 5 17:20:52 2012 -0800 ---------------------------------------------------------------------- DEVNOTES | 5 + .../src/common/spark/do-component-build | 29 +++ bigtop-packages/src/common/spark/install_spark.sh | 158 +++++++++++++++ bigtop-packages/src/rpm/spark/SPECS/spark.spec | 88 ++++++++ bigtop.mk | 16 ++ package.mk | 1 + 6 files changed, 297 insertions(+), 0 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/cbacdf9a/DEVNOTES ---------------------------------------------------------------------- diff --git a/DEVNOTES b/DEVNOTES index 8c13886..e6c1c4b 100644 --- a/DEVNOTES +++ b/DEVNOTES @@ -16,5 +16,10 @@ 1. Debian 1.0. apt-get install -y git subversion build-essential dh-make debhelper devscripts ant ant-optional autoconf automake subversion liblzo2-dev libzip-dev sharutils libfuse-dev reprepro libssl-dev + FIXME?: You also need to have Scala 2.9.1 or later installed and SCALA_HOME + env. var pointed properly. Otherwise, Spark project won't be build as it + requires scala. Only older version of the language environment is available + on Ubuntu at the moment. + 2. RedHat 2.0 yum install -y git subversion fuse-devel fuse fuse-libs http://git-wip-us.apache.org/repos/asf/bigtop/blob/cbacdf9a/bigtop-packages/src/common/spark/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/do-component-build b/bigtop-packages/src/common/spark/do-component-build new file mode 100644 index 0000000..3f2ef16 --- /dev/null +++ b/bigtop-packages/src/common/spark/do-component-build @@ -0,0 +1,29 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${HOME} \ + -Drepo.maven.org=$IVY_MIRROR_PROP -Dreactor.repo=file://${HOME}/.m2/repository" + +# FIXME: this is a really unfortunate hack +sed -ie 's#EXTRA_ARGS=""#EXTRA_ARGS="'"$BUILD_OPTS"'"#' sbt/sbt + +HADOOP_VERSION=1.0.3 +# FIXME: even more crazy hack +sed -ie 's#val HADOOP_VERSION =.*#val HADOOP_VERSION = "'$HADOOP_VERSION'"#' project/SparkBuild.scala + +sbt/sbt $BUILD_OPTS package "$@" http://git-wip-us.apache.org/repos/asf/bigtop/blob/cbacdf9a/bigtop-packages/src/common/spark/install_spark.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/install_spark.sh b/bigtop-packages/src/common/spark/install_spark.sh new file mode 100644 index 0000000..17fedc2 --- /dev/null +++ b/bigtop-packages/src/common/spark/install_spark.sh @@ -0,0 +1,158 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +usage() { + echo " +usage: $0 <options> + Required not-so-options: + --build-dir=DIR path to dist.dir + --prefix=PREFIX path to install into + + Optional options: + --doc-dir=DIR path to install docs into [/usr/share/doc/spark] + --lib-dir=DIR path to install Spark home [/usr/lib/spark] + --installed-lib-dir=DIR path where lib-dir will end up on target system + --bin-dir=DIR path to install bins [/usr/bin] + --examples-dir=DIR path to install examples [doc-dir/examples] + ... [ see source for more similar options ] + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'doc-dir:' \ + -l 'lib-dir:' \ + -l 'installed-lib-dir:' \ + -l 'bin-dir:' \ + -l 'examples-dir:' \ + -l 'build-dir:' -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --build-dir) + BUILD_DIR=$2 ; shift 2 + ;; + --doc-dir) + DOC_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --installed-lib-dir) + INSTALLED_LIB_DIR=$2 ; shift 2 + ;; + --bin-dir) + BIN_DIR=$2 ; shift 2 + ;; + --examples-dir) + EXAMPLES_DIR=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX BUILD_DIR ; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + + +if [ -z "${SCALA_HOME}" ]; then + echo Missing env. var SCALA_HOME + usage +fi + +MAN_DIR=${MAN_DIR:-/usr/share/man/man1} +DOC_DIR=${DOC_DIR:-/usr/share/doc/spark} +LIB_DIR=${LIB_DIR:-/usr/lib/spark} +INSTALLED_LIB_DIR=${INSTALLED_LIB_DIR:-/usr/lib/spark} +EXAMPLES_DIR=${EXAMPLES_DIR:-$DOC_DIR/examples} +BIN_DIR=${BIN_DIR:-/usr/bin} +CONF_DIR=${CONF_DIR:-/etc/spark/conf.dist} +SCALA_HOME=${SCALA_HOME:-/usr/share/scala} + +install -d -m 0755 $PREFIX/$LIB_DIR +install -d -m 0755 $PREFIX/$DOC_DIR + +# FIXME: at some point we need to create a separate package for Scala +cp -ra $SCALA_HOME $PREFIX/$LIB_DIR/scala + +# FIXME: lib_managed is a terrible name for this +cp -ra ${BUILD_DIR}/lib_managed $PREFIX/$LIB_DIR + +# FIXME: these need to be split into individual packages +for comp in core repl bagel examples ; do + install -d -m 0755 $PREFIX/$LIB_DIR/$comp/lib + cp -a ${BUILD_DIR}/$comp/target/scala-*/*.jar $PREFIX/$LIB_DIR/$comp/lib + cp -a ${BUILD_DIR}/$comp/lib/* $PREFIX/$LIB_DIR/$comp/lib || : +done + +# FIXME: executor scripts need to reside in bin +cp -a ${BUILD_DIR}/run $PREFIX/$LIB_DIR +cp -a ${BUILD_DIR}/spark-executor $PREFIX/$LIB_DIR +cp -a ${BUILD_DIR}/spark-shell $PREFIX/$LIB_DIR + +# Copy in the configuration files +install -d -m 0755 $PREFIX/$CONF_DIR +cp -a ${BUILD_DIR}/conf/* $PREFIX/$CONF_DIR +ln -s /etc/spark/conf $PREFIX/$LIB_DIR/conf + +# Copy in the example files +cp -a ${BUILD_DIR}/examples/ $PREFIX/$DOC_DIR/ + +# Copy in the wrappers +install -d -m 0755 $PREFIX/$BIN_DIR +for wrap in spark-executor spark-shell ; do + cat > $PREFIX/$BIN_DIR/$wrap <<EOF +#!/bin/sh + +# Autodetect JAVA_HOME if not defined +if [ -e /usr/libexec/bigtop-detect-javahome ]; then + . /usr/libexec/bigtop-detect-javahome +elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then + . /usr/lib/bigtop-utils/bigtop-detect-javahome +fi + +export SCALA_HOME=\${SCALA_HOME:-$LIB_DIR/scala} +export PATH=\$PATH:\$SCALA_HOME/bin + +exec $INSTALLED_LIB_DIR/$wrap "\$@" +EOF + chmod 755 $PREFIX/$BIN_DIR/$wrap +done http://git-wip-us.apache.org/repos/asf/bigtop/blob/cbacdf9a/bigtop-packages/src/rpm/spark/SPECS/spark.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec new file mode 100644 index 0000000..818fed2 --- /dev/null +++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%define spark_name spark +%define lib_spark /usr/lib/%{spark_name} +%define etc_spark /etc/%{spark_name} +%define config_spark %{etc_spark}/conf +%define log_spark /var/log/%{spark_name} +%define bin_spark /usr/bin +%define man_dir /usr/share/man + +%if %{?suse_version:1}0 +%define doc_spark %{_docdir}/spark +%define alternatives_cmd update-alternatives +%else +%define doc_spark %{_docdir}/spark-%{spark_version} +%define alternatives_cmd alternatives +%endif + +# disable repacking jars +%define __os_install_post %{nil} + +Name: spark +Version: %{spark_version} +Release: %{spark_release} +Summary: Lightning-Fast Cluster Computing +URL: http://www.spark-project.org/ +Group: Development/Libraries +BuildArch: noarch +Buildroot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +License: ASL 2.0 +Source0: %{name}-%{spark_base_version}.tar.gz +Source1: do-component-build +Source2: install_%{name}.sh +Requires: bigtop-utils + + +%description +Spark is a MapReduce-like cluster computing framework designed to support +low-latency iterative jobs and interactive use from an interpreter. It is +written in Scala, a high-level language for the JVM, and exposes a clean +language-integrated syntax that makes it easy to write parallel jobs. +Spark runs on top of the Apache Mesos cluster manager. + +%prep +#%setup -n %{name}-%{spark_base_version} +%setup -n mesos-spark-0472cf8 + +%build +bash $RPM_SOURCE_DIR/do-component-build + +%install +%__rm -rf $RPM_BUILD_ROOT +sh $RPM_SOURCE_DIR/install_spark.sh \ + --build-dir=`pwd` \ + --prefix=$RPM_BUILD_ROOT \ + --doc-dir=%{doc_spark} + +%post +%{alternatives_cmd} --install %{config_spark} %{spark_name}-conf %{config_spark}.dist 30 + +%preun +if [ "$1" = 0 ]; then + %{alternatives_cmd} --remove %{spark_name}-conf %{config_spark}.dist || : +fi + +####################### +#### FILES SECTION #### +####################### +%files +%defattr(-,root,root,755) +%config(noreplace) %{config_spark}.dist +%doc %{doc_spark} +%{lib_spark} +%{bin_spark}/spark-shell +%{bin_spark}/spark-executor http://git-wip-us.apache.org/repos/asf/bigtop/blob/cbacdf9a/bigtop.mk ---------------------------------------------------------------------- diff --git a/bigtop.mk b/bigtop.mk index bd9327d..be969d1 100644 --- a/bigtop.mk +++ b/bigtop.mk @@ -155,6 +155,22 @@ FLUME_SITE=$(APACHE_MIRROR)$(FLUME_DOWNLOAD_PATH) FLUME_ARCHIVE=$(APACHE_ARCHIVE)$(FLUME_DOWNLOAD_PATH) $(eval $(call PACKAGE,flume,FLUME)) +# Spark +SPARK_NAME=spark +SPARK_RELNOTES_NAME=Spark +SPARK_PKG_NAME=spark +SPARK_BASE_VERSION=0.5.0 +SPARK_PKG_VERSION=0.5.0 +SPARK_RELEASE_VERSION=1 +SPARK_TARBALL_DST=$(SPARK_NAME)-$(SPARK_BASE_VERSION).tar.gz +SPARK_TARBALL_SRC=v0.5.0 +#SPARK_DOWNLOAD_PATH=/incubator/flume/$(SPARK_PKG_NAME)-$(SPARK_BASE_VERSION) +#SPARK_SITE=$(APACHE_MIRROR)$(SPARK_DOWNLOAD_PATH) +#SPARK_ARCHIVE=$(APACHE_ARCHIVE)$(SPARK_DOWNLOAD_PATH) +SPARC_SITE=https://github.com/mesos/spark/tarball +SPARK_ARCHIVE=$(SPARC_SITE) +$(eval $(call PACKAGE,spark,SPARK)) + # Bigtop-utils BIGTOP_UTILS_NAME=bigtop-utils BIGTOP_UTILS__RELNOTES_NAME=Bigtop-utils http://git-wip-us.apache.org/repos/asf/bigtop/blob/cbacdf9a/package.mk ---------------------------------------------------------------------- diff --git a/package.mk b/package.mk index 0d5ac66..5eaac8a 100644 --- a/package.mk +++ b/package.mk @@ -106,6 +106,7 @@ $(BUILD_DIR)/%/.deb: --preserve-envvar FORREST_HOME \ --preserve-envvar MAVEN3_HOME \ --preserve-envvar JAVA_HOME \ + --preserve-envvar SCALA_HOME \ --set-envvar=$(PKG)_BASE_VERSION=$($(PKG)_BASE_VERSION) \ --set-envvar=$(PKG)_VERSION=$($(PKG)_PKG_VERSION)$(BIGTOP_BUILD_STAMP) \ --set-envvar=$(PKG)_RELEASE=$($(PKG)_RELEASE_VERSION) \
