Updated Branches: refs/heads/master 83906bb8e -> d2112eaf1
BIGTOP-1181. Add pyspark to spark package Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/b1425046 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/b1425046 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/b1425046 Branch: refs/heads/master Commit: b1425046e452f6cc176e36fc89443b19e87e69fc Parents: 83906bb Author: Sean Mackrory <[email protected]> Authored: Tue Jan 14 13:34:40 2014 -0800 Committer: Sean Mackrory <[email protected]> Committed: Tue Jan 21 12:03:30 2014 -0800 ---------------------------------------------------------------------- .../src/common/spark/install_spark.sh | 21 +++++++++++++++++ bigtop-packages/src/deb/spark/control | 6 +++++ bigtop-packages/src/deb/spark/rules | 2 +- .../src/deb/spark/spark-core.install | 18 +++++++++++++++ .../src/deb/spark/spark-python.install | 4 ++++ bigtop-packages/src/rpm/spark/SPECS/spark.spec | 24 ++++++++++++++++++-- .../package/src/main/resources/package_data.xml | 10 ++++++++ 7 files changed, 82 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/common/spark/install_spark.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/install_spark.sh b/bigtop-packages/src/common/spark/install_spark.sh index 75dd337..664c271 100644 --- a/bigtop-packages/src/common/spark/install_spark.sh +++ b/bigtop-packages/src/common/spark/install_spark.sh @@ -31,6 +31,7 @@ usage: $0 <options> --installed-lib-dir=DIR path where lib-dir will end up on target system --bin-dir=DIR path to install bins [/usr/bin] --examples-dir=DIR path to install examples [doc-dir/examples] + --pyspark-python executable to use for Python interpreter [python] ... [ see source for more similar options ] " exit 1 @@ -46,6 +47,7 @@ OPTS=$(getopt \ -l 'bin-dir:' \ -l 'source-dir:' \ -l 'examples-dir:' \ + -l 'pyspark-python:' \ -l 'build-dir:' -- "$@") if [ $? != 0 ] ; then @@ -79,6 +81,9 @@ while true ; do --examples-dir) EXAMPLES_DIR=$2 ; shift 2 ;; + --pyspark-python) + PYSPARK_PYTHON=$2 ; shift 2 + ;; --) shift ; break ;; @@ -114,6 +119,7 @@ EXAMPLES_DIR=${EXAMPLES_DIR:-$DOC_DIR/examples} BIN_DIR=${BIN_DIR:-/usr/bin} CONF_DIR=${CONF_DIR:-/etc/spark/conf.dist} SCALA_HOME=${SCALA_HOME:-/usr/share/scala} +PYSPARK_PYTHON=${PYSPARK_PYTHON:-python} install -d -m 0755 $PREFIX/$LIB_DIR install -d -m 0755 $PREFIX/$LIB_DIR/lib @@ -190,3 +196,18 @@ export STANDALONE_SPARK_MASTER_HOST=\`hostname\` EOF ln -s /var/run/spark/work $PREFIX/$LIB_DIR/work + +cp -r ${BUILD_DIR}/python ${PREFIX}/${INSTALLED_LIB_DIR}/ +cp ${BUILD_DIR}/pyspark ${PREFIX}/${INSTALLED_LIB_DIR}/ +cat > $PREFIX/$BIN_DIR/pyspark <<EOF +#!/bin/bash + +# Autodetect JAVA_HOME if not defined +. /usr/lib/bigtop-utils/bigtop-detect-javahome + +export PYSPARK_PYTHON=${PYSPARK_PYTHON} + +exec $INSTALLED_LIB_DIR/pyspark "\$@" +EOF +chmod 755 $PREFIX/$BIN_DIR/pyspark + http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/deb/spark/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark/control b/bigtop-packages/src/deb/spark/control index fc61489..14e3440 100644 --- a/bigtop-packages/src/deb/spark/control +++ b/bigtop-packages/src/deb/spark/control @@ -42,3 +42,9 @@ Architecture: all Depends: spark-core (= ${source:Version}) Description: Server for Spark worker Server for Spark worker + +Package: spark-python +Architecture: all +Depends: spark (= ${source:Version}), python +Description: Python client for Spark + Includes PySpark, an interactive Python shell for Spark, and related libraries http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/deb/spark/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark/rules b/bigtop-packages/src/deb/spark/rules index bd0ad6b..f559f49 100644 --- a/bigtop-packages/src/deb/spark/rules +++ b/bigtop-packages/src/deb/spark/rules @@ -40,4 +40,4 @@ override_dh_auto_install: $(svcs) --build-dir=`pwd` \ --doc-dir=/usr/share/doc/spark \ --source-dir=debian \ - --prefix=debian/spark-core + --prefix=debian/tmp http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/deb/spark/spark-core.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark/spark-core.install b/bigtop-packages/src/deb/spark/spark-core.install new file mode 100644 index 0000000..0c5a6f5 --- /dev/null +++ b/bigtop-packages/src/deb/spark/spark-core.install @@ -0,0 +1,18 @@ +/etc/spark +/usr/bin/spark-executor +/usr/bin/spark-shell +/usr/lib/spark/bagel +/usr/lib/spark/bin +/usr/lib/spark/core +/usr/lib/spark/examples +/usr/lib/spark/lib +/usr/lib/spark/mllib +/usr/lib/spark/repl +/usr/lib/spark/streaming +/usr/lib/spark/ui-resources +/usr/lib/spark/conf +/usr/lib/spark/RELEASE +/usr/lib/spark/spark-class +/usr/lib/spark/spark-executor +/usr/lib/spark/spark-shell + http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/deb/spark/spark-python.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark/spark-python.install b/bigtop-packages/src/deb/spark/spark-python.install new file mode 100644 index 0000000..b5461bb --- /dev/null +++ b/bigtop-packages/src/deb/spark/spark-python.install @@ -0,0 +1,4 @@ +/usr/bin/pyspark +/usr/lib/spark/pyspark +/usr/lib/spark/python + http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-packages/src/rpm/spark/SPECS/spark.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec index a8db290..c78fa97 100644 --- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec +++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec @@ -33,6 +33,9 @@ %define alternatives_cmd alternatives %endif +%define pyspark_python python + + # disable repacking jars %define __os_install_post %{nil} @@ -90,7 +93,15 @@ Requires: spark-core = %{version}-%{release} %description worker Server for Spark worker - + +%package python +Summary: Python client for Spark +Group: Development/Libraries +Requires: spark = %{version}-%{release}, %{pyspark_python} + +%description python +Includes PySpark, an interactive Python shell for Spark, and related libraries + %prep %setup -n %{spark_name}-%{spark_base_version} @@ -110,7 +121,8 @@ sh $RPM_SOURCE_DIR/install_spark.sh \ --build-dir=`pwd` \ --source-dir=$RPM_SOURCE_DIR \ --prefix=$RPM_BUILD_ROOT \ - --doc-dir=%{doc_spark} + --doc-dir=%{doc_spark} \ + --pyspark-python=%{pyspark_python} for service in %{spark_services} do @@ -146,6 +158,8 @@ done %config(noreplace) %{config_spark}.dist %doc %{doc_spark} %{lib_spark} +%exclude %{lib_spark}/pyspark +%exclude %{lib_spark}/python %{etc_spark} %attr(0755,spark,spark) %{var_lib_spark} %attr(0755,spark,spark) %{var_run_spark} @@ -154,6 +168,12 @@ done %{bin}/spark-shell %{bin}/spark-executor +%files python +%defattr(-,root,root,755) +%attr(0755,root,root) %{bin}/pyspark +%attr(0755,root,root) %{lib_spark}/pyspark +%{lib_spark}/python + %define service_macro() \ %files %1 \ %attr(0755,root,root)/%{initd_dir}/%{spark_name}-%1 \ http://git-wip-us.apache.org/repos/asf/bigtop/blob/b1425046/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml index 166323c..766dfd2 100644 --- a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml +++ b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml @@ -132,6 +132,16 @@ Spark runs on top of the Apache Mesos cluster manager.</description> <spark>/self</spark> </deps> </spark-worker> + <spark-python> + <metadata> + <summary>Python client for Spark</summary> + <description>Includes PySpark, an interactive Python shell for Spark, and related libraries</description> + <url>http://spark.incubator.apache.org/</url> + </metadata> + <deps> + <spark>/self</spark> + </deps> + </spark-python> <whirr> <metadata> <summary>Scripts and libraries for running software services on cloud infrastructure.</summary>
