BIGTOP-612. Add Crunch to Bigtop
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/57fbb284 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/57fbb284 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/57fbb284 Branch: refs/heads/master Commit: 57fbb284954ee81f9d6a381c159cf415867dffab Parents: aa35450 Author: Roman Shaposhnik <[email protected]> Authored: Wed Nov 28 21:35:16 2012 -0800 Committer: Roman Shaposhnik <[email protected]> Committed: Tue Dec 4 13:59:45 2012 -0800 ---------------------------------------------------------------------- bigtop-deploy/puppet/manifests/cluster.pp | 2 + .../puppet/modules/crunch/manifests/init.pp | 22 ++++ bigtop-deploy/puppet/modules/crunch/tests/init.pp | 17 +++ .../src/common/crunch/do-component-build | 30 +++++ .../src/common/crunch/install_crunch.sh | 99 +++++++++++++++ bigtop-packages/src/deb/crunch/changelog | 1 + bigtop-packages/src/deb/crunch/compat | 1 + bigtop-packages/src/deb/crunch/control | 36 ++++++ bigtop-packages/src/deb/crunch/copyright | 15 +++ bigtop-packages/src/deb/crunch/crunch-doc.install | 1 + bigtop-packages/src/deb/crunch/crunch.install | 1 + bigtop-packages/src/deb/crunch/rules | 41 ++++++ bigtop-packages/src/deb/crunch/source/format | 1 + bigtop-packages/src/rpm/crunch/SPECS/crunch.spec | 77 +++++++++++ .../package/src/main/resources/package_data.xml | 17 +++ bigtop.mk | 14 ++ 16 files changed, 375 insertions(+), 0 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index 57d73b3..99d5af3 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -286,6 +286,8 @@ class hadoop_gateway_node inherits hadoop_cluster_node { giraph::client { "giraph client": zookeeper_quorum => $giraph_zookeeper_quorum, } + crunch::client { "crunch client": + } hadoop-pig::client { "pig client": } hadoop-hive::client { "hive client": http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-deploy/puppet/modules/crunch/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/crunch/manifests/init.pp b/bigtop-deploy/puppet/modules/crunch/manifests/init.pp new file mode 100644 index 0000000..d446667 --- /dev/null +++ b/bigtop-deploy/puppet/modules/crunch/manifests/init.pp @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class crunch { + define client { + package { ["crunch", "crunch-doc"]: + ensure => latest, + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-deploy/puppet/modules/crunch/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/crunch/tests/init.pp b/bigtop-deploy/puppet/modules/crunch/tests/init.pp new file mode 100644 index 0000000..abef371 --- /dev/null +++ b/bigtop-deploy/puppet/modules/crunch/tests/init.pp @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include crunch +crunch::client { "test-crunch": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-packages/src/common/crunch/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/crunch/do-component-build b/bigtop-packages/src/common/crunch/do-component-build new file mode 100644 index 0000000..4285f6f --- /dev/null +++ b/bigtop-packages/src/common/crunch/do-component-build @@ -0,0 +1,30 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +. `dirname $0`/bigtop.bom + +mvn -DskipTests -Dcrunch.platform=2 \ + -Dslf4j.version=1.6.1 \ + -Dhadoop.version=$HADOOP_VERSION \ + -Dhbase.version=$HBASE_VERSION \ + -Papache-release clean site:site package "$@" + +rm -rf build +mkdir build +tar --strip-component=1 -C build -xzvf crunch-dist/target/apache-crunch-*-bin.tar.gz +cp -r target/site build http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-packages/src/common/crunch/install_crunch.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/crunch/install_crunch.sh b/bigtop-packages/src/common/crunch/install_crunch.sh new file mode 100644 index 0000000..d759882 --- /dev/null +++ b/bigtop-packages/src/common/crunch/install_crunch.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +usage() { + echo " +usage: $0 <options> + Required not-so-options: + --build-dir=DIR path to crunch build dir + --prefix=PREFIX path to install into + + Optional options: + --lib-dir=DIR path to install crunch home [/usr/lib/crunch] + --doc-dir=DIR path to install crunch docs [/usr/share/doc/crunch] + ... [ see source for more similar options ] + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'lib-dir:' \ + -l 'doc-dir:' \ + -l 'build-dir:' -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --build-dir) + BUILD_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --doc-dir) + DOC_DIR=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX BUILD_DIR ; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +LIB_DIR=${LIB_DIR:-/usr/lib/crunch} +DOC_DIR=${DOC_DIR:-/usr/share/doc/crunch} + +# First we'll move everything into lib +install -d -m 0755 $PREFIX/$LIB_DIR +cp $BUILD_DIR/*.jar $PREFIX/$LIB_DIR +install -d -m 0755 $PREFIX/$LIB_DIR/lib +cp $BUILD_DIR/lib/*.jar $PREFIX/$LIB_DIR/lib + +# Now take care of the docs +install -d -m 0755 $PREFIX/$DOC_DIR +cp $BUILD_DIR/DISCLAIMER $PREFIX/$DOC_DIR +cp $BUILD_DIR/LICENSE $PREFIX/$DOC_DIR +cp $BUILD_DIR/NOTICE $PREFIX/$DOC_DIR +cp $BUILD_DIR/README $PREFIX/$DOC_DIR +cp -r $BUILD_DIR/site/* $PREFIX/$DOC_DIR + +# Finally move some file from lib to docs +mv $PREFIX/$LIB_DIR/crunch-examples-*.jar $PREFIX/$DOC_DIR +mv $PREFIX/$LIB_DIR/crunch-test-*.jar $PREFIX/$DOC_DIR http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-packages/src/deb/crunch/changelog ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/crunch/changelog b/bigtop-packages/src/deb/crunch/changelog new file mode 100644 index 0000000..547ed02 --- /dev/null +++ b/bigtop-packages/src/deb/crunch/changelog @@ -0,0 +1 @@ +--- This is auto-generated http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-packages/src/deb/crunch/compat ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/crunch/compat b/bigtop-packages/src/deb/crunch/compat new file mode 100644 index 0000000..7f8f011 --- /dev/null +++ b/bigtop-packages/src/deb/crunch/compat @@ -0,0 +1 @@ +7 http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-packages/src/deb/crunch/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/crunch/control b/bigtop-packages/src/deb/crunch/control new file mode 100644 index 0000000..a05f621 --- /dev/null +++ b/bigtop-packages/src/deb/crunch/control @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Source: crunch +Section: misc +Priority: extra +Maintainer: Bigtop <[email protected]> +Build-Depends: debhelper (>= 7.0.50~) +Standards-Version: 3.8.0 +Homepage: http://incubator.apache.org/crunch/ + +Package: crunch +Architecture: all +Description: Simple and Efficient MapReduce Pipelines. + Apache Crunch (incubating) is a Java library for writing, testing, and running + MapReduce pipelines, based on Google's FlumeJava. Its goal is to make + pipelines that are composed of many user-defined functions simple to write, + easy to test, and efficient to run. + +Package: crunch-doc +Architecture: all +Description: Apache Crunch (incubating) documentation + Apache Crunch (incubating) documentation + http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-packages/src/deb/crunch/copyright ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/crunch/copyright b/bigtop-packages/src/deb/crunch/copyright new file mode 100644 index 0000000..be638e0 --- /dev/null +++ b/bigtop-packages/src/deb/crunch/copyright @@ -0,0 +1,15 @@ +Format: http://dep.debian.net/deps/dep5 +Source: http://incubator.apache.org/crunch/ +Upstream-Name: Apache Crunch (incubating) + +Files: * +Copyright: 2012, The Apache Software Foundation +License: Apache-2.0 + +Files debian/* +Copyright: 2011, The Apache Software Foundation +License: Apache-2.0 + +License: Apache-2.0 + On Debian systems, the complete text of the Apache 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-packages/src/deb/crunch/crunch-doc.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/crunch/crunch-doc.install b/bigtop-packages/src/deb/crunch/crunch-doc.install new file mode 100644 index 0000000..77b99dd --- /dev/null +++ b/bigtop-packages/src/deb/crunch/crunch-doc.install @@ -0,0 +1 @@ +/usr/share/doc/crunch http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-packages/src/deb/crunch/crunch.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/crunch/crunch.install b/bigtop-packages/src/deb/crunch/crunch.install new file mode 100644 index 0000000..a955cc9 --- /dev/null +++ b/bigtop-packages/src/deb/crunch/crunch.install @@ -0,0 +1 @@ +/usr/lib/crunch http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-packages/src/deb/crunch/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/crunch/rules b/bigtop-packages/src/deb/crunch/rules new file mode 100644 index 0000000..d3f973f --- /dev/null +++ b/bigtop-packages/src/deb/crunch/rules @@ -0,0 +1,41 @@ +#!/usr/bin/make -f +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# -*- makefile -*- + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +# This has to be exported to make some magic below work. +export DH_OPTIONS + + +%: + dh $@ + +override_dh_auto_build: + # we'll just use the build from the tarball. + # I could not find a way to add debian/ to RAT exclude list through cmd line + # or to unbind rat:check goal + # So I am redirecting its attention with a decoy + mkdir -p debian/tmp/bigtop-empty + bash debian/do-component-build -Drat.basedir=$${PWD}/debian/tmp/bigtop-empty -Dmaven.repo.local=${HOME}/.m2/repository + +override_dh_auto_install: + sh -x debian/install_crunch.sh \ + --build-dir=$${PWD}/build \ + --prefix=debian/tmp http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-packages/src/deb/crunch/source/format ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/crunch/source/format b/bigtop-packages/src/deb/crunch/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/bigtop-packages/src/deb/crunch/source/format @@ -0,0 +1 @@ +3.0 (quilt) http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec b/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec new file mode 100644 index 0000000..ece4775 --- /dev/null +++ b/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%define crunch_name crunch +%define lib_crunch /usr/lib/crunch +%define crunch_folder apache-%{crunch_name}-%{crunch_base_version}-src + +%if %{?suse_version:1}0 +%define doc_crunch %{_docdir}/crunch-doc +%else +%define doc_crunch %{_docdir}/crunch-doc-%{crunch_version} +%endif + +# disable repacking jars +%define __os_install_post %{nil} + +Name: crunch +Version: %{crunch_version} +Release: %{crunch_release} +Summary: Simple and Efficient MapReduce Pipelines. +URL: http://incubator.apache.org/crunch/ +Group: Development/Libraries +BuildArch: noarch +Buildroot: %(mktemp -ud %{_tmppath}/%{crunch_name}-%{version}-%{release}-XXXXXX) +License: ASL 2.0 +Source0: %{crunch_folder}.tar.gz +Source1: do-component-build +Source2: install_%{crunch_name}.sh + + +%description +Apache Crunch (incubating) is a Java library for writing, testing, and running +MapReduce pipelines, based on Google's FlumeJava. Its goal is to make +pipelines that are composed of many user-defined functions simple to write, +easy to test, and efficient to run. + +%package doc +Summary: Apache Crunch (incubating) documentation +Group: Documentation +%description doc +Apache Crunch (incubating) documentation + +%prep +%setup -n %{crunch_folder} + +%build +bash $RPM_SOURCE_DIR/do-component-build + +%install +%__rm -rf $RPM_BUILD_ROOT +sh $RPM_SOURCE_DIR/install_crunch.sh \ + --build-dir=${PWD}/build \ + --doc-dir=%{doc_crunch} \ + --prefix=$RPM_BUILD_ROOT + +####################### +#### FILES SECTION #### +####################### +%files +%defattr(-,root,root,755) +%{lib_crunch} + +%files doc +%defattr(-,root,root,755) +%{doc_crunch} http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml index 17b0baa..a192056 100644 --- a/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml +++ b/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml @@ -343,6 +343,23 @@ server.</description> <zookeeper>/self</zookeeper> </deps> </zookeeper-server> + <crunch> + <metadata> + <summary>Simple and Efficient MapReduce Pipelines.</summary> + <description>Apache Crunch (incubating) is a Java library for writing, testing, and running +MapReduce pipelines, based on Google's FlumeJava. Its goal is to make +pipelines that are composed of many user-defined functions simple to write, +easy to test, and efficient to run.</description> + <url></url> + </metadata> + </crunch> + <crunch-doc> + <metadata> + <summary>Apache Crunch (incubating) documentation</summary> + <description>Apache Crunch (incubating) documentation</description> + <url></url> + </metadata> + </crunch-doc> <pig> <metadata> <summary>Pig is a platform for analyzing large data sets</summary> http://git-wip-us.apache.org/repos/asf/bigtop/blob/57fbb284/bigtop.mk ---------------------------------------------------------------------- diff --git a/bigtop.mk b/bigtop.mk index f59437e..0af8b1d 100644 --- a/bigtop.mk +++ b/bigtop.mk @@ -211,6 +211,20 @@ SOLR_SITE=$(APACHE_MIRROR)$(SOLR_DOWNLOAD_PATH) SOLR_ARCHIVE=$(APACHE_ARCHIVE)$(SOLR_DOWNLOAD_PATH) $(eval $(call PACKAGE,solr,SOLR)) +# Crunch +CRUNCH_NAME=crunch +CRUNCH_RELNOTES_NAME=Java library for MapReduce pipelines +CRUNCH_PKG_NAME=crunch +CRUNCH_BASE_VERSION=0.4.0-incubating +CRUNCH_PKG_VERSION=0.4.0 +CRUNCH_RELEASE_VERSION=1 +CRUNCH_TARBALL_DST=apache-crunch-$(CRUNCH_BASE_VERSION)-src.tar.gz +CRUNCH_TARBALL_SRC=$(CRUNCH_TARBALL_DST) +CRUNCH_DOWNLOAD_PATH=/incubator/crunch/crunch-$(CRUNCH_BASE_VERSION) +CRUNCH_SITE=$(APACHE_MIRROR)$(CRUNCH_DOWNLOAD_PATH) +CRUNCH_ARCHIVE=$(CRUNCH_SITE) +$(eval $(call PACKAGE,crunch,CRUNCH)) + # Bigtop-utils BIGTOP_UTILS_NAME=bigtop-utils BIGTOP_UTILS__RELNOTES_NAME=Bigtop-utils
