Repository: bigtop Updated Branches: refs/heads/master 8bd519cd1 -> e5e065a7f
BIGTOP-1877. Upgrade Crunch to 0.12.0 Signed-off-by: Olaf Flebbe <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/e5e065a7 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/e5e065a7 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/e5e065a7 Branch: refs/heads/master Commit: e5e065a7f69cc68707b8dca396e43a782111f5d3 Parents: 8bd519c Author: YoungWoo Kim <[email protected]> Authored: Tue May 19 11:48:40 2015 +0900 Committer: Olaf Flebbe <[email protected]> Committed: Sat Jun 27 22:37:51 2015 +0200 ---------------------------------------------------------------------- bigtop-packages/src/common/crunch/do-component-build | 2 +- bigtop-packages/src/common/crunch/install_crunch.sh | 6 +++++- bigtop-packages/src/deb/crunch/rules | 13 +++++++++++++ bigtop-packages/src/rpm/crunch/SPECS/crunch.spec | 12 ++++++++++++ bigtop.mk | 4 ++-- 5 files changed, 33 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/e5e065a7/bigtop-packages/src/common/crunch/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/crunch/do-component-build b/bigtop-packages/src/common/crunch/do-component-build index 7fa11d5..ca68401 100644 --- a/bigtop-packages/src/common/crunch/do-component-build +++ b/bigtop-packages/src/common/crunch/do-component-build @@ -19,9 +19,9 @@ set -ex . `dirname $0`/bigtop.bom mvn -DskipTests -Dcrunch.platform=2 \ - -Dslf4j.version=1.6.1 \ -Dhadoop.version=$HADOOP_VERSION \ -Dhbase.version=$HBASE_VERSION-hadoop2 \ + -Dspark.version=$SPARK_VERSION \ -Papache-release clean site:site package "$@" rm -rf build http://git-wip-us.apache.org/repos/asf/bigtop/blob/e5e065a7/bigtop-packages/src/common/crunch/install_crunch.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/crunch/install_crunch.sh b/bigtop-packages/src/common/crunch/install_crunch.sh index 236047d..546191f 100644 --- a/bigtop-packages/src/common/crunch/install_crunch.sh +++ b/bigtop-packages/src/common/crunch/install_crunch.sh @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e +set -ex usage() { echo " @@ -96,3 +96,7 @@ cp -r $BUILD_DIR/site/* $PREFIX/$DOC_DIR # Finally move some file from lib to docs mv $PREFIX/$LIB_DIR/crunch-examples-*.jar $PREFIX/$DOC_DIR mv $PREFIX/$LIB_DIR/crunch-test-*.jar $PREFIX/$DOC_DIR + +rm -f $PREFIX/$LIB_DIR/lib/zookeeper-*.jar +rm -f $PREFIX/$LIB_DIR/lib/hadoop-*.jar + http://git-wip-us.apache.org/repos/asf/bigtop/blob/e5e065a7/bigtop-packages/src/deb/crunch/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/crunch/rules b/bigtop-packages/src/deb/crunch/rules index 7a9c858..aed4d77 100644 --- a/bigtop-packages/src/deb/crunch/rules +++ b/bigtop-packages/src/deb/crunch/rules @@ -23,6 +23,10 @@ export DH_VERBOSE=1 # This has to be exported to make some magic below work. export DH_OPTIONS +lib_crunch=/usr/lib/crunch +zookeeper_home=/usr/lib/zookeeper +hadoop_home=/usr/lib/hadoop + %: dh $@ @@ -39,3 +43,12 @@ override_dh_auto_install: bash -x debian/install_crunch.sh \ --build-dir=$${PWD}/build \ --prefix=debian/tmp + + ln -fs ${zookeeper_home}/zookeeper.jar debian/tmp/${lib_crunch}/lib/ + + ln -fs ${hadoop_home}/client/hadoop-annotations.jar debian/tmp/${lib_crunch}/lib/ + ln -fs ${hadoop_home}/client/hadoop-auth.jar debian/tmp/${lib_crunch}/lib/ + ln -fs ${hadoop_home}/client/hadoop-common.jar debian/tmp/${lib_crunch}/lib/ + ln -fs ${hadoop_home}/client/hadoop-mapreduce-client-core.jar debian/tmp/${lib_crunch}/lib/ + ln -fs ${hadoop_home}/client/hadoop-yarn-api.jar debian/tmp/${lib_crunch}/lib/ + ln -fs ${hadoop_home}/client/hadoop-yarn-common.jar debian/tmp/${lib_crunch}/lib/ http://git-wip-us.apache.org/repos/asf/bigtop/blob/e5e065a7/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec b/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec index 0f7fb9b..fd958f5 100644 --- a/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec +++ b/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec @@ -16,6 +16,8 @@ %define crunch_name crunch %define lib_crunch /usr/lib/crunch %define crunch_folder apache-%{crunch_name}-%{crunch_base_version}-src +%define zookeeper_home /usr/lib/zookeeper +%define hadoop_home /usr/lib/hadoop %if %{?suse_version:1}0 %define doc_crunch %{_docdir}/crunch-doc @@ -38,6 +40,7 @@ License: ASL 2.0 Source0: %{crunch_folder}.tar.gz Source1: do-component-build Source2: install_%{crunch_name}.sh +Source3: bigtop.bom Requires: hadoop-client, bigtop-utils >= 0.7 %description @@ -65,6 +68,15 @@ bash $RPM_SOURCE_DIR/install_crunch.sh \ --doc-dir=%{doc_crunch} \ --prefix=$RPM_BUILD_ROOT +ln -fs %{zookeeper_home}/zookeeper.jar $RPM_BUILD_ROOT/%{lib_crunch}/lib/ + +ln -fs %{hadoop_home}/client/hadoop-annotations.jar $RPM_BUILD_ROOT/%{lib_crunch}/lib/ +ln -fs %{hadoop_home}/client/hadoop-auth.jar $RPM_BUILD_ROOT/%{lib_crunch}/lib/ +ln -fs %{hadoop_home}/client/hadoop-common.jar $RPM_BUILD_ROOT/%{lib_crunch}/lib/ +ln -fs %{hadoop_home}/client/hadoop-mapreduce-client-core.jar $RPM_BUILD_ROOT/%{lib_crunch}/lib/ +ln -fs %{hadoop_home}/client/hadoop-yarn-api.jar $RPM_BUILD_ROOT/%{lib_crunch}/lib/ +ln -fs %{hadoop_home}/client/hadoop-yarn-common.jar $RPM_BUILD_ROOT/%{lib_crunch}/lib/ + ####################### #### FILES SECTION #### ####################### http://git-wip-us.apache.org/repos/asf/bigtop/blob/e5e065a7/bigtop.mk ---------------------------------------------------------------------- diff --git a/bigtop.mk b/bigtop.mk index d6f9507..1079831 100644 --- a/bigtop.mk +++ b/bigtop.mk @@ -248,8 +248,8 @@ $(eval $(call PACKAGE,solr,SOLR)) CRUNCH_NAME=crunch CRUNCH_RELNOTES_NAME=Java library for MapReduce pipelines CRUNCH_PKG_NAME=crunch -CRUNCH_BASE_VERSION=0.10.0 -CRUNCH_PKG_VERSION=0.10.0 +CRUNCH_BASE_VERSION=0.12.0 +CRUNCH_PKG_VERSION=$(CRUNCH_BASE_VERSION) CRUNCH_RELEASE_VERSION=1 CRUNCH_TARBALL_DST=apache-crunch-$(CRUNCH_BASE_VERSION)-src.tar.gz CRUNCH_TARBALL_SRC=$(CRUNCH_TARBALL_DST)
