Repository: bigtop Updated Branches: refs/heads/master b00e9da8f -> f506701c8
BIGTOP-1179. Add Apache Tez to bigtop Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/f506701c Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/f506701c Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/f506701c Branch: refs/heads/master Commit: f506701c8e8cd1b9e596c3f728996c988705e7d0 Parents: b00e9da Author: Olaf Flebbe <o.fle...@science-computing.de> Authored: Thu Nov 27 11:54:41 2014 +0100 Committer: Olaf Flebbe <o...@oflebbe.de> Committed: Mon Feb 9 21:49:50 2015 +0100 ---------------------------------------------------------------------- .../src/common/tez/do-component-build | 31 +++++ bigtop-packages/src/common/tez/install_tez.sh | 122 +++++++++++++++++++ bigtop-packages/src/common/tez/tez-site.xml | 27 ++++ bigtop-packages/src/common/tez/tez.1 | 27 ++++ bigtop-packages/src/deb/tez/compat | 1 + bigtop-packages/src/deb/tez/control | 29 +++++ bigtop-packages/src/deb/tez/copyright | 16 +++ bigtop-packages/src/deb/tez/rules | 42 +++++++ bigtop-packages/src/deb/tez/source/format | 1 + bigtop-packages/src/deb/tez/tez.install | 3 + bigtop-packages/src/rpm/tez/RPMS/.gitignore | 1 + bigtop-packages/src/rpm/tez/SPECS/tez.spec | 110 +++++++++++++++++ bigtop.mk | 14 +++ 13 files changed, 424 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/common/tez/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/tez/do-component-build b/bigtop-packages/src/common/tez/do-component-build new file mode 100644 index 0000000..88f802a --- /dev/null +++ b/bigtop-packages/src/common/tez/do-component-build @@ -0,0 +1,31 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +. `dirname $0`/bigtop.bom + +BUILD_TEZ_OPTS="clean package \ +-Dtar -Dhadoop.version=${HADOOP_VERSION} \ +-Phadoop24 -P!hadoop26 +-DskipTests" + +#mvn versions:set -DnewVersion=${TEZ_VERSION} + +mvn ${BUILD_TEZ_OPTS} "$@" + + + http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/common/tez/install_tez.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/tez/install_tez.sh b/bigtop-packages/src/common/tez/install_tez.sh new file mode 100644 index 0000000..c533dcd --- /dev/null +++ b/bigtop-packages/src/common/tez/install_tez.sh @@ -0,0 +1,122 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +usage() { + echo " +usage: $0 <options> + Required not-so-options: + --build-dir=DIR path to tez dist.dir + --prefix=PREFIX path to install into + + Optional options: + --doc-dir=DIR path to install docs into [/usr/share/doc/tez] + --lib-dir=DIR path to install tez home [/usr/lib/tez] + --installed-lib-dir=DIR path where lib-dir will end up on target system + --bin-dir=DIR path to install bins [/usr/bin] + --examples-dir=DIR path to install examples [doc-dir/examples] + ... [ see source for more similar options ] + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'doc-dir:' \ + -l 'lib-dir:' \ + -l 'installed-lib-dir:' \ + -l 'bin-dir:' \ + -l 'examples-dir:' \ + -l 'conf-dir:' \ + -l 'sbin-dir:' \ + -l 'libexec-dir:' \ + -l 'build-dir:' -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --build-dir) + BUILD_DIR=$2 ; shift 2 + ;; + --doc-dir) + DOC_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --installed-lib-dir) + INSTALLED_LIB_DIR=$2 ; shift 2 + ;; + --examples-dir) + EXAMPLES_DIR=$2 ; shift 2 + ;; + --libexec-dir) + LIBEXEC_DIR=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX BUILD_DIR ; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +MAN_DIR=${MAN_DIR:-/usr/share/man/man1} +DOC_DIR=${DOC_DIR:-/usr/share/doc/tez} +LIB_DIR=${LIB_DIR:-/usr/lib/tez} +CONF_DIR=${CONF_DIR:-/etc/tez/conf} +LIBEXEC_DIR=${LIBEXEC_DIR:-${LIB_DIR}/tez} + +install -d -m 0755 $PREFIX/$MAN_DIR +gzip -c tez.1 > $PREFIX/$MAN_DIR/tez.1.gz + +install -d -m 0755 $PREFIX/$LIB_DIR +install -d -m 0755 $PREFIX/$LIB_DIR/lib +install -d -m 0755 $PREFIX/$DOC_DIR +install -d -m 0755 $PREFIX/$CONF_DIR +install -d -m 0755 $PREFIX/$MAN_DIR +install -d -m 0755 $PREFIX/$LIBEXEC_DIR + +cp tez-site.xml $PREFIX/$CONF_DIR + +tar -C $PREFIX/$LIB_DIR -xzf $BUILD_DIR/tez-dist/target/tez*-minimal.tar.gz + +#test ! -d $BUILD_DIR/lib || cp -ra $BUILD_DIR/lib/* ${PREFIX}/${LIB_DIR}/lib/ +#test ! -d $BUILD_DIR || cp $BUILD_DIR/tez*.jar $PREFIX/$LIB_DIR +#test ! -d $BUILD_DIR/docs || cp -a $BUILD_DIR/docs/* $PREFIX/$DOC_DIR +#test ! -d $BUILD_DIR/libexec || cp -a $BUILD_DIR/libexec/* $PREFIX/$LIBEXEC_DIR + http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/common/tez/tez-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/tez/tez-site.xml b/bigtop-packages/src/common/tez/tez-site.xml new file mode 100644 index 0000000..cd29d26 --- /dev/null +++ b/bigtop-packages/src/common/tez/tez-site.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<configuration> +<property> +<name>tez.lib.uris</name> +<value>${fs.default.name}/apps/tez/,${fs.default.name}/apps/tez/lib/</value> +</property> + +<property> +<name>tez.use.cluster.hadoop-libs</name> +<value>true</value> +</property> +</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/common/tez/tez.1 ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/tez/tez.1 b/bigtop-packages/src/common/tez/tez.1 new file mode 100644 index 0000000..177d1d7 --- /dev/null +++ b/bigtop-packages/src/common/tez/tez.1 @@ -0,0 +1,27 @@ +.\" Licensed to the Apache Software Foundation (ASF) under one or more +.\" contributor license agreements. See the NOTICE file distributed with +.\" this work for additional information regarding copyright ownership. +.\" The ASF licenses this file to You under the Apache License, Version 2.0 +.\" (the "License"); you may not use this file except in compliance with +.\" the License. You may obtain a copy of the License at +.\" +.\" http://www.apache.org/licenses/LICENSE-2.0 +.\" +.\" Unless required by applicable law or agreed to in writing, software +.\" distributed under the License is distributed on an "AS IS" BASIS, +.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.\" See the License for the specific language governing permissions and +.\" limitations under the License. +.\" +.\" Process this file with +.\" groff -man -Tascii tez.1 +.\" +.TH tez 1 "January 2014 " Linux "User Manuals" + +.SH NAME +The Apache Tez project is aimed at building an application framework +which allows for a complex directed-acyclic-graph of tasks for +processing data. It is currently built atop Apache Hadoop YARN + +.SH COPYRIGHT +Copyright (C) 2010 The Apache Software Foundation. All rights reserved. http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/compat ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/tez/compat b/bigtop-packages/src/deb/tez/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/bigtop-packages/src/deb/tez/compat @@ -0,0 +1 @@ +9 http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/tez/control b/bigtop-packages/src/deb/tez/control new file mode 100644 index 0000000..93e6604 --- /dev/null +++ b/bigtop-packages/src/deb/tez/control @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Source: tez +Section: misc +Priority: extra +Maintainer: Bigtop <d...@bigtop.apache.org> +Build-Depends: debhelper (>= 9) +Standards-Version: 3.9.4 +Homepage: http://tez.apache.org/ + +Package: tez +Architecture: all +Depends: hadoop-hdfs (>= 2.4.1), bigtop-utils +Description: Framework for building complex DAG of tasks. + The Apache Tez project is aimed at building an application framework + which allows for a complex directed-acyclic-graph of tasks for + processing data. It is currently built atop Apache Hadoop YARN \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/copyright ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/tez/copyright b/bigtop-packages/src/deb/tez/copyright new file mode 100644 index 0000000..17bcffe --- /dev/null +++ b/bigtop-packages/src/deb/tez/copyright @@ -0,0 +1,16 @@ +Format: http://dep.debian.net/deps/dep5 +Source: http://tez.apache.org +Upstream-Name: Apache Tez Incubator + +Files: * +Copyright: 2008-2014, The Apache Software Foundation +License: Apache-2.0 + +Files debian/* +Copyright: 2014, The Apache Software Foundation +License: Apache-2.0 + +License: Apache-2.0 + On Debian systems, the complete text of the Apache 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". + http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/tez/rules b/bigtop-packages/src/deb/tez/rules new file mode 100644 index 0000000..467de0a --- /dev/null +++ b/bigtop-packages/src/deb/tez/rules @@ -0,0 +1,42 @@ +#!/usr/bin/make -f +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# -*- makefile -*- +SHELL=/bin/bash + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +# This has to be exported to make some magic below work. +export DH_OPTIONS + +tez_pkg_name=tez + +%: + dh $@ + +override_dh_auto_build: + bash debian/do-component-build -Dmaven.repo.local=${HOME}/.m2/repository + + +override_dh_auto_install: + cp debian/tez.1 . + cp debian/tez-site.xml . + bash -x debian/install_tez.sh \ + --build-dir=. \ + --doc-dir=usr/share/doc/${tez_pkg_name}-doc \ + --prefix=debian/tmp http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/source/format ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/tez/source/format b/bigtop-packages/src/deb/tez/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/bigtop-packages/src/deb/tez/source/format @@ -0,0 +1 @@ +3.0 (quilt) http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/tez.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/tez/tez.install b/bigtop-packages/src/deb/tez/tez.install new file mode 100644 index 0000000..dbcba69 --- /dev/null +++ b/bigtop-packages/src/deb/tez/tez.install @@ -0,0 +1,3 @@ +usr/lib/tez +usr/share/man/man1 +etc/tez/conf http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/rpm/tez/RPMS/.gitignore ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/tez/RPMS/.gitignore b/bigtop-packages/src/rpm/tez/RPMS/.gitignore new file mode 100644 index 0000000..1ab8c79 --- /dev/null +++ b/bigtop-packages/src/rpm/tez/RPMS/.gitignore @@ -0,0 +1 @@ +repodata http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/rpm/tez/SPECS/tez.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/tez/SPECS/tez.spec b/bigtop-packages/src/rpm/tez/SPECS/tez.spec new file mode 100644 index 0000000..0082056 --- /dev/null +++ b/bigtop-packages/src/rpm/tez/SPECS/tez.spec @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +%define tez_home /usr/lib/%{name} +%define lib_tez %{tez_home}/lib +%define man_dir %{_mandir} + + +%if %{!?suse_version:1}0 && %{!?mgaversion:1}0 + +%define __os_install_post \ + /usr/lib/rpm/redhat/brp-compress ; \ + /usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} ; \ + /usr/lib/rpm/redhat/brp-strip-comment-note %{__strip} %{__objdump} ; \ + /usr/lib/rpm/brp-python-bytecompile ; \ + %{nil} + +%define doc_tez %{_docdir}/tez-%{tez_version} + +%endif + + +%if %{?suse_version:1}0 + +# Only tested on openSUSE 11.4. le'ts update it for previous +#release when confirmed +%if 0%{suse_version} > 1130 +# Define an empty suse_check for compatibility with older sles +%define suse_check \ +%endif + +%define doc_tez %{_docdir}/tez +%define alternatives_cmd update-alternatives +%define __os_install_post \ + %{suse_check} ; \ + /usr/lib/rpm/brp-compress ; \ + %{nil} + +%endif + +Name: tez +Version: %{tez_version} +Release: %{tez_release} +Summary:Apache Tez is the Hadoop enhanced Map/Reduce module. +URL: http://tez.apache.org +Group: Development/Libraries +Buildroot: %{_topdir}/INSTALL/%{name}-%{version} +License: Apache License v2.0 +Source0: apache-%{name}-%{tez_base_version}-src.tar.gz +Source1: do-component-build +Source2: install_tez.sh +Source3: tez.1 +Source4: tez-site.xml +BuildArch: noarch +Requires: hadoop hadoop-hdfs hadoop-yarn hadoop-mapreduce + +%if 0%{?mgaversion} +Requires: bsh-utils +%else +Requires: sh-utils +%endif + + +%description +The Apache Tez project is aimed at building an application framework +which allows for a complex directed-acyclic-graph of tasks for +processing data. It is currently built atop Apache Hadoop YARN + +%prep +%setup -q -n %{name}-%{tez_base_version} + +%build +env TEZ_VERSION=%{version} bash %{SOURCE1} + +%install +%__rm -rf $RPM_BUILD_ROOT + +cp %{SOURCE3} %{SOURCE4} . +sh %{SOURCE2} \ + --build-dir=. \ + --doc-dir=%{doc_tez} \ + --libexec-dir=%{libexec_tez} \ + --prefix=$RPM_BUILD_ROOT + +%pre + +%post + +%preun + +####################### +#### FILES SECTION #### +####################### +%files +%defattr(-,root,root) +%{tez_home} +%doc %{doc_tez} +%{man_dir}/man1/tez.1.* +%{tez_home}/tez-*.jar http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop.mk ---------------------------------------------------------------------- diff --git a/bigtop.mk b/bigtop.mk index af9045b..3925992 100644 --- a/bigtop.mk +++ b/bigtop.mk @@ -103,6 +103,20 @@ HIVE_SITE=$(APACHE_MIRROR)$(HIVE_DOWNLOAD_PATH) HIVE_ARCHIVE=$(APACHE_ARCHIVE)$(HIVE_DOWNLOAD_PATH) $(eval $(call PACKAGE,hive,HIVE)) +# TEZ +TEZ_NAME=tez +TEZ_RELNOTES_NAME=Apache TEZ +TEZ_PKG_NAME=tez +TEZ_BASE_VERSION=0.6.0 +TEZ_PKG_VERSION=$(TEZ_BASE_VERSION) +TEZ_RELEASE_VERSION=1 +TEZ_TARBALL_DST=apache-tez-$(TEZ_BASE_VERSION)-src.tar.gz +TEZ_TARBALL_SRC=$(TEZ_TARBALL_DST) +TEZ_DOWNLOAD_PATH=/tez/$(TEZ_BASE_VERSION) +TEZ_SITE=$(APACHE_MIRROR)$(TEZ_DOWNLOAD_PATH) +TEZ_ARCHIVE=$(APACHE_ARCHIVE)$(TEZ_DOWNLOAD_PATH) +$(eval $(call PACKAGE,tez,TEZ)) + # Sqoop SQOOP_NAME=sqoop SQOOP_RELNOTES_NAME=Sqoop