Repository: bigtop Updated Branches: refs/heads/master 2d5fd34fd -> 8c0d359fc
BIGTOP-1594: Upgrade Pig to 0.14.0 Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/8c0d359f Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/8c0d359f Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/8c0d359f Branch: refs/heads/master Commit: 8c0d359fcf17bb4497b9ac7a080587b409041d38 Parents: 2d5fd34 Author: Youngwoo Kim <[email protected]> Authored: Wed Jan 14 19:00:42 2015 +0900 Committer: Roman Shaposhnik <[email protected]> Committed: Wed Jan 21 16:37:37 2015 -0800 ---------------------------------------------------------------------- .../src/common/pig/do-component-build | 35 ++-------- bigtop-packages/src/common/pig/install_pig.sh | 30 ++++----- bigtop-packages/src/common/pig/log4j.properties | 23 ------- bigtop-packages/src/common/pig/pig.properties | 69 -------------------- bigtop-packages/src/deb/pig/control | 2 +- bigtop-packages/src/deb/pig/rules | 19 +++++- bigtop-packages/src/rpm/pig/SPECS/pig.spec | 32 ++++++--- bigtop.mk | 4 +- 8 files changed, 62 insertions(+), 152 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/8c0d359f/bigtop-packages/src/common/pig/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/pig/do-component-build b/bigtop-packages/src/common/pig/do-component-build index 46740c1..f7dc291 100644 --- a/bigtop-packages/src/common/pig/do-component-build +++ b/bigtop-packages/src/common/pig/do-component-build @@ -20,7 +20,7 @@ set -ex cat <<__EOT__ >> ivy/libraries.properties -hbase.version=$HBASE_VERSION +hbase95.version=$HBASE_VERSION-hadoop2 hive.version=$HIVE_VERSION zookeeper.version=$ZOOKEEPER_VERSION hadoop-common.version=$HADOOP_VERSION @@ -28,38 +28,13 @@ hadoop-hdfs.version=$HADOOP_VERSION hadoop-mapreduce.version=$HADOOP_VERSION __EOT__ -# Workaround because renamed o.a.h.h.serde.Constants to o.a.h.h.serde.serdeConstants in a non-backwards compatible way in 0.10.0 -# This is being fixed via HIVE-4001. Please remove this workaround once that gets into a release (Hive 0.11?) -sed -i -e "s/org.apache.hadoop.hive.serde.Constants/org.apache.hadoop.hive.serde.serdeConstants/g" \ --e "s/Constants\./serdeConstants\./g" contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/HiveColumnarLoader.java - -# FIXME PIG-3905 -sed -i -e '/.*<dependency.*/ {N; s/.*<dependency.*\n.*hadoop20.*//g}' ivy.xml - -BUILD_OPTS="-Djavac.version=1.6 -Dforrest.home=${FORREST_HOME} -Dversion=${PIG_VERSION} -Dhadoopversion=23" - -# FIXME: BIGTOP-894. Please get rid of the line below when Bigtop integrates a version of Pig that includes the fix for PIG-3262 -export LC_ALL=C.UTF-8 - -# FIXME: there's a weird synchronization issue between ivy and Maven caches wrt. jetty -# If we don't do the following the build fails with (note zip instead of jar): -# [ivy:resolve] :::::::::::::::::::::::::::::::::::::::::::::: -# [ivy:resolve] :: FAILED DOWNLOADS :: -# [ivy:resolve] :: ^ see resolution messages for details ^ :: -# [ivy:resolve] :::::::::::::::::::::::::::::::::::::::::::::: -# [ivy:resolve] :: org.mortbay.jetty#jetty;6.1.26!jetty.zip -# [ivy:resolve] :::::::::::::::::::::::::::::::::::::::::::::: -for h in ${HOME} `mvn help:system | grep '^user.home=' | cut -f2 -d=` ; do - rm -rf ${h}/.ivy2/cache/*jetty ${h}/.m2/repository/org/mortbay ${h}/.m2/repository/org/eclipse/jetty -done +BUILD_OPTS="-Djavac.version=${JDK_VERSION} -Dforrest.home=${FORREST_HOME} -Dversion=${PIG_VERSION} -Dhadoopversion=23" echo "forrest.maxmemory=256m" >> src/docs/forrest.properties -ant $BUILD_OPTS clean jar jar-withouthadoop pigunit-jar smoketests-jar "$@" -for build_file in contrib/piggybank/java/build.xml contrib/zebra/build.xml ; do +export ANT_OPTS="-Xmx4096m" +ant $BUILD_OPTS clean jar pigunit-jar smoketests-jar javadoc "$@" +for build_file in contrib/piggybank/java/build.xml ; do ant $BUILD_OPTS -buildfile $build_file clean jar "$@" done ant $BUILD_OPTS tar "$@" - -# install the Pig jar into the local Maven cache -mvn install:install-file -DcreateChecksum=true -DgroupId=org.apache.pig -DartifactId=pig -Dversion=${PIG_VERSION} -Dpackaging=jar -Dfile=pig.jar http://git-wip-us.apache.org/repos/asf/bigtop/blob/8c0d359f/bigtop-packages/src/common/pig/install_pig.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/pig/install_pig.sh b/bigtop-packages/src/common/pig/install_pig.sh index f297712..6bf0782 100644 --- a/bigtop-packages/src/common/pig/install_pig.sh +++ b/bigtop-packages/src/common/pig/install_pig.sh @@ -109,21 +109,14 @@ find $PREFIX/$LIB_DIR/contrib -name \*.jar -exec cp {} $PREFIX/$LIB_DIR \; cp $BUILD_DIR/pig-*-smoketests.jar $PREFIX/$LIB_DIR/ # Remove directories that are going elsewhere -for dir in shims conf src lib-src docs tutorial test build.xml contrib ivy pig-*.stage.jar ivy.xml build.properties +for dir in shims src lib-src docs tutorial test build.xml contrib ivy pig-*.stage.jar ivy.xml build.properties legacy scripts conf do rm -rf $PREFIX/$LIB_DIR/$dir done -# Remove a fat JAR that contains system Hadoop dependencies -for jar in $PREFIX/$LIB_DIR/pig*.jar ; do - if jar tvf $jar | fgrep -q ' org/apache/hadoop/hdfs' ; then - rm -f $jar - fi -done - # Copy in the configuration files install -d -m 0755 $PREFIX/$CONF_DIST_DIR -cp *.properties $PREFIX/$CONF_DIST_DIR +cp $BUILD_DIR/tar/pig*/conf/* $PREFIX/$CONF_DIST_DIR ln -s /etc/pig/conf $PREFIX/$LIB_DIR/conf # Copy in the /usr/bin/pig wrapper @@ -134,8 +127,13 @@ cat > $PREFIX/$BIN_DIR/pig <<EOF # Autodetect JAVA_HOME if not defined . /usr/lib/bigtop-utils/bigtop-detect-javahome -BIGTOP_DEFAULTS_DIR=\${BIGTOP_DEFAULTS_DIR-/etc/default} -[ -n "\${BIGTOP_DEFAULTS_DIR}" -a -r \${BIGTOP_DEFAULTS_DIR}/hbase ] && . \${BIGTOP_DEFAULTS_DIR}/hbase +export HADOOP_PREFIX=\${HADOOP_PREFIX:-/usr/lib/hadoop} +export HADOOP_CONF_DIR=\${HADOOP_CONF_DIR:-/etc/hadoop/conf} +export HBASE_HOME=\${HBASE_HOME:-/usr/lib/hbase} +export HBASE_CONF_DIR=\${HBASE_HOME}/conf +export ZOOKEEPER_HOME=\${ZOOKEEPER_HOME:-/usr/lib/zookeeper} +export HIVE_HOME=\${HIVE_HOME:-/usr/lib/hive} +export HCAT_HOME=\${HCAT_HOME:-/usr/lib/hive-hcatalog} exec $INSTALLED_LIB_DIR/bin/pig "\$@" EOF @@ -149,14 +147,12 @@ install -d -m 0755 $PREFIX/$DOC_DIR (cd $BUILD_DIR/tar/pig*/docs && tar -cf - .)|(cd $PREFIX/$DOC_DIR && tar -xf -) mv $PREFIX/$LIB_DIR/license $PREFIX/$DOC_DIR +# Tutorial install -d -m 0755 $PREFIX/$EXAMPLES_DIR -(cd $PREFIX/$LIB_DIR ; ln -s pig*withouthadoop.jar pig.jar) (cd $BUILD_DIR/tar/pig*/tutorial && tar -cf - .)|(cd $PREFIX/$EXAMPLES_DIR && tar -xf -) -sed -i -e "s|../pig.jar|/usr/lib/pig/pig.jar|" $PREFIX/$EXAMPLES_DIR/build.xml - -# It's somewhat silly that the hadoop jars are included in the pig lib -# dir, since we depend on hadoop in our packages. We can rm them -rm -f $PREFIX/$LIB_DIR/lib/hadoop*jar +sed -i -e "s|../build/ivy/lib/Pig|/usr/lib/pig/lib/|" $PREFIX/$EXAMPLES_DIR/build.xml +sed -i -e "s|<fileset dir=\"..\">|<fileset dir=\"/usr/lib/pig/\">|" $PREFIX/$EXAMPLES_DIR/build.xml +sed -i -e "s|<path id=\"tutorial.classpath\">|<path id=\"tutorial.classpath\"> <fileset dir=\"/usr/lib/hadoop\"> <include name=\"*.jar\"/> </fileset>|" $PREFIX/$EXAMPLES_DIR/build.xml # Pig log directory install -d -m 1777 $PREFIX/var/log/pig http://git-wip-us.apache.org/repos/asf/bigtop/blob/8c0d359f/bigtop-packages/src/common/pig/log4j.properties ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/pig/log4j.properties b/bigtop-packages/src/common/pig/log4j.properties deleted file mode 100644 index 5d3e510..0000000 --- a/bigtop-packages/src/common/pig/log4j.properties +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ***** Set root logger level to DEBUG and its only appender to A. -log4j.logger.org.apache.pig=info, A - -# ***** A is set to be a ConsoleAppender. -log4j.appender.A=org.apache.log4j.ConsoleAppender -# ***** A uses PatternLayout. -log4j.appender.A.layout=org.apache.log4j.PatternLayout -log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n http://git-wip-us.apache.org/repos/asf/bigtop/blob/8c0d359f/bigtop-packages/src/common/pig/pig.properties ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/pig/pig.properties b/bigtop-packages/src/common/pig/pig.properties deleted file mode 100644 index 037286d..0000000 --- a/bigtop-packages/src/common/pig/pig.properties +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Pig configuration file. All values can be overwritten by command line arguments. -# see bin/pig -help - -# log4jconf log4j configuration file -# log4jconf=./conf/log4j.properties - -# brief logging (no timestamps) -brief=false - -# clustername, name of the hadoop jobtracker. If no port is defined port 50020 will be used. -#cluster - -#debug level, INFO is default -debug=INFO - -# a file that contains pig script -#file= - -# load jarfile, colon separated -#jar= - -#verbose print all log messages to screen (default to print only INFO and above to screen) -verbose=false - -#exectype local|mapreduce, mapreduce is default -#exectype=mapreduce -# hod realted properties -#ssh.gateway -#hod.expect.root -#hod.expect.uselatest -#hod.command -#hod.config.dir -#hod.param - - -#Do not spill temp files smaller than this size (bytes) -pig.spill.size.threshold=5000000 -#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) -#This should help reduce the number of files being spilled. -pig.spill.gc.activation.size=40000000 - - -###################### -# Everything below this line is Yahoo specific. Note that I've made -# (almost) no changes to the lines above to make merging in from Apache -# easier. Any values I don't want from above I override below. -# -# This file is configured for use with HOD on the production clusters. If you -# want to run pig with a static cluster you will need to remove everything -# below this line and set the cluster value (above) to the -# hostname and port of your job tracker. - -exectype=mapreduce -log.file= http://git-wip-us.apache.org/repos/asf/bigtop/blob/8c0d359f/bigtop-packages/src/deb/pig/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/pig/control b/bigtop-packages/src/deb/pig/control index 4cfe4ba..85b6423 100644 --- a/bigtop-packages/src/deb/pig/control +++ b/bigtop-packages/src/deb/pig/control @@ -23,7 +23,7 @@ Homepage: http://pig.apache.org/ Package: pig Architecture: all -Depends: hadoop-client, bigtop-utils (>= 0.7) +Depends: hadoop-client, hbase, hive, zookeeper, bigtop-utils (>= 0.7) Description: Pig is a platform for analyzing large data sets Pig is a platform for analyzing large data sets that consists of a high-level language for expressing data analysis programs, coupled with infrastructure for evaluating these http://git-wip-us.apache.org/repos/asf/bigtop/blob/8c0d359f/bigtop-packages/src/deb/pig/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/pig/rules b/bigtop-packages/src/deb/pig/rules index 78fa232..bb27873 100755 --- a/bigtop-packages/src/deb/pig/rules +++ b/bigtop-packages/src/deb/pig/rules @@ -23,6 +23,9 @@ # This has to be exported to make some magic below work. export DH_OPTIONS +hive_home=/usr/lib/hive +zookeeper_home=/usr/lib/zookeeper +hbase_home=/usr/lib/hbase pig_pkg_name=pig %: @@ -32,10 +35,24 @@ override_dh_auto_build: bash debian/do-component-build -Divy.home=${HOME}/.ivy2 override_dh_auto_install: - cp debian/*.properties . cp debian/pig.1 . sh -x debian/install_pig.sh \ --build-dir=build \ --prefix=debian/${pig_pkg_name} \ --doc-dir=/usr/share/doc/${pig_pkg_name} \ --examples-dir=/usr/share/doc/${pig_pkg_name}/examples + + rm -f debian/pig/usr/lib/${pig_pkg_name}/lib/{hive,zookeeper}*.jar + ln -f -s ${zookeeper_home}/zookeeper.jar debian/pig/usr/lib/${pig_pkg_name}/lib/ + ln -f -s ${hive_home}/lib/hive-common.jar debian/pig/usr/lib/${pig_pkg_name}/lib/ + ln -f -s ${hive_home}/lib/hive-exec.jar debian/pig/usr/lib/${pig_pkg_name}/lib/ + ln -f -s ${hive_home}/lib/hive-serde.jar debian/pig/usr/lib/${pig_pkg_name}/lib/ + ln -f -s ${hive_home}/lib/hive-shims-common.jar debian/pig/usr/lib/${pig_pkg_name}/lib/ + ln -f -s ${hive_home}/lib/hive-shims-common-secure.jar debian/pig/usr/lib/${pig_pkg_name}/lib/ + rm -f debian/pig/usr/lib/${pig_pkg_name}/lib/h2/hbase-*.jar + ln -f -s ${hbase_home}/hbase-client.jar debian/pig/usr/lib/${pig_pkg_name}/lib/h2/ + ln -f -s ${hbase_home}/hbase-common.jar debian/pig/usr/lib/${pig_pkg_name}/lib/h2/ + ln -f -s ${hbase_home}/hbase-hadoop2-compat.jar debian/pig/usr/lib/${pig_pkg_name}/lib/h2/ + ln -f -s ${hbase_home}/hbase-hadoop-compat.jar debian/pig/usr/lib/${pig_pkg_name}/lib/h2/ + ln -f -s ${hbase_home}/hbase-protocol.jar debian/pig/usr/lib/${pig_pkg_name}/lib/h2/ + ln -f -s ${hbase_home}/hbase-server.jar debian/pig/usr/lib/${pig_pkg_name}/lib/h2/ http://git-wip-us.apache.org/repos/asf/bigtop/blob/8c0d359f/bigtop-packages/src/rpm/pig/SPECS/pig.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/pig/SPECS/pig.spec b/bigtop-packages/src/rpm/pig/SPECS/pig.spec index 86fba00..370b7aa 100644 --- a/bigtop-packages/src/rpm/pig/SPECS/pig.spec +++ b/bigtop-packages/src/rpm/pig/SPECS/pig.spec @@ -20,6 +20,9 @@ %define bin_pig /usr/bin %define pig_config_virtual pig_active_configuration %define man_dir %{_mandir} +%define hive_home /usr/lib/hive +%define zookeeper_home /usr/lib/zookeeper +%define hbase_home /usr/lib/hbase # CentOS 5 does not have any dist macro # So I will suppose anything that is not Mageia or a SUSE will be a RHEL/CentOS/Fedora @@ -75,14 +78,12 @@ URL: http://pig.apache.org/ Group: Development/Libraries Buildroot: %{_topdir}/INSTALL/%{name}-%{version} BuildArch: noarch -Source0: pig-%{pig_base_version}.tar.gz +Source0: pig-%{pig_base_version}-src.tar.gz Source1: do-component-build Source2: install_pig.sh -Source3: log4j.properties -Source4: pig.1 -Source5: pig.properties -Source6: bigtop.bom -Requires: hadoop-client, bigtop-utils >= 0.7 +Source3: pig.1 +Source4: bigtop.bom +Requires: hadoop-client, hbase, hive, zookeeper, bigtop-utils >= 0.7 %description Pig is a platform for analyzing large data sets that consists of a high-level language @@ -108,7 +109,7 @@ language called Pig Latin, which has the following key properties: %prep -%setup -n %{name}-%{pig_base_version} +%setup -n %{name}-%{pig_base_version}-src %build env PIG_BASE_VERSION=%{pig_base_version} bash %{SOURCE1} @@ -120,14 +121,27 @@ env PIG_BASE_VERSION=%{pig_base_version} bash %{SOURCE1} %install %__rm -rf $RPM_BUILD_ROOT -cp $RPM_SOURCE_DIR/log4j.properties . cp $RPM_SOURCE_DIR/pig.1 . -cp $RPM_SOURCE_DIR/pig.properties . sh -x %{SOURCE2} \ --build-dir=build \ --doc-dir=%{doc_pig} \ --prefix=$RPM_BUILD_ROOT +rm -f $RPM_BUILD_ROOT/%{lib_pig}/lib/{hive,zookeeper}*.jar +ln -f -s %{zookeeper_home}/zookeeper.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/ +ln -f -s %{hive_home}/lib/hive-common.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/ +ln -f -s %{hive_home}/lib/hive-exec.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/ +ln -f -s %{hive_home}/lib/hive-serde.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/ +ln -f -s %{hive_home}/lib/hive-shims-common.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/ +ln -f -s %{hive_home}/lib/hive-shims-common-secure.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/ +rm -f $RPM_BUILD_ROOT/%{lib_pig}/lib/h2/hbase-*.jar +ln -f -s %{hbase_home}/hbase-client.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/h2/ +ln -f -s %{hbase_home}/hbase-common.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/h2/ +ln -f -s %{hbase_home}/hbase-hadoop2-compat.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/h2/ +ln -f -s %{hbase_home}/hbase-hadoop-compat.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/h2/ +ln -f -s %{hbase_home}/hbase-protocol.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/h2/ +ln -f -s %{hbase_home}/hbase-server.jar $RPM_BUILD_ROOT/%{lib_pig}/lib/h2/ + %pre # workaround for old style Pig conf dir if [ $1 -gt 1 -a -d %{lib_pig}/conf ]; then http://git-wip-us.apache.org/repos/asf/bigtop/blob/8c0d359f/bigtop.mk ---------------------------------------------------------------------- diff --git a/bigtop.mk b/bigtop.mk index 4d5304c..9991c4a 100644 --- a/bigtop.mk +++ b/bigtop.mk @@ -76,13 +76,13 @@ HBASE_ARCHIVE=$(APACHE_ARCHIVE)$(HBASE_DOWNLOAD_PATH) $(eval $(call PACKAGE,hbase,HBASE)) # Pig -PIG_BASE_VERSION=0.12.1 +PIG_BASE_VERSION=0.14.0 PIG_PKG_VERSION=$(PIG_BASE_VERSION) PIG_RELEASE_VERSION=1 PIG_NAME=pig PIG_RELNOTES_NAME=Apache Pig PIG_PKG_NAME=pig -PIG_TARBALL_DST=pig-$(PIG_BASE_VERSION).tar.gz +PIG_TARBALL_DST=pig-$(PIG_BASE_VERSION)-src.tar.gz PIG_TARBALL_SRC=$(PIG_TARBALL_DST) PIG_DOWNLOAD_PATH=/pig/pig-$(PIG_BASE_VERSION) PIG_SITE=$(APACHE_MIRROR)$(PIG_DOWNLOAD_PATH)
