Repository: bigtop Updated Branches: refs/heads/master 2e1795361 -> fb26cdadd
BIGTOP-2893 Bump Hadoop to 2.8.1 Patches: ======= HADOOP-12366 was ported to 2.8.1 HADOOP-11628 was removed because it was merged to 2.8 Build time issues fixed: ======================== Fuse_dfs target dir has been changed in 2.8 ------------------------------------------- cp: cannot stat 'hadoop-hdfs-project/hadoop-hdfs/target/native/main/native/fuse-dfs/fuse_dfs': No such file or directory add package goal to mvn ---------------------- Without that goal tar is incomplete: for dir in '${HADOOP_DIR}/{lib,}' '${HDFS_DIR}/{lib,}' '${YARN_DIR}/{lib,}' '${MAPREDUCE_DIR}/{lib,}' '[' -e /code/build/hadoop/rpm/BUILDROOT/hadoop-2.8.1-243.el7.centos.x86_64/usr/lib/hadoop-mapreduce//hadoop-common-2.8.1.jar ']' exit 1 error: Bad exit status from /var/tmp/rpm-tmp.5Etrah (%install) Bad exit status from /var/tmp/rpm-tmp.5Etrah (%install) Signed-off-by: Youngwoo Kim <yw...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/fb26cdad Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/fb26cdad Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/fb26cdad Branch: refs/heads/master Commit: fb26cdadd11d6b0164ac25bca7a5fb8f4d1f97cf Parents: 2e17953 Author: Anton Chevychalov <c...@arenadata.io> Authored: Wed Sep 27 14:48:38 2017 +0300 Committer: Youngwoo Kim <yw...@apache.org> Committed: Tue Dec 19 15:23:34 2017 +0900 ---------------------------------------------------------------------- .../src/common/hadoop/do-component-build | 4 +-- ...0-HADOOP-12366.-expose-calculated-paths.diff | 38 ++++++++++---------- .../src/common/hadoop/patch1-HADOOP-11628.diff | 22 ------------ bigtop.bom | 2 +- 4 files changed, 22 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb26cdad/bigtop-packages/src/common/hadoop/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hadoop/do-component-build b/bigtop-packages/src/common/hadoop/do-component-build index 32429c2..dcd802f 100644 --- a/bigtop-packages/src/common/hadoop/do-component-build +++ b/bigtop-packages/src/common/hadoop/do-component-build @@ -56,7 +56,7 @@ mkdir build/src # Build artifacts MAVEN_OPTS="-Dzookeeper.version=$ZOOKEEPER_VERSION" MAVEN_OPTS="$MAVEN_OPTS -DskipTests -DskipTest -DskipITs" -mvn $ANT_OPTS $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Dtar ${MAVEN_OPTS} -Dtomcat.version=${BIGTOP_TOMCAT_VERSION} -Dtomcat.download.url="http://archive.apache.org/dist/tomcat/tomcat-6/v${BIGTOP_TOMCAT_VERSION}/bin/apache-tomcat-${BIGTOP_TOMCAT_VERSION}.tar.gz" install "$@" +mvn $ANT_OPTS $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Dtar ${MAVEN_OPTS} -Dtomcat.version=${BIGTOP_TOMCAT_VERSION} -Dtomcat.download.url="http://archive.apache.org/dist/tomcat/tomcat-6/v${BIGTOP_TOMCAT_VERSION}/bin/apache-tomcat-${BIGTOP_TOMCAT_VERSION}.tar.gz" install package "$@" mvn site site:stage ${MAVEN_OPTS} $@ (cd build ; tar --strip-components=1 -xzvf ../hadoop-dist/target/hadoop-${HADOOP_VERSION}.tar.gz) @@ -70,4 +70,4 @@ cp -r target/staging/hadoop-project build/share/doc (cd hadoop-client/target/hadoop-client-*/share/hadoop/client/lib ; ls) > build/hadoop-client.list # Copy fuse output to the build directory -cp hadoop-hdfs-project/hadoop-hdfs/target/native/main/native/fuse-dfs/fuse_dfs build/bin/ +cp hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs build/bin http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb26cdad/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff b/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff index b0c831c..026c01f 100644 --- a/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff +++ b/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff @@ -1,9 +1,9 @@ -From 3131744ed6e1e874d7cd0666208adc424fec3224 Mon Sep 17 00:00:00 2001 -From: Konstantin Boudnik <c...@boudnik.org> -Date: Mon, 15 Feb 2016 10:42:16 +0300 -Subject: [PATCH] HADOOP-12366. expose calculated paths +From f10ec9f9bec56bcfa5e2015ea87a8a5054d096c5 Mon Sep 17 00:00:00 2001 +From: Anton Chevychalov <p...@mnu.pp.ru> +Date: Tue, 5 Sep 2017 15:14:19 +0300 +Subject: [PATCH] [PATCH] HADOOP-12366. expose calculated paths -Adapted the patch to apply cleanly on branch-2.7.1 +Adapted the patch to apply cleanly on branch-2.8.1 --- hadoop-common-project/hadoop-common/src/main/bin/hadoop | 10 ++++++++++ hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 9 +++++++++ @@ -12,7 +12,7 @@ Adapted the patch to apply cleanly on branch-2.7.1 4 files changed, 37 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop -index a5e8885..715ad9b 100755 +index 1575996..44be2e3 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop @@ -38,6 +38,7 @@ function print_usage(){ @@ -22,7 +22,7 @@ index a5e8885..715ad9b 100755 + echo " envvars display computed Hadoop environment variables" echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive" echo " classpath prints the class path needed to get the" - echo " credential interact with credential providers" + echo " Hadoop jar and the required libraries" @@ -116,6 +117,15 @@ case $COMMAND in CLASSPATH=${CLASSPATH}:${TOOL_PATH} elif [ "$COMMAND" = "daemonlog" ] ; then @@ -40,18 +40,18 @@ index a5e8885..715ad9b 100755 CLASS=org.apache.hadoop.tools.HadoopArchives CLASSPATH=${CLASSPATH}:${TOOL_PATH} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs -index 155df14..b744761 100755 +index fbfbaf2..714280e 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs -@@ -46,6 +46,7 @@ function print_usage(){ - echo " zkfc run the ZK Failover Controller daemon" +@@ -47,6 +47,7 @@ function print_usage(){ echo " datanode run a DFS datanode" + echo " debug run a Debug Admin to execute HDFS debug commands" echo " dfsadmin run a DFS admin client" + echo " envvars display computed Hadoop environment variables" echo " haadmin run a DFS HA admin client" echo " fsck run a DFS filesystem checking utility" echo " balancer run a cluster balancing utility" -@@ -160,6 +161,14 @@ elif [ "$COMMAND" = "haadmin" ] ; then +@@ -161,6 +162,14 @@ elif [ "$COMMAND" = "haadmin" ] ; then CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin CLASSPATH=${CLASSPATH}:${TOOL_PATH} HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" @@ -67,7 +67,7 @@ index 155df14..b744761 100755 CLASS=org.apache.hadoop.hdfs.tools.DFSck HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred -index fe16e07..6a21fed 100755 +index 6f3bc04..fd1300d 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -38,6 +38,7 @@ function print_usage(){ @@ -76,9 +76,9 @@ index fe16e07..6a21fed 100755 echo " distcp <srcurl> <desturl> copy file or directories recursively" + echo " envvars display computed Hadoop environment variables" echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive" + echo " archive-logs combine aggregated logs into hadoop archives" echo " hsadmin job history server admin interface" - echo "" -@@ -74,6 +75,14 @@ elif [ "$COMMAND" = "sampler" ] ; then +@@ -75,6 +76,14 @@ elif [ "$COMMAND" = "sampler" ] ; then HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" elif [ "$COMMAND" = "classpath" ] ; then echo -n @@ -94,18 +94,18 @@ index fe16e07..6a21fed 100755 CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer HADOOP_OPTS="$HADOOP_OPTS -Dmapred.jobsummary.logger=${HADOOP_JHS_LOGGER:-INFO,console} $HADOOP_JOB_HISTORYSERVER_OPTS" diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn -index 71347a2..ce3e24a 100644 +index 552cef4..5495387 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn -@@ -87,6 +87,7 @@ function print_usage(){ +@@ -89,6 +89,7 @@ function print_usage(){ echo " cluster prints cluster information" echo " daemonlog get/set the log level for each" echo " daemon" + echo " envvars display computed Hadoop environment variables" + echo " top run cluster usage tool" echo "" echo "Most commands print help when invoked w/o parameters." - } -@@ -214,6 +215,14 @@ if [ "$COMMAND" = "classpath" ] ; then +@@ -217,6 +218,14 @@ if [ "$COMMAND" = "classpath" ] ; then echo $CLASSPATH exit 0 fi @@ -121,5 +121,5 @@ index 71347a2..ce3e24a 100644 CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI' YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" -- -2.0.4 +2.7.4 http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb26cdad/bigtop-packages/src/common/hadoop/patch1-HADOOP-11628.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hadoop/patch1-HADOOP-11628.diff b/bigtop-packages/src/common/hadoop/patch1-HADOOP-11628.diff deleted file mode 100644 index 1c2de37..0000000 --- a/bigtop-packages/src/common/hadoop/patch1-HADOOP-11628.diff +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java -index 9852460..8cd39c7 100644 ---- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java -+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java -@@ -37,6 +37,7 @@ - - import java.io.File; - import java.io.IOException; -+import java.net.InetAddress; - import java.security.PrivilegedActionException; - import java.security.PrivilegedExceptionAction; - import java.util.ArrayList; -@@ -343,7 +344,8 @@ public AuthenticationToken authenticate(HttpServletRequest request, final HttpSe - authorization = authorization.substring(KerberosAuthenticator.NEGOTIATE.length()).trim(); - final Base64 base64 = new Base64(0); - final byte[] clientToken = base64.decode(authorization); -- final String serverName = request.getServerName(); -+ final String serverName = InetAddress.getByName(request.getServerName()) -+ .getCanonicalHostName(); - try { - token = Subject.doAs(serverSubject, new PrivilegedExceptionAction<AuthenticationToken>() { - http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb26cdad/bigtop.bom ---------------------------------------------------------------------- diff --git a/bigtop.bom b/bigtop.bom index f346d88..1a9fabe 100644 --- a/bigtop.bom +++ b/bigtop.bom @@ -144,7 +144,7 @@ bigtop { 'hadoop' { name = 'hadoop' relNotes = 'Apache Hadoop' - version { base = '2.7.4'; pkg = base; release = 1 } + version { base = '2.8.1'; pkg = base; release = 1 } tarball { destination = "${name}-${version.base}.tar.gz" source = "${name}-${version.base}-src.tar.gz" } url { download_path = "/$name/common/$name-${version.base}"