This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 30a749e3acc HBASE-28331 Client integration test fails after upgrading
hadoop3 version to 3.3.x (#5657)
30a749e3acc is described below
commit 30a749e3acc4e91443ad54160c18a176a5ed0c70
Author: Duo Zhang <[email protected]>
AuthorDate: Mon Jan 29 23:52:48 2024 +0800
HBASE-28331 Client integration test fails after upgrading hadoop3 version
to 3.3.x (#5657)
Signed-off-by: Yi Mei <[email protected]>
(cherry picked from commit f3db223587685388f655dfe83d2f8cc745ea9443)
---
dev-support/Jenkinsfile | 5 +++++
dev-support/patch-hadoop3.sh | 24 ++++++++++++++++++++++++
2 files changed, 29 insertions(+)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 9c03d97a3c4..a767775b36b 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -661,6 +661,9 @@ pipeline {
rm -rf "hadoop-3" && mkdir "hadoop-3"
rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
rm -rf ".m2-for-src" && mkdir ".m2-for-src"
+ # remove old hadoop tarballs in workspace
+ rm -rf hadoop-2*.tar.gz
+ rm -rf hadoop-3*.tar.gz
echo "(x) {color:red}-1 source release artifact{color}\n--
Something went wrong with this stage, [check relevant console
output|${BUILD_URL}/console]." >output-srctarball/commentfile
echo "(x) {color:red}-1 client integration test{color}\n--
Something went wrong with this stage, [check relevant console
output|${BUILD_URL}/console]." >output-integration/commentfile
'''
@@ -726,6 +729,8 @@ pipeline {
echo "Attempting to use run an instance on top of Hadoop 3."
artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1)
tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3"
+ # we need to patch some files otherwise minicluster will fail to
start, see MAPREDUCE-7471
+ ${BASEDIR}/dev-support/patch-hadoop3.sh hadoop-3
if !
"${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
--single-process \
--working-dir output-integration/hadoop-3 \
diff --git a/dev-support/patch-hadoop3.sh b/dev-support/patch-hadoop3.sh
new file mode 100755
index 00000000000..fed5a535bcd
--- /dev/null
+++ b/dev-support/patch-hadoop3.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+hadoop_dir=$1
+
+sed -i
"s/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"share\/hadoop\/tools\"}/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"\$HADOOP_TOOLS_HOME\/share\/hadoop\/tools\"}/g"
"$hadoop_dir/libexec/hadoop-functions.sh"
+sed -i "/hadoop_add_classpath \"\${junitjar}\"/a mockitojar=\$(echo
\"\${HADOOP_TOOLS_LIB_JARS_DIR}\"\/mockito-core-[0-9]*.jar)\nhadoop_add_classpath
\"\${mockitojar}\"" "$hadoop_dir/bin/mapred"
+curl
https://repo1.maven.org/maven2/org/mockito/mockito-core/2.28.2/mockito-core-2.28.2.jar
-o "$hadoop_dir/share/hadoop/tools/lib/mockito-core-2.28.2.jar"
