This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new f558aa4f1b HDDS-8795. Run all unsecure Hadoop (MR) acceptance tests
with the same single Ozone cluster (#4862)
f558aa4f1b is described below
commit f558aa4f1bd35d50d0dbd94a6613bbc2b9596a46
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Jun 20 19:27:46 2023 +0200
HDDS-8795. Run all unsecure Hadoop (MR) acceptance tests with the same
single Ozone cluster (#4862)
---
.../dist/src/main/compose/common/hadoop-3.2.sh | 25 ------------
.../dist/src/main/compose/common/hadoop-3.3.sh | 25 ------------
.../compose/{ozone-ha => common}/hadoop-test.sh | 25 ++++++++----
.../dist/src/main/compose/common/hadoop.yaml | 4 ++
.../src/main/compose/ozone-ha/test-hadoop-2.7.sh | 1 -
.../src/main/compose/ozone-ha/test-hadoop-3.1.sh | 1 -
.../src/main/compose/ozone-ha/test-hadoop-3.2.sh | 1 -
.../src/main/compose/ozone-ha/test-hadoop-3.3.sh | 1 -
.../hadoop-3.1.sh => ozone-ha/test-hadoop.sh} | 6 ++-
.../dist/src/main/compose/ozone/hadoop-test.sh | 45 ----------------------
.../dist/src/main/compose/ozone/test-ec.sh | 0
.../dist/src/main/compose/ozone/test-hadoop-2.7.sh | 1 -
.../dist/src/main/compose/ozone/test-hadoop-3.1.sh | 1 -
.../dist/src/main/compose/ozone/test-hadoop-3.2.sh | 1 -
.../dist/src/main/compose/ozone/test-hadoop-3.3.sh | 1 -
.../{common/hadoop-2.7.sh => ozone/test-hadoop.sh} | 5 ++-
.../src/main/compose/ozone/test-legacy-bucket.sh | 0
.../ozonesecure/test-certificate-rotation.sh | 0
hadoop-ozone/dist/src/main/compose/restart/test.sh | 0
hadoop-ozone/dist/src/main/compose/testlib.sh | 12 +++++-
20 files changed, 39 insertions(+), 116 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-3.2.sh
b/hadoop-ozone/dist/src/main/compose/common/hadoop-3.2.sh
deleted file mode 100755
index 3eee1436d7..0000000000
--- a/hadoop-ozone/dist/src/main/compose/common/hadoop-3.2.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#suite:MR
-
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export COMPOSE_DIR
-
-export HADOOP_VERSION=3.2.2
-
-source ${COMPOSE_DIR}/hadoop-test.sh
diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-3.3.sh
b/hadoop-ozone/dist/src/main/compose/common/hadoop-3.3.sh
deleted file mode 100755
index e49f95f59d..0000000000
--- a/hadoop-ozone/dist/src/main/compose/common/hadoop-3.3.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#suite:MR
-
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export COMPOSE_DIR
-
-export HADOOP_VERSION=3.3.1
-
-source ${COMPOSE_DIR}/hadoop-test.sh
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/hadoop-test.sh
b/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
similarity index 64%
rename from hadoop-ozone/dist/src/main/compose/ozone-ha/hadoop-test.sh
rename to hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
index e9a4965cc8..3982888a31 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/hadoop-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
@@ -16,12 +16,9 @@
# limitations under the License.
export COMPOSE_FILE=docker-compose.yaml:../common/hadoop.yaml
-export HADOOP_MAJOR_VERSION=${HADOOP_VERSION%%.*}
-
-export SECURITY_ENABLED=false
+export HADOOP_MAJOR_VERSION=3
+export HADOOP_VERSION=unused # will be set for each test version below
export OZONE_REPLICATION_FACTOR=3
-export SCM=scm1
-export OM_SERVICE_ID=omservice
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
@@ -36,9 +33,21 @@ export OZONE_DIR=/opt/ozone
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
-for scheme in o3fs ofs; do
- execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
- execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-mapreduce-${scheme}" mapreduce.robot
+for HADOOP_VERSION in 2.7.3 3.1.2 3.2.2 3.3.1; do
+ export HADOOP_VERSION
+ export HADOOP_MAJOR_VERSION=${HADOOP_VERSION%%.*}
+
+ docker-compose --ansi never --profile hadoop up -d nm rm
+
+ execute_command_in_container rm hadoop version
+
+ for scheme in o3fs ofs; do
+ execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
+ execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-mapreduce-${scheme}" mapreduce.robot
+ done
+
+ save_container_logs nm rm
+ stop_containers nm rm
done
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
b/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
index 6f2eab1769..6d3cc3d4fb 100644
--- a/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
@@ -29,6 +29,8 @@ services:
- ../common/hadoop.conf
- ../common/hadoop${HADOOP_MAJOR_VERSION}.conf
command: ["yarn", "resourcemanager"]
+ profiles:
+ - hadoop
nm:
image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
hostname: nm
@@ -42,3 +44,5 @@ services:
environment:
WAITFOR: rm:8088
command: ["yarn","nodemanager"]
+ profiles:
+ - hadoop
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-2.7.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-2.7.sh
deleted file mode 120000
index 59f3b43dd8..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-2.7.sh
+++ /dev/null
@@ -1 +0,0 @@
-../common/hadoop-2.7.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.1.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.1.sh
deleted file mode 120000
index 73ddffe9f3..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.1.sh
+++ /dev/null
@@ -1 +0,0 @@
-../common/hadoop-3.1.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.2.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.2.sh
deleted file mode 120000
index d4ac16ad07..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.2.sh
+++ /dev/null
@@ -1 +0,0 @@
-../common/hadoop-3.2.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.3.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.3.sh
deleted file mode 120000
index da90e3a998..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.3.sh
+++ /dev/null
@@ -1 +0,0 @@
-../common/hadoop-3.3.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-3.1.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop.sh
similarity index 88%
rename from hadoop-ozone/dist/src/main/compose/common/hadoop-3.1.sh
rename to hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop.sh
index 318523e288..40a12b85e1 100755
--- a/hadoop-ozone/dist/src/main/compose/common/hadoop-3.1.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop.sh
@@ -20,6 +20,8 @@
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
-export HADOOP_VERSION=3.1.2
+export SECURITY_ENABLED=false
+export SCM=scm1
+export OM_SERVICE_ID=omservice
-source ${COMPOSE_DIR}/hadoop-test.sh
+source "$COMPOSE_DIR/../common/hadoop-test.sh"
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/hadoop-test.sh
b/hadoop-ozone/dist/src/main/compose/ozone/hadoop-test.sh
deleted file mode 100755
index e2b34d137f..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone/hadoop-test.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export COMPOSE_FILE=docker-compose.yaml:../common/hadoop.yaml
-export HADOOP_MAJOR_VERSION=${HADOOP_VERSION%%.*}
-
-export SECURITY_ENABLED=false
-export OZONE_REPLICATION_FACTOR=3
-export SCM=scm
-
-# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../testlib.sh"
-
-start_docker_env
-
-execute_robot_test ${SCM} createmrenv.robot
-
-# reinitialize the directories to use
-export OZONE_DIR=/opt/ozone
-
-# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../testlib.sh"
-
-for scheme in o3fs ofs; do
- execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
- execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-mapreduce-${scheme}" mapreduce.robot
-done
-
-stop_docker_env
-
-generate_report
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-ec.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-ec.sh
old mode 100644
new mode 100755
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-2.7.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-2.7.sh
deleted file mode 120000
index 59f3b43dd8..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-2.7.sh
+++ /dev/null
@@ -1 +0,0 @@
-../common/hadoop-2.7.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.1.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.1.sh
deleted file mode 120000
index 73ddffe9f3..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.1.sh
+++ /dev/null
@@ -1 +0,0 @@
-../common/hadoop-3.1.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.2.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.2.sh
deleted file mode 120000
index d4ac16ad07..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.2.sh
+++ /dev/null
@@ -1 +0,0 @@
-../common/hadoop-3.2.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.3.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.3.sh
deleted file mode 120000
index da90e3a998..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.3.sh
+++ /dev/null
@@ -1 +0,0 @@
-../common/hadoop-3.3.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-2.7.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop.sh
similarity index 90%
rename from hadoop-ozone/dist/src/main/compose/common/hadoop-2.7.sh
rename to hadoop-ozone/dist/src/main/compose/ozone/test-hadoop.sh
index 41f49758b0..e687fac5ad 100755
--- a/hadoop-ozone/dist/src/main/compose/common/hadoop-2.7.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop.sh
@@ -20,6 +20,7 @@
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
-export HADOOP_VERSION=2.7.3
+export SECURITY_ENABLED=false
+export SCM=scm
-source ${COMPOSE_DIR}/hadoop-test.sh
+source "$COMPOSE_DIR/../common/hadoop-test.sh"
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-legacy-bucket.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-legacy-bucket.sh
old mode 100644
new mode 100755
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-certificate-rotation.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure/test-certificate-rotation.sh
old mode 100644
new mode 100755
diff --git a/hadoop-ozone/dist/src/main/compose/restart/test.sh
b/hadoop-ozone/dist/src/main/compose/restart/test.sh
old mode 100644
new mode 100755
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh
b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 4c031e00cd..82e63e9bca 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -286,6 +286,16 @@ start_containers() {
set +e
}
+create_containers() {
+ set -e
+ docker-compose --ansi never up -d $@
+ set +e
+}
+
+save_container_logs() {
+ docker-compose --ansi never logs $@ >> "$RESULT_DIR/docker-$OUTPUT_NAME.log"
+}
+
## @description wait until the port is available on the given host
## @param The host to check for the port
@@ -345,7 +355,7 @@ wait_for_execute_command(){
## @description Stops a docker-compose based test environment (with saving
the logs)
stop_docker_env(){
- docker-compose --ansi never logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log"
+ save_container_logs
if [ "${KEEP_RUNNING:-false}" = false ]; then
docker-compose --ansi never down
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
