This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 598de2f9e0 HDDS-7975. Rebalance acceptance tests (#4437)
598de2f9e0 is described below
commit 598de2f9e04b0142a6f2b6faed89fc583c0af9d6
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Mar 22 09:51:11 2023 +0100
HDDS-7975. Rebalance acceptance tests (#4437)
---
.github/workflows/ci.yml | 3 ++-
hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config | 2 ++
hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh | 15 ++++++++++++---
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 3 +--
.../dist/src/main/compose/ozonesecure-ha/docker-config | 5 +++++
hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh | 15 +++++++++++++--
hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh | 8 --------
7 files changed, 35 insertions(+), 16 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f540c1df77..2697a6e3f8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -248,7 +248,8 @@ jobs:
- secure
- unsecure
- compat
- - HA
+ - HA-secure
+ - HA-unsecure
- MR
- misc
fail-fast: false
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config
index 6fb4003b6b..bfe0c03f77 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config
@@ -19,6 +19,7 @@ CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=*
CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=*
CORE-SITE.XML_fs.defaultFS=ofs://omservice/
+CORE-SITE.XML_fs.trash.interval=1
OZONE-SITE.XML_ozone.om.service.ids=omservice
OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
@@ -47,6 +48,7 @@ OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_ozone.recon.address=recon:9891
OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
+OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true
OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
index 867c6923f5..2c2834cc3b 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-#suite:HA
+#suite:HA-unsecure
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
@@ -28,11 +28,20 @@ export OM_SERVICE_ID=omservice
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
-start_docker_env
+start_docker_env 5
execute_robot_test ${SCM} basic/ozone-shell-single.robot
execute_robot_test ${SCM} basic/links.robot
-execute_robot_test ${SCM} s3
+
+execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N
ozonefs-ofs-link ozonefs/ozonefs.robot
+
+exclude=""
+for bucket in generated; do
+ execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
+ # some tests are independent of the bucket type, only need to be run once
+ exclude="--exclude no-bucket-type"
+done
+
execute_robot_test ${SCM} freon
execute_robot_test ${SCM} -v USERNAME:httpfs httpfs
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
index b61c6277c9..6c84c30543 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
@@ -40,7 +40,7 @@ execute_robot_test scm gdpr
execute_robot_test scm security/ozone-secure-token.robot
exclude=""
-for bucket in erasure link generated; do
+for bucket in erasure; do
execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
# some tests are independent of the bucket type, only need to be run once
exclude="--exclude no-bucket-type"
@@ -58,7 +58,6 @@ execute_robot_test scm admincli
execute_robot_test scm -v USERNAME:httpfs httpfs
execute_debug_tests
-execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link
ozonefs/ozonefs.robot
execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N
ozonefs-o3fs-bucket ozonefs/ozonefs.robot
prefix=${RANDOM}
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
index ed82642df8..dce3cf4460 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
@@ -19,6 +19,7 @@ CORE-SITE.XML_hadoop.proxyuser.httpfs.hosts=*
CORE-SITE.XML_hadoop.proxyuser.httpfs.groups=*
CORE-SITE.XML_fs.defaultFS=ofs://id1
+CORE-SITE.XML_fs.trash.interval=1
OZONE-SITE.XML_ozone.om.service.ids=id1
OZONE-SITE.XML_ozone.om.internal.service.id=id1
@@ -56,6 +57,7 @@ OZONE-SITE.XML_ozone.replication=3
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_hdds.container.report.interval=60s
OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
+OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true
OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
@@ -142,6 +144,9 @@
HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/httpfs@EXAMPL
HTTPFS-SITE.XML_httpfs.hadoop.authentication.type=kerberos
HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal=httpfs/[email protected]
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=*
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=*
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=*
#Enable this variable to print out all hadoop rpc traffic to the stdout. See
http://byteman.jboss.org/ to define your own instrumentation.
#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
index dcd16a802b..c87e6dce23 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-#suite:HA
+#suite:HA-secure
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
@@ -24,18 +24,29 @@ export SECURITY_ENABLED=true
export OM_SERVICE_ID="id1"
export SCM=scm1.org
+: ${OZONE_BUCKET_KEY_NAME:=key1}
+
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
start_docker_env
+execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME}
+
execute_robot_test ${SCM} kinit.robot
execute_robot_test ${SCM} freon
+execute_robot_test ${SCM} -v SCHEME:o3fs -v BUCKET_TYPE:link -N
ozonefs-o3fs-link ozonefs/ozonefs.robot
+
execute_robot_test ${SCM} basic/links.robot
-execute_robot_test ${SCM} s3
+exclude=""
+for bucket in encrypted link; do
+ execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
+ # some tests are independent of the bucket type, only need to be run once
+ exclude="--exclude no-bucket-type"
+done
execute_robot_test ${SCM} admincli
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
index df96734ed9..a1e31830a1 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
@@ -38,14 +38,6 @@ execute_robot_test scm basic
execute_robot_test scm security
execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N
ozonefs-ofs-bucket ozonefs/ozonefs.robot
-execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link
ozonefs/ozonefs.robot
-
-exclude=""
-for bucket in encrypted link generated; do
- execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
- # some tests are independent of the bucket type, only need to be run once
- exclude="--exclude no-bucket-type"
-done
#expects 4 pipelines, should be run before
#admincli which creates STANDALONE pipeline
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
