This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 281761b956 HDDS-8640. Support multiple acceptance test scripts with
custom config (#4750)
281761b956 is described below
commit 281761b956d2aaf1f903b7936627744105f4beb1
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Fri Jun 2 11:43:27 2023 +0200
HDDS-8640. Support multiple acceptance test scripts with custom config
(#4750)
---
.../src/main/compose/ozone-legacy-bucket/README.md | 21 ---
.../ozone-legacy-bucket/docker-compose.yaml | 78 ---------
.../main/compose/ozone-legacy-bucket/docker-config | 52 ------
.../{test_disabled.sh => disabled-test.sh} | 0
.../.env => ozone/legacy-bucket.yaml} | 27 ++-
.../test.sh => ozone/test-legacy-bucket.sh} | 7 +-
.../compose/ozonesecure-vault/docker-compose.yaml | 138 ---------------
.../main/compose/ozonesecure-vault/docker-config | 193 ---------------------
.../test.sh => ozonesecure/test-vault.sh} | 1 +
.../.env => ozonesecure/vault.conf} | 11 +-
.../krb5.conf => ozonesecure/vault.yaml} | 45 +++--
hadoop-ozone/dist/src/main/compose/testlib.sh | 45 +++--
12 files changed, 80 insertions(+), 538 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/README.md
b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/README.md
deleted file mode 100644
index d31d8f20fb..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# For Legacy Bucket Operations
-
-For Legacy buckets, set `ozone.om.enable.filesystem.paths` to `true` for them
to behave like FSO buckets,
-otherwise Legacy buckets act like OBS buckets.
-
-This is the same as `compose/ozone` but for testing operations that need
`ozone.om.enable.filesystem.paths`
-flag enabled.
\ No newline at end of file
diff --git
a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-compose.yaml
deleted file mode 100644
index 72303abaf6..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-compose.yaml
+++ /dev/null
@@ -1,78 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3.4"
-
-# reusable fragments (see
https://docs.docker.com/compose/compose-file/#extension-fields)
-x-common-config:
- &common-config
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- env_file:
- - docker-config
-
-x-replication:
- &replication
- OZONE-SITE.XML_ozone.replication: ${OZONE_REPLICATION_FACTOR:-1}
-
-services:
- datanode:
- <<: *common-config
- ports:
- - 9864
- - 9882
- environment:
- <<: *replication
- OZONE_OPTS:
- command: ["ozone","datanode"]
- om:
- <<: *common-config
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
- OZONE_OPTS:
- <<: *replication
- ports:
- - 9874:9874
- - 9862:9862
- command: ["ozone","om"]
- scm:
- <<: *common-config
- ports:
- - 9876:9876
- - 9860:9860
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
${OZONE_SAFEMODE_MIN_DATANODES:-1}
- OZONE_OPTS:
- <<: *replication
- command: ["ozone","scm"]
- s3g:
- <<: *common-config
- environment:
- OZONE_OPTS:
- <<: *replication
- ports:
- - 9878:9878
- command: ["ozone","s3g"]
- recon:
- <<: *common-config
- ports:
- - 9888:9888
- environment:
- OZONE_OPTS:
- <<: *replication
- command: ["ozone","recon"]
diff --git
a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-config
deleted file mode 100644
index 90d62dcd00..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-config
+++ /dev/null
@@ -1,52 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-CORE-SITE.XML_fs.defaultFS=ofs://om
-CORE-SITE.XML_fs.trash.interval=1
-
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.enable.filesystem.paths=true
-OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.http-address=scm:9876
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.block.size=1MB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_ozone.recon.address=recon:9891
-OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
-OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
-OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
-OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_hdds.container.report.interval=60s
-OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
-OZONE-SITE.XML_ozone.scm.stale.node.interval=30s
-OZONE-SITE.XML_ozone.scm.dead.node.interval=45s
-OZONE-SITE.XML_hdds.heartbeat.interval=5s
-
-OZONE_CONF_DIR=/etc/hadoop
-OZONE_LOG_DIR=/var/log/hadoop
-
-no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test_disabled.sh
b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/disabled-test.sh
similarity index 100%
rename from hadoop-ozone/dist/src/main/compose/ozone-om-ha/test_disabled.sh
rename to hadoop-ozone/dist/src/main/compose/ozone-om-ha/disabled-test.sh
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/.env
b/hadoop-ozone/dist/src/main/compose/ozone/legacy-bucket.yaml
similarity index 66%
rename from hadoop-ozone/dist/src/main/compose/ozonesecure-vault/.env
rename to hadoop-ozone/dist/src/main/compose/ozone/legacy-bucket.yaml
index 1689084ef2..c893c16799 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozone/legacy-bucket.yaml
@@ -14,13 +14,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-HDDS_VERSION=${hdds.version}
-HADOOP_VERSION=3
-OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
-OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
-OZONE_OPTS=
+version: "3.4"
-VAULT_PORT=8200
-VAULT_ADDR=http://127.0.0.1:8200
-NETWORK_NAME=project_web
+x-legacy-bucket-config:
+ &legacy-bucket-config
+ environment:
+ - OZONE-SITE.XML_ozone.om.enable.filesystem.paths=true
+ - OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY
+
+services:
+ datanode:
+ <<: *legacy-bucket-config
+ om:
+ <<: *legacy-bucket-config
+ scm:
+ <<: *legacy-bucket-config
+ s3g:
+ <<: *legacy-bucket-config
+ recon:
+ <<: *legacy-bucket-config
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-legacy-bucket.sh
similarity index 85%
rename from hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/test.sh
rename to hadoop-ozone/dist/src/main/compose/ozone/test-legacy-bucket.sh
index 4f776686a5..d31d18a9b0 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test-legacy-bucket.sh
@@ -23,10 +23,15 @@ export COMPOSE_DIR
export SECURITY_ENABLED=false
export OZONE_REPLICATION_FACTOR=3
+# This is for testing operations on LEGACY buckets that need
+# `ozone.om.enable.filesystem.paths` flag enabled.
+
+export COMPOSE_FILE=docker-compose.yaml:legacy-bucket.yaml
+
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
-start_docker_env 5
+start_docker_env 3
execute_robot_test scm -v BUCKET_LAYOUT:LEGACY recon/recon-nssummary.robot
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/docker-compose.yaml
deleted file mode 100644
index 11be8dd4d6..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/docker-compose.yaml
+++ /dev/null
@@ -1,138 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- kdc:
- image: ${OZONE_TESTKRB5_IMAGE}
- hostname: kdc
- volumes:
- - ../..:/opt/hadoop
- - ../_keytabs:/etc/security/keytabs
- command: ["krb5kdc","-n"]
- kms:
- image: apache/hadoop:${HADOOP_VERSION}
- ports:
- - 9600:9600
- env_file:
- - ./docker-config
- environment:
- HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
- volumes:
- - ../../libexec/transformation.py:/opt/transformation.py
- command: ["hadoop", "kms"]
- datanode:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - docker-config
- environment:
- OZONE_OPTS:
- om:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: om
- volumes:
- - ../..:/opt/hadoop
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- ports:
- - 9874:9874
- - 9862:9862
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
- OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false
- OZONE_CLASSPATH:
/opt/hadoop/share/ozone/lib/[email protected]@.jar:/opt/hadoop/share/ozone/lib/vault-java-driver-*.jar
- env_file:
- - docker-config
- command: ["/opt/hadoop/bin/ozone","om"]
- httpfs:
- image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
- hostname: httpfs
- volumes:
- - ../..:/opt/hadoop
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- ports:
- - 14000:14000
- env_file:
- - ./docker-config
- command: [ "/opt/hadoop/bin/ozone","httpfs" ]
- environment:
- OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
${OZONE_SAFEMODE_MIN_DATANODES:-1}
- OZONE_OPTS:
- s3g:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: s3g
- volumes:
- - ../..:/opt/hadoop
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- ports:
- - 9878:9878
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","s3g"]
- environment:
- OZONE_OPTS:
- recon:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: recon
- volumes:
- - ../..:/opt/hadoop
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- ports:
- - 9888:9888
- env_file:
- - ./docker-config
- environment:
- OZONE_OPTS:
- command: ["/opt/hadoop/bin/ozone","recon"]
- scm:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: scm
- volumes:
- - ../..:/opt/hadoop
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- ports:
- - 9876:9876
- - 9860:9860
- env_file:
- - docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
"${OZONE_SAFEMODE_MIN_DATANODES:-1}"
- OZONE_OPTS:
- command: ["/opt/hadoop/bin/ozone","scm"]
- vault:
- image: hashicorp/vault:1.13.2
- hostname: vault
- cap_add:
- - IPC_LOCK
- ports:
- - ${VAULT_PORT}:8200
- command:
- - server
- - -dev
- - -dev-root-token-id=00000000-0000-0000-0000-000000000000
- - -namespace=namespace
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/docker-config
b/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/docker-config
deleted file mode 100644
index ae61a714bd..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/docker-config
+++ /dev/null
@@ -1,193 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-CORE-SITE.XML_fs.defaultFS=ofs://om
-CORE-SITE.XML_fs.trash.interval=1
-# For HttpFS service it is required to enable proxying users.
-CORE-SITE.XML_hadoop.proxyuser.httpfs.hosts=*
-CORE-SITE.XML_hadoop.proxyuser.httpfs.groups=*
-
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-# TODO: HDDS-7495
-# S3 security tests need object store layout from s3 gateway. Other tests on
-# this cluster use `ozone fs`. The config to set an S3 gateway specific
bucket
-# layout will be added in HDDS-7495. Until then, the server side default must
-# be changed for all tests on this cluster to work.
-OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY
-OZONE-SITE.XML_ozone.scm.http-address=scm:9876
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.block.token.enabled=true
-OZONE-SITE.XML_hdds.container.token.enabled=true
-OZONE-SITE.XML_hdds.grpc.tls.enabled=true
-OZONE-SITE.XML_ozone.replication=3
-OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-
-OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
-OZONE-SITE.XML_ozone.recon.om.snapshot.task.initial.delay=20s
-OZONE-SITE.XML_ozone.recon.address=recon:9891
-
-OZONE-SITE.XML_ozone.security.enabled=true
-OZONE-SITE.XML_ozone.acl.enabled=true
-OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
-OZONE-SITE.XML_ozone.administrators="testuser,recon"
-OZONE-SITE.XML_ozone.recon.administrators="testuser2"
-
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
-HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-CORE-SITE.XML_dfs.data.transfer.protection=authentication
-CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT"
-CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
-
-
-OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/[email protected]
-OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.kerberos.principal=om/[email protected]
-OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab
-OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/[email protected]
-
-OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
-OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/[email protected]
-
-OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
-OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/[email protected]
-
-OZONE-SITE.XML_hdds.scm.replication.thread.interval=5s
-OZONE-SITE.XML_hdds.scm.replication.enable.legacy=false
-OZONE-SITE.XML_ozone.scm.stale.node.interval=30s
-OZONE-SITE.XML_ozone.scm.dead.node.interval=45s
-OZONE-SITE.XML_hdds.container.report.interval=60s
-OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
-
-OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true
-
-HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/[email protected]
-HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/[email protected]
-HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-
-
-OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
-OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer
-
-OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
-OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
-OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
-OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
-OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos
-OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos
-
-OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/[email protected]
-OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/[email protected]
-OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/[email protected]
-OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/dn.keytab
-OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab
-OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/[email protected]
-OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
-OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/[email protected]
-OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=HTTP/[email protected]
-OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
-
-CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false
-CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret
-CORE-SITE.XML_hadoop.http.authentication.type=kerberos
-CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/[email protected]
-CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-
-
-CORE-SITE.XML_hadoop.security.authorization=true
-HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
-
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-
-HTTPFS-SITE.XML_hadoop.http.authentication.type=kerberos
-HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
-HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/[email protected]
-HTTPFS-SITE.XML_httpfs.hadoop.authentication.type=kerberos
-HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
-HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal=httpfs/[email protected]
-KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=*
-KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=*
-KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=*
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See
http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-OZONE_DATANODE_SECURE_USER=root
-JAVA_HOME=/usr/lib/jvm/jre
-JSVC_HOME=/usr/bin
-
-OZONE_CONF_DIR=/etc/hadoop
-OZONE_LOG_DIR=/var/log/hadoop
-
-no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1
-
-# Multi-Tenancy configs
-OZONE-SITE.XML_ozone.om.multitenancy.enabled=true
-OZONE-SITE.XML_ozone.om.ranger.service=cm_ozone
-
-# Note: Ranger address and credentials here doesn't matter when OM uses
-# InMemoryMultiTenantAccessController (used when dev flag is set).
-# But the values can't be empty otherwise OM config check would report failure.
-OZONE-SITE.XML_ozone.om.ranger.https-address=https://ranger:6182
-OZONE-SITE.XML_ozone.om.ranger.https.admin.api.user=admin
-OZONE-SITE.XML_ozone.om.ranger.https.admin.api.passwd=Passwd1
-
-# ozone.om.kerberos.principal and ozone.om.kerberos.keytab.file
-# (can be used for the RangerClient) are already defined above.
-
-OZONE-SITE.XML_ozone.om.multitenancy.ranger.sync.interval=30s
-OZONE-SITE.XML_ozone.om.multitenancy.ranger.sync.timeout=10s
-
-# Use InMemoryMultiTenantAccessController as we don't have Ranger Admin Server
here.
-# This is fine with one OM. But for OM HA, each OM would have its own in-memory
-# "Ranger" state as a result.
-# New OM leader's in-memory "Ranger" state won't be consistent with OM DB until
-# a BG sync run has successfully finished because only previously leader OM
-# pushes updates to Ranger.
-#
-# Potential TODO: We could trigger BG sync automatically during OM leadership
-# change or let all OMs write to AccessController if this dev flag is set.
-#
-OZONE-SITE.XML_ozone.om.tenant.dev.skip.ranger=true
-
-OZONE-SITE.XML_ozone.secret.s3.store.provider=org.apache.hadoop.ozone.s3.remote.vault.VaultS3SecretStorageProvider
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.address=http://vault:8200
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.namespace="namespace"
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.enginever=2
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.secretpath=secret
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth=TOKEN
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth.token=00000000-0000-0000-0000-000000000000
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/test.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh
similarity index 95%
rename from hadoop-ozone/dist/src/main/compose/ozonesecure-vault/test.sh
rename to hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh
index 028325c3dc..010da774ec 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh
@@ -24,6 +24,7 @@ export COMPOSE_DIR
source "$COMPOSE_DIR/../testlib.sh"
export SECURITY_ENABLED=true
+export COMPOSE_FILE=docker-compose.yaml:vault.yaml
start_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure/vault.conf
similarity index 58%
rename from hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/.env
rename to hadoop-ozone/dist/src/main/compose/ozonesecure/vault.conf
index 2de359fc5d..a3467c0e2f 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/vault.conf
@@ -14,7 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-HDDS_VERSION=${hdds.version}
-OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
-OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_OPTS=
+OZONE-SITE.XML_ozone.secret.s3.store.provider=org.apache.hadoop.ozone.s3.remote.vault.VaultS3SecretStorageProvider
+OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.address=http://vault:8200
+OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.namespace="namespace"
+OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.enginever=2
+OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.secretpath=secret
+OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth=TOKEN
+OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth.token=00000000-0000-0000-0000-000000000000
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/krb5.conf
b/hadoop-ozone/dist/src/main/compose/ozonesecure/vault.yaml
similarity index 58%
rename from hadoop-ozone/dist/src/main/compose/ozonesecure-vault/krb5.conf
rename to hadoop-ozone/dist/src/main/compose/ozonesecure/vault.yaml
index eefc5b9c68..cf5be8df6a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-vault/krb5.conf
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/vault.yaml
@@ -14,28 +14,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-[logging]
-default = FILE:/var/log/krb5libs.log
-kdc = FILE:/var/log/krb5kdc.log
-admin_server = FILE:/var/log/kadmind.log
-
-[libdefaults]
- dns_canonicalize_hostname = false
- dns_lookup_realm = false
- ticket_lifetime = 24h
- renew_lifetime = 7d
- forwardable = true
- rdns = false
- default_realm = EXAMPLE.COM
-
-[realms]
- EXAMPLE.COM = {
- kdc = kdc
- admin_server = kdc
- max_renewable_life = 7d
- }
-
-[domain_realm]
- .example.com = EXAMPLE.COM
- example.com = EXAMPLE.COM
-
+version: "3"
+services:
+ om:
+ env_file:
+ - vault.conf
+ environment:
+ - OZONE_OPTS=-Dcom.sun.net.ssl.checkRevocation=false
+ -
OZONE_CLASSPATH=/opt/hadoop/share/ozone/lib/[email protected]@.jar:/opt/hadoop/share/ozone/lib/vault-java-driver-*.jar
+ vault:
+ image: hashicorp/vault:1.13.2
+ hostname: vault
+ cap_add:
+ - IPC_LOCK
+ ports:
+ - ${VAULT_PORT:-8200}:8200
+ command:
+ - server
+ - -dev
+ - -dev-root-token-id=00000000-0000-0000-0000-000000000000
+ - -namespace=namespace
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh
b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 87d1773a82..4c031e00cd 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -39,12 +39,12 @@ create_results_dir() {
chmod ogu+w "$RESULT_DIR"
}
-## @description find all the test.sh scripts in the immediate child dirs
+## @description find all the test*.sh scripts in the immediate child dirs
all_tests_in_immediate_child_dirs() {
- find . -mindepth 2 -maxdepth 2 -name test.sh | cut -c3- | sort
+ find . -mindepth 2 -maxdepth 2 -name 'test*.sh' | cut -c3- | sort
}
-## @description Find all test.sh scripts in immediate child dirs,
+## @description Find all test*.sh scripts in immediate child dirs,
## @description applying OZONE_ACCEPTANCE_SUITE or OZONE_TEST_SELECTOR filter.
find_tests(){
if [[ -n "${OZONE_ACCEPTANCE_SUITE}" ]]; then
@@ -381,35 +381,44 @@ generate_report(){
copy_results() {
local test_dir="$1"
local all_result_dir="$2"
+ local test_script="${3:-test.sh}"
local result_dir="${test_dir}/result"
- local test_dir_name=$(basename ${test_dir})
+ local test_dir_name="$(basename ${test_dir})"
+ local test_name="${test_dir_name}"
+ local target_dir="${all_result_dir}"/"${test_dir_name}"
+
+ if [[ -n "${test_script}" ]] && [[ "${test_script}" != "test.sh" ]]; then
+ local test_script_name=${test_script}
+ test_script_name=${test_script_name#test-}
+ test_script_name=${test_script_name#test_}
+ test_script_name=${test_script_name%.sh}
+ test_name="${test_name}-${test_script_name}"
+ target_dir="${target_dir}/${test_script_name}"
+ fi
+
if [[ -n "$(find "${result_dir}" -name "*.xml")" ]]; then
- rebot --nostatusrc -N "${test_dir_name}" -l NONE -r NONE -o
"${all_result_dir}/${test_dir_name}.xml" "${result_dir}"/*.xml
+ rebot --nostatusrc -N "${test_name}" -l NONE -r NONE -o
"${all_result_dir}/${test_name}.xml" "${result_dir}"/*.xml
rm -fv "${result_dir}"/*.xml "${result_dir}"/log.html
"${result_dir}"/report.html
fi
- mkdir -p "${all_result_dir}"/"${test_dir_name}"
- mv -v "${result_dir}"/* "${all_result_dir}"/"${test_dir_name}"/
+ mkdir -p "${target_dir}"
+ mv -v "${result_dir}"/* "${target_dir}"/
}
run_test_script() {
local d="$1"
- local test_script="$2"
-
- if [[ -z "$test_script" ]]; then
- test_script=./test.sh
- fi
+ local test_script="${2:-test.sh}"
- echo "Executing test in ${d}"
+ echo "Executing test ${d}/${test_script}"
#required to read the .env file from the right location
cd "${d}" || return
local ret=0
- if ! "$test_script"; then
+ if ! ./"$test_script"; then
ret=1
- echo "ERROR: Test execution of ${d} is FAILED!!!!"
+ echo "ERROR: Test execution of ${d}/${test_script} is FAILED!!!!"
fi
cd - > /dev/null
@@ -419,15 +428,17 @@ run_test_script() {
run_test_scripts() {
local ret=0
+ local d f t
for t in "$@"; do
d="$(dirname "${t}")"
+ f="$(basename "${t}")"
- if ! run_test_script "${d}"; then
+ if ! run_test_script "${d}" "${f}"; then
ret=1
fi
- copy_results "${d}" "${ALL_RESULT_DIR}"
+ copy_results "${d}" "${ALL_RESULT_DIR}" "${f}"
if [[ "${ret}" == "1" ]] && [[ "${FAIL_FAST:-}" == "true" ]]; then
break
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
