This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d4c41e542e HDDS-11334. Improve EC xcompat acceptance test (#7492)
d4c41e542e is described below
commit d4c41e542e77685ecd3de24a2619a34acab9dcae
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Fri Dec 6 13:37:03 2024 +0100
HDDS-11334. Improve EC xcompat acceptance test (#7492)
---
hadoop-ozone/dist/src/main/compose/testlib.sh | 36 ++++--
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 2 +-
.../dist/src/main/compose/xcompat/clients.yaml | 89 ++++++--------
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 94 +++++----------
.../src/main/smoketest/compatibility/read.robot | 132 ++++++++++++++++++---
.../src/main/smoketest/compatibility/setup.robot | 9 +-
.../src/main/smoketest/compatibility/write.robot | 42 +++++--
.../src/main/smoketest/ec/backward-compat.robot | 107 -----------------
.../dist/src/main/smoketest/ec/lib.resource | 12 --
.../dist/src/main/smoketest/ozone-lib/shell.robot | 14 ++-
10 files changed, 258 insertions(+), 279 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh
b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 9bb8839266..8ced94e500 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -30,8 +30,29 @@ fi
source ${_testlib_dir}/compose_v2_compatibility.sh
+: ${OZONE_COMPOSE_RUNNING:=false}
: ${SCM:=scm}
+# create temp directory for test data; only once, even if testlib.sh is
sourced again
+if [[ -z "${TEST_DATA_DIR:-}" ]] && [[ "${KEEP_RUNNING:-false}" == "false" ]];
then
+ export TEST_DATA_DIR="$(mktemp -d "${TMPDIR:-/tmp}"/robot-data-XXXXXX)"
+ chmod go+rx "${TEST_DATA_DIR}"
+ _compose_delete_test_data() {
+ rm -frv "${TEST_DATA_DIR}"
+ }
+
+ trap _compose_cleanup EXIT HUP INT TERM
+fi
+
+_compose_cleanup() {
+ if [[ "${OZONE_COMPOSE_RUNNING}" == "true" ]]; then
+ stop_docker_env || true
+ fi
+ if [[ "$(type -t _compose_delete_test_data || true)" == "function" ]]; then
+ _compose_delete_test_data
+ fi
+}
+
## @description create results directory, purging any prior data
create_results_dir() {
#delete previous results
@@ -140,13 +161,13 @@ start_docker_env(){
docker-compose --ansi never down --remove-orphans
- trap stop_docker_env EXIT HUP INT TERM
-
opts=""
if has_scalable_datanode; then
opts="--scale datanode=${datanode_count}"
fi
+ OZONE_COMPOSE_RUNNING=true
+ trap _compose_cleanup EXIT HUP INT TERM
docker-compose --ansi never up -d $opts
wait_for_safemode_exit
@@ -184,11 +205,11 @@ execute_robot_test(){
local output_name=$(get_output_name)
# find unique filename
- declare -i i=0
- OUTPUT_FILE="robot-${output_name}1.xml"
- while [[ -f $RESULT_DIR/$OUTPUT_FILE ]]; do
- let ++i
- OUTPUT_FILE="robot-${output_name}${i}.xml"
+ for ((i=1; i<1000; i++)); do
+ OUTPUT_FILE="robot-${output_name}$(printf "%03d" ${i}).xml"
+ if [[ ! -f $RESULT_DIR/$OUTPUT_FILE ]]; then
+ break;
+ fi
done
SMOKETEST_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/smoketest"
@@ -368,6 +389,7 @@ stop_docker_env(){
for i in $(seq 1 $down_repeats)
do
if docker-compose --ansi never --profile "*" down --remove-orphans; then
+ OZONE_COMPOSE_RUNNING=false
return
fi
if [[ ${i} -eq 1 ]]; then
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
index cd2d93e219..1893053802 100755
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-#suite:compat
+#suite:upgrade
TEST_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml
b/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml
index 25cb4fb19f..567845e088 100644
--- a/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml
+++ b/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml
@@ -14,69 +14,58 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+x-common-config:
+ &common-config
+ dns_search: .
+ env_file:
+ - docker-config
+ command: ["sleep","1000000"]
+
+x-old-config:
+ &old-config
+ <<: *common-config
+ volumes:
+ - ../..:/opt/ozone
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${TEST_DATA_DIR}:/testdata
+
+x-new-config:
+ &new-config
+ image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+ <<: *common-config
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${TEST_DATA_DIR}:/testdata
+
services:
old_client_1_0_0:
image: ${OZONE_IMAGE}:1.0.0${OZONE_IMAGE_FLAVOR}
- env_file:
- - docker-config
- volumes:
- - ../..:/opt/ozone
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- command: ["sleep","1000000"]
+ <<: *old-config
+
old_client_1_1_0:
image: ${OZONE_IMAGE}:1.1.0${OZONE_IMAGE_FLAVOR}
- env_file:
- - docker-config
- volumes:
- - ../..:/opt/ozone
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- command: ["sleep","1000000"]
+ <<: *old-config
+
old_client_1_2_1:
image: ${OZONE_IMAGE}:1.2.1${OZONE_IMAGE_FLAVOR}
- env_file:
- - docker-config
- volumes:
- - ../..:/opt/ozone
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- command: ["sleep","1000000"]
+ <<: *old-config
+
old_client_1_3_0:
image: ${OZONE_IMAGE}:1.3.0${OZONE_IMAGE_FLAVOR}
- env_file:
- - docker-config
- volumes:
- - ../..:/opt/ozone
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- command: ["sleep","1000000"]
+ <<: *old-config
+
old_client_1_4_0:
image: ${OZONE_IMAGE}:1.4.0${OZONE_IMAGE_FLAVOR}
- env_file:
- - docker-config
- volumes:
- - ../..:/opt/ozone
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- command: ["sleep","1000000"]
+ <<: *old-config
+
old_client_1_4_1:
image: ${OZONE_IMAGE}:1.4.1${OZONE_IMAGE_FLAVOR}
- env_file:
- - docker-config
- volumes:
- - ../..:/opt/ozone
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
- command: ["sleep","1000000"]
+ <<: *old-config
+
new_client:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- env_file:
- - docker-config
- volumes:
- - ../..:/opt/hadoop
- - ../_keytabs:/etc/security/keytabs
- - ./krb5.conf:/etc/krb5.conf
+ <<: *new-config
environment:
OZONE_OPTS:
- command: ["sleep","1000000"]
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh
b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh
index ed63f20f12..cf23b60bf7 100755
--- a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh
@@ -32,16 +32,17 @@ source "${COMPOSE_DIR}/../testlib.sh"
export SECURITY_ENABLED=true
: ${OZONE_BUCKET_KEY_NAME:=key1}
-old_client() {
- OZONE_DIR=/opt/ozone
- container=${client}
- "$@"
-}
+echo 'Compatibility Test' > "${TEST_DATA_DIR}"/small
+
+client() {
+ if [[ "${client_version}" == "${current_version}" ]]; then
+ OZONE_DIR=/opt/hadoop
+ container=new_client
+ else
+ OZONE_DIR=/opt/ozone
+ container="old_client_${client_version//./_}"
+ fi
-new_client() {
- OZONE_DIR=/opt/hadoop
- container=new_client
- client_version=${current_version}
"$@"
}
@@ -60,7 +61,7 @@ _write() {
execute_robot_test ${container} -N
"xcompat-cluster-${cluster_version}-client-${client_version}-write" \
-v CLIENT_VERSION:${client_version} \
-v CLUSTER_VERSION:${cluster_version} \
- -v SUFFIX:${client_version} \
+ -v TEST_DATA_DIR:/testdata \
compatibility/write.robot
}
@@ -71,87 +72,46 @@ _read() {
-v CLIENT_VERSION:${client_version} \
-v CLUSTER_VERSION:${cluster_version} \
-v DATA_VERSION:${data_version} \
- -v SUFFIX:${data_version} \
+ -v TEST_DATA_DIR:/testdata \
compatibility/read.robot
}
test_cross_compatibility() {
echo "Starting ${cluster_version} cluster with COMPOSE_FILE=${COMPOSE_FILE}"
- OZONE_KEEP_RESULTS=true start_docker_env
+ OZONE_KEEP_RESULTS=true start_docker_env 5
execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME}
_init
- new_client _write
- new_client _read ${current_version}
+ # first write with client matching cluster version
+ client_version="${cluster_version}" client _write
for client_version in "$@"; do
- client="old_client_${client_version//./_}"
-
- old_client _write
- old_client _read ${client_version}
-
- old_client _read ${current_version}
- new_client _read ${client_version}
+ # skip write, since already done
+ if [[ "${client_version}" == "${cluster_version}" ]]; then
+ continue
+ fi
+ client _write
done
- KEEP_RUNNING=false stop_docker_env
-}
-
-test_ec_cross_compatibility() {
- echo "Running Erasure Coded storage backward compatibility tests."
- # local cluster_versions_with_ec="1.3.0 1.4.0 ${current_version}"
- local cluster_versions_with_ec="${current_version}" # until HDDS-11334
- # TODO: debug acceptance test failures for client versions 1.0.0 on secure
clusters
- local non_ec_client_versions="1.1.0 1.2.1"
-
- for cluster_version in ${cluster_versions_with_ec}; do
- export COMPOSE_FILE=new-cluster.yaml:clients.yaml
cluster_version=${cluster_version}
- OZONE_KEEP_RESULTS=true start_docker_env 5
-
- echo -n "Generating data locally... "
- dd if=/dev/urandom of=/tmp/1mb bs=1048576 count=1 >/dev/null 2>&1
- dd if=/dev/urandom of=/tmp/2mb bs=1048576 count=2 >/dev/null 2>&1
- dd if=/dev/urandom of=/tmp/3mb bs=1048576 count=3 >/dev/null 2>&1
- echo "done"
- echo -n "Copy data into client containers... "
- for container in $(docker ps --format '{{.Names}}' | grep client); do
- docker cp /tmp/1mb ${container}:/tmp/1mb
- docker cp /tmp/2mb ${container}:/tmp/2mb
- docker cp /tmp/3mb ${container}:/tmp/3mb
- done
- echo "done"
- rm -f /tmp/1mb /tmp/2mb /tmp/3mb
-
-
- local prefix=$(LC_CTYPE=C tr -dc '[:alnum:]' < /dev/urandom | head -c 5 |
tr '[:upper:]' '[:lower:]')
- OZONE_DIR=/opt/hadoop
- new_client _kinit
- execute_robot_test new_client --include setup-ec-data -N
"xcompat-cluster-${cluster_version}-setup-data" -v prefix:"${prefix}"
ec/backward-compat.robot
- OZONE_DIR=/opt/ozone
-
- for client_version in ${non_ec_client_versions}; do
- client="old_client_${client_version//./_}"
- unset OUTPUT_PATH
- container="${client}" _kinit
- execute_robot_test "${client}" --include test-ec-compat -N
"xcompat-cluster-${cluster_version}-client-${client_version}-read-${cluster_version}"
-v prefix:"${prefix}" ec/backward-compat.robot
+ for client_version in "$@"; do
+ for data_version in $(echo "$client_version" "$cluster_version"
"$current_version" | xargs -n1 | sort -u); do
+ client _read ${data_version}
done
-
- KEEP_RUNNING=false stop_docker_env
done
+
+ KEEP_RUNNING=false stop_docker_env
}
create_results_dir
# current cluster with various clients
-COMPOSE_FILE=new-cluster.yaml:clients.yaml cluster_version=${current_version}
test_cross_compatibility ${old_versions}
+COMPOSE_FILE=new-cluster.yaml:clients.yaml cluster_version=${current_version}
test_cross_compatibility ${old_versions} ${current_version}
# old cluster with clients: same version and current version
for cluster_version in ${old_versions}; do
export OZONE_VERSION=${cluster_version}
- COMPOSE_FILE=old-cluster.yaml:clients.yaml test_cross_compatibility
${cluster_version}
+ COMPOSE_FILE=old-cluster.yaml:clients.yaml test_cross_compatibility
${cluster_version} ${current_version}
done
-
-test_ec_cross_compatibility
diff --git a/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot
b/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot
index 0415eb5401..9061677eae 100644
--- a/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot
@@ -18,40 +18,140 @@ Documentation Read Compatibility
Resource ../ozone-lib/shell.robot
Resource setup.robot
Test Timeout 5 minutes
-Suite Setup Create Local Test File
-*** Variables ***
-${SUFFIX} ${EMPTY}
+
+*** Keywords ***
+Key List With Replication
+ [arguments] ${args}
+ ${list} = Execute ozone sh key list ${args}
+ ${result} = Execute echo '${list}' | jq -r '[.name,
.replicationType, (.replicationFactor | tostring)] | join (" ")'
+ [return] ${result}
+
*** Test Cases ***
+Buckets Can Be Listed
+ ${result} = Execute ozone sh bucket list /vol1
+ Should Contain ${result} bucket1
+
+ IF '${CLUSTER_VERSION}' >= '${EC_VERSION}'
+ Should Contain ${result} ratis-${CLUSTER_VERSION}
+ Should Contain ${result} ecbucket-${CLUSTER_VERSION}
+ END
+
+Bucket Without Replication Config
+ Verify Bucket Empty Replication Config /vol1/bucket1
+
+Bucket With Replication Config
+ Pass Execution If '${CLUSTER_VERSION}' < '${EC_VERSION}' Cluster does
not support EC
+
+ IF '${CLIENT_VERSION}' >= '${EC_VERSION}'
+ Verify Bucket Replica Replication Config
/vol1/ratis-${CLUSTER_VERSION} RATIS THREE
+ Verify Bucket EC Replication Config
/vol1/ecbucket-${CLUSTER_VERSION} RS 3 2 1048576
+ ELSE
+ Verify Bucket Empty Replication Config
/vol1/ratis-${CLUSTER_VERSION}
+ Verify Bucket Empty Replication Config
/vol1/ecbucket-${CLUSTER_VERSION}
+ END
+
Key Can Be Read
- Key Should Match Local File /vol1/bucket1/key-${SUFFIX} ${TESTFILE}
+ Key Should Match Local File /vol1/bucket1/key-${DATA_VERSION}
${TESTFILE}
Encrypted Key Can Be Read
- Key Should Match Local File /vol1/encrypted-${SUFFIX}/key ${TESTFILE}
+ Key Should Match Local File /vol1/encrypted-${DATA_VERSION}/key
${TESTFILE}
+ File Should Match Local File
ofs://om/vol1/encrypted-${DATA_VERSION}/key ${TESTFILE}
+
+Key Read From Bucket With Replication
+ Pass Execution If '${CLUSTER_VERSION}' < '${EC_VERSION}' Cluster does
not support EC
+
+ Key Should Match Local File
/vol1/ratis-${CLUSTER_VERSION}/key-${DATA_VERSION} ${TESTFILE}
+
+ IF '${CLIENT_VERSION}' >= '${EC_VERSION}' or '${DATA_VERSION}' ==
'${CLIENT_VERSION}'
+ Key Should Match Local File
/vol1/ecbucket-${CLUSTER_VERSION}/key-${DATA_VERSION} ${TESTFILE}
+ ELSE
+ Assert Unsupported ozone sh key get -f
/vol1/ecbucket-${CLUSTER_VERSION}/key-${DATA_VERSION} /dev/null
+ END
Dir Can Be Listed
- Execute ozone fs -ls o3fs://bucket1.vol1/dir-${SUFFIX}
+ ${result} = Execute ozone fs -ls
o3fs://bucket1.vol1/dir-${DATA_VERSION}
+ Should contain ${result}
dir-${DATA_VERSION}/file-${DATA_VERSION}
+
+ Pass Execution If '${CLUSTER_VERSION}' < '${EC_VERSION}' Cluster does
not support EC
+
+# TODO HDDS-11803
+# ${result} = Execute ozone fs -ls ofs://om/vol1/
+# Should contain ${result} /vol1/ratis-${CLUSTER_VERSION}
+# Should contain ${result}
/vol1/ecbucket-${CLUSTER_VERSION}
+
+ IF '${CLIENT_VERSION}' < '${EC_VERSION}'
+ ${result} = Execute and checkrc ozone fs -ls
ofs://om/vol1/ecbucket-${CLUSTER_VERSION}/ 1
+ Should contain ${result} ls: The list of keys
contains keys with Erasure Coded replication set
+ END
+
+File Can Be Listed
+ ${result} = Execute ozone fs -ls
o3fs://bucket1.vol1/dir-${DATA_VERSION}/file-${DATA_VERSION}
+ Should contain ${result}
dir-${DATA_VERSION}/file-${DATA_VERSION}
+
+ Pass Execution If '${CLUSTER_VERSION}' < '${EC_VERSION}' Cluster does
not support EC
+
+ ${result} = Execute ozone fs -ls
ofs://om/vol1/ratis-${CLUSTER_VERSION}/file-${DATA_VERSION}
+ Should contain ${result}
/vol1/ratis-${CLUSTER_VERSION}/file-${DATA_VERSION}
+
+ IF '${CLIENT_VERSION}' >= '${EC_VERSION}' or '${DATA_VERSION}' ==
'${CLIENT_VERSION}'
+ ${result} = Execute ozone fs -ls
ofs://om/vol1/ecbucket-${CLUSTER_VERSION}/file-${DATA_VERSION}
+ Should contain ${result}
/vol1/ecbucket-${CLUSTER_VERSION}/file-${DATA_VERSION}
+ ELSE
+ ${result} = Execute and checkrc ozone fs -ls
ofs://om/vol1/ecbucket-${CLUSTER_VERSION}/file-${DATA_VERSION} 1
+ Should contain ${result} : No such file or directory
+ END
+
+Key List
+ IF '${CLIENT_VERSION}' >= '${EC_VERSION}'
+ ${result} = Execute ozone sh key list /vol1/bucket1
+ Should Contain ${result} key-${DATA_VERSION}
+ ELSE IF '${DATA_VERSION}' < '${EC_VERSION}' # New client creates
RATIS/ONE key by default: BUG?
+ ${result} = Key List With Replication /vol1/bucket1
+ Should contain ${result} key-${DATA_VERSION}
RATIS 3
+ END
+
+Key List In Bucket With Replication
+ Pass Execution If '${CLUSTER_VERSION}' < '${EC_VERSION}' Cluster does
not support EC
+
+ IF '${CLIENT_VERSION}' < '${EC_VERSION}'
+ ${result} = Key List With Replication
/vol1/ratis-${CLUSTER_VERSION}/
+ Should contain ${result} key-${DATA_VERSION} RATIS 3
+
+ Assert Unsupported ozone sh key list
/vol1/ecbucket-${CLUSTER_VERSION}/
+ ELSE
+ ${result} = Execute ozone sh key list
/vol1/ratis-${CLUSTER_VERSION}
+ Should Contain ${result} key-${DATA_VERSION}
+ ${result} = Execute ozone sh key list
/vol1/ecbucket-${CLUSTER_VERSION}
+ Should Contain ${result} key-${DATA_VERSION}
+ END
-Dir Can Be Listed Using Shell
- ${result} = Execute ozone sh key list /vol1/bucket1
- Should Contain ${result} key-${SUFFIX}
File Can Be Get
- Execute ozone fs -get o3fs://bucket1.vol1/dir-${SUFFIX}/file-${SUFFIX}
/tmp/
- Execute diff -q ${TESTFILE} /tmp/file-${SUFFIX}
- [teardown] Execute rm -f /tmp/file-${SUFFIX}
+ Key Should Match Local File
/vol1/bucket1/dir-${DATA_VERSION}/file-${DATA_VERSION} ${TESTFILE}
+ File Should Match Local File
o3fs://bucket1.vol1/dir-${DATA_VERSION}/file-${DATA_VERSION} ${TESTFILE}
+
+File Can Be Get From Bucket With Replication
+ Pass Execution If '${CLUSTER_VERSION}' < '${EC_VERSION}' Cluster does
not support EC
+
+ File Should Match Local File
ofs://om/vol1/ratis-${CLUSTER_VERSION}/file-${DATA_VERSION} ${TESTFILE}
+
+ IF '${CLIENT_VERSION}' >= '${EC_VERSION}' or '${DATA_VERSION}' ==
'${CLIENT_VERSION}'
+ File Should Match Local File
ofs://om/vol1/ecbucket-${CLUSTER_VERSION}/key-${DATA_VERSION} ${TESTFILE}
+ ELSE
+ ${result} = Execute and checkrc ozone fs -get
ofs://om/vol1/ecbucket-${CLUSTER_VERSION}/key-${DATA_VERSION} 1
+ Should contain ${result} : No such file or directory
+ END
FSO Bucket Can Be Read
Pass Execution If '${DATA_VERSION}' < '${FSO_VERSION}' Skipped
write test case
Pass Execution If '${CLIENT_VERSION}' < '${FSO_VERSION}' Client does
not support FSO
Pass Execution If '${CLUSTER_VERSION}' < '${FSO_VERSION}' Cluster
does not support FSO
- Execute ozone fs -get
ofs://om/vol1/fso-bucket-${SUFFIX}/dir/subdir/file ${TEMP_DIR}/
- Execute diff -q ${TESTFILE} ${TEMP_DIR}/file
- [teardown] Execute rm -f ${TEMP_DIR}/file
+ File Should Match Local File
ofs://om/vol1/fso-bucket-${DATA_VERSION}/dir/subdir/file ${TESTFILE}
HSync Lease Recover Can Be Used
Pass Execution If '${DATA_VERSION}' < '${FSO_VERSION}' Skipped
write test case
Pass Execution If '${CLIENT_VERSION}' < '${HSYNC_VERSION}' Client
does not support HSYNC
Pass Execution If '${CLUSTER_VERSION}' < '${HSYNC_VERSION}' Cluster
does not support HSYNC
- Execute ozone debug recover
--path=ofs://om/vol1/fso-bucket-${SUFFIX}/dir/subdir/file
+ Execute ozone debug recover
--path=ofs://om/vol1/fso-bucket-${DATA_VERSION}/dir/subdir/file
diff --git a/hadoop-ozone/dist/src/main/smoketest/compatibility/setup.robot
b/hadoop-ozone/dist/src/main/smoketest/compatibility/setup.robot
index 575c715a57..4f41d280a6 100644
--- a/hadoop-ozone/dist/src/main/smoketest/compatibility/setup.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/compatibility/setup.robot
@@ -19,12 +19,7 @@ Library OperatingSystem
Resource ../ozone-lib/shell.robot
*** Variables ***
-${SUFFIX} ${EMPTY}
+${EC_VERSION} 1.3.0
${FSO_VERSION} 1.3.0
${HSYNC_VERSION} 2.0.0
-
-
-*** Keywords ***
-Create Local Test File
- Set Suite Variable ${TESTFILE} /tmp/test-data-${SUFFIX}.txt
- Create File ${TESTFILE} Compatibility Test
+${TESTFILE} ${TEST_DATA_DIR}/small
diff --git a/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot
b/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot
index 9a1ae6c75a..0497edaca1 100644
--- a/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot
@@ -20,35 +20,57 @@ Resource setup.robot
Resource ../lib/fs.robot
Resource ../ozone-lib/freon.robot
Test Timeout 5 minutes
-Suite Setup Create Local Test File
*** Variables ***
-${SUFFIX} ${EMPTY}
${ENCRYPTION_KEY} key1
*** Test Cases ***
+Create Bucket With Replication Type
+ Pass Execution If '${CLIENT_VERSION}' < '${EC_VERSION}' Client does
not support EC
+ Pass Execution If '${CLUSTER_VERSION}' < '${EC_VERSION}' Cluster does
not support EC
+ Execute ozone sh bucket create --replication 3 --type RATIS
/vol1/ratis-${CLIENT_VERSION}
+ Execute ozone sh bucket create --replication rs-3-2-1024k
--type EC /vol1/ecbucket-${CLIENT_VERSION}
+
Create Encrypted Bucket
- Execute ozone sh bucket create -k ${ENCRYPTION_KEY}
/vol1/encrypted-${SUFFIX}
+ Execute ozone sh bucket create -k ${ENCRYPTION_KEY}
/vol1/encrypted-${CLIENT_VERSION}
Create Key in Encrypted Bucket
- Execute ozone sh key put /vol1/encrypted-${SUFFIX}/key ${TESTFILE}
+ Execute ozone sh key put /vol1/encrypted-${CLIENT_VERSION}/key
${TESTFILE}
Key Can Be Written
- Create Key /vol1/bucket1/key-${SUFFIX} ${TESTFILE}
+ Create Key /vol1/bucket1/key-${CLIENT_VERSION} ${TESTFILE}
+
+Key Can Be Written To Bucket With Replication Type
+ Pass Execution If '${CLUSTER_VERSION}' < '${EC_VERSION}' Cluster does
not support EC
+ Execute ozone sh key put
/vol1/ratis-${CLUSTER_VERSION}/key-${CLIENT_VERSION} ${TESTFILE}
+ Execute ozone sh key put
/vol1/ecbucket-${CLUSTER_VERSION}/key-${CLIENT_VERSION} ${TESTFILE}
+
+Key Can Be Deleted
+ Create Key /vol1/bucket1/to-be-deleted-${CLIENT_VERSION} ${TESTFILE}
+ Execute ozone sh key delete
/vol1/bucket1/to-be-deleted-${CLIENT_VERSION}
Dir Can Be Created
- Execute ozone fs -mkdir o3fs://bucket1.vol1/dir-${SUFFIX}
+ Execute ozone fs -mkdir o3fs://bucket1.vol1/dir-${CLIENT_VERSION}
File Can Be Put
- Execute ozone fs -put ${TESTFILE}
o3fs://bucket1.vol1/dir-${SUFFIX}/file-${SUFFIX}
+ Execute ozone fs -put ${TESTFILE}
o3fs://bucket1.vol1/dir-${CLIENT_VERSION}/file-${CLIENT_VERSION}
+
+File Can Be Put To Bucket With Replication Type
+ Pass Execution If '${CLUSTER_VERSION}' < '${EC_VERSION}' Cluster does
not support EC
+ Execute ozone fs -put ${TESTFILE}
ofs://om/vol1/ratis-${CLUSTER_VERSION}/file-${CLIENT_VERSION}
+ Execute ozone fs -put ${TESTFILE}
ofs://om/vol1/ecbucket-${CLUSTER_VERSION}/file-${CLIENT_VERSION}
+
+File Can Be Deleted
+ Execute ozone fs -put ${TESTFILE}
o3fs://bucket1.vol1/dir-${CLIENT_VERSION}/to-be-deleted
+ Execute ozone fs -rm -skipTrash
o3fs://bucket1.vol1/dir-${CLIENT_VERSION}/to-be-deleted
FSO Bucket Can Be Created and Used
Pass Execution If '${CLIENT_VERSION}' < '${FSO_VERSION}' Client does
not support FSO
Pass Execution If '${CLUSTER_VERSION}' < '${FSO_VERSION}' Cluster
does not support FSO
- Execute ozone sh bucket create --layout FILE_SYSTEM_OPTIMIZED
/vol1/fso-bucket-${SUFFIX}
- Execute ozone fs -mkdir -p ofs://om/vol1/fso-bucket-${SUFFIX}/dir/subdir
- Execute ozone fs -put ${TESTFILE}
ofs://om/vol1/fso-bucket-${SUFFIX}/dir/subdir/file
+ Execute ozone sh bucket create --layout FILE_SYSTEM_OPTIMIZED
/vol1/fso-bucket-${CLIENT_VERSION}
+ Execute ozone fs -mkdir -p
ofs://om/vol1/fso-bucket-${CLIENT_VERSION}/dir/subdir
+ Execute ozone fs -put ${TESTFILE}
ofs://om/vol1/fso-bucket-${CLIENT_VERSION}/dir/subdir/file
HSync Can Be Used To Create Keys
Pass Execution If '${CLIENT_VERSION}' < '${HSYNC_VERSION}' Client
does not support HSYNC
diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/backward-compat.robot
b/hadoop-ozone/dist/src/main/smoketest/ec/backward-compat.robot
deleted file mode 100644
index d64c768687..0000000000
--- a/hadoop-ozone/dist/src/main/smoketest/ec/backward-compat.robot
+++ /dev/null
@@ -1,107 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Test EC backward compatibility
-Library OperatingSystem
-Resource lib.resource
-
-*** Variables ***
-${PREFIX} ${EMPTY}
-${VOLUME} vol${PREFIX}
-
-*** Test Cases ***
-Setup Cluster Data
- [Tags] setup-ec-data
- Prepare Data For Xcompat Tests
-
-Test Read Key Compat
- [Tags] test-ec-compat
- Key Should Match Local File /${VOLUME}/ratis/3mb /tmp/3mb
- Key Should Match Local File /${VOLUME}/default/3mb /tmp/3mb
-
- ${result} = Execute and checkrc ozone sh key get -f
/${VOLUME}/ecbucket/3mb /dev/null 255
- Should Contain ${result} NOT_SUPPORTED_OPERATION
-
-Test Listing Compat
- [Tags] test-ec-compat
- ${result} = Execute ozone sh volume list | jq -r '.name'
- Should contain ${result} ${VOLUME}
- ${result} = Execute ozone sh bucket list /${VOLUME}/ | jq -r
'.name'
- Should contain ${result} default
- Should contain ${result} ratis
- Should contain ${result} ec
- ${result} = Execute ozone sh key list /${VOLUME}/default/ | jq -r
'[.name, .replicationType, (.replicationFactor | tostring)] | join (" ")'
- Should contain ${result} 3mb RATIS 3
- ${result} = Execute ozone sh key list /${VOLUME}/ratis/ | jq -r
'[.name, .replicationType, (.replicationFactor | tostring)] | join (" ")'
- Should contain ${result} 3mb RATIS 3
-
- ${result} = Execute and checkrc ozone sh key list
/${VOLUME}/ecbucket/ 255
- Should contain ${result} NOT_SUPPORTED_OPERATION
-
-Test Info Compat
- [Tags] test-ec-compat
- ${result} = Execute ozone sh volume info ${VOLUME} | jq -r '.name'
- Should contain ${result} ${VOLUME}
- ${result} = Execute ozone sh bucket info /${VOLUME}/default | jq
-r '[.name, .replicationType, .replicationFactor] | join (" ")'
- Should contain ${result} default # there is no
replication config in the old client for bucket info
- ${result} = Execute ozone sh bucket info /${VOLUME}/ratis | jq -r
'[.name, .replicationType, .replicationFactor] | join (" ")'
- Should contain ${result} ratis # there is no
replication config in the old client for bucket info
- ${result} = Execute ozone sh bucket info /${VOLUME}/ecbucket | jq
-r '[.name, .replicationType, .replicationFactor] | join (" ")'
- Should contain ${result} ec # there is no
replication config in the old client for bucket info
-
-Test FS Compat
- [Tags] test-ec-compat
- ${result} = Execute ozone fs -ls ofs://om/
- Should contain ${result} /${VOLUME}
- ${result} = Execute ozone fs -ls ofs://om/${VOLUME}/
- Should contain ${result} /${VOLUME}/default
- Should contain ${result} /${VOLUME}/ratis
- Should contain ${result} /${VOLUME}/ecbucket
- ${result} = Execute ozone fs -ls ofs://om/${VOLUME}/default/3mb
- Should contain ${result} /${VOLUME}/default/3mb
- ${result} = Execute ozone fs -ls ofs://om/${VOLUME}/ratis/3mb
- Should contain ${result} /${VOLUME}/ratis/3mb
-
- ${result} = Execute and checkrc ozone fs -ls
ofs://om/${VOLUME}/ecbucket/ 1
- Should contain ${result} ls: The list of keys contains
keys with Erasure Coded replication set
- ${result} = Execute and checkrc ozone fs -ls
ofs://om/${VOLUME}/ecbucket/3mb 1
- Should contain ${result} : No such file or directory
- ${result} = Execute and checkrc ozone fs -get
ofs://om/${VOLUME}/ecbucket/3mb 1
- Should contain ${result} : No such file or directory
-
-Test FS Client Can Read Own Writes
- [Tags] test-ec-compat
- Execute ozone fs -put /tmp/1mb ofs://om/${VOLUME}/default/1mb
- Execute ozone fs -put /tmp/1mb ofs://om/${VOLUME}/ratis/1mb
- Execute ozone fs -put /tmp/1mb ofs://om/${VOLUME}/ecbucket/1mb
- Key Should Match Local File /${VOLUME}/ratis/1mb /tmp/1mb
- Key Should Match Local File /${VOLUME}/ratis/1mb /tmp/1mb
- Key Should Match Local File /${VOLUME}/ratis/1mb /tmp/1mb
- Execute ozone fs -rm -skipTrash ofs://om/${VOLUME}/default/1mb
- Execute ozone fs -rm -skipTrash ofs://om/${VOLUME}/ratis/1mb
- Execute ozone fs -rm -skipTrash ofs://om/${VOLUME}/ecbucket/1mb
-
-Test Client Can Read Own Writes
- [Tags] test-ec-compat
- Execute ozone sh key put /${VOLUME}/default/2mb /tmp/2mb
- Execute ozone sh key put /${VOLUME}/ratis/2mb /tmp/2mb
- Execute ozone sh key put /${VOLUME}/ecbucket/2mb /tmp/2mb
- Key Should Match Local File /${VOLUME}/ratis/2mb /tmp/2mb
- Key Should Match Local File /${VOLUME}/ratis/2mb /tmp/2mb
- Key Should Match Local File /${VOLUME}/ratis/2mb /tmp/2mb
- Execute ozone sh key delete /${VOLUME}/default/2mb
- Execute ozone sh key delete /${VOLUME}/ratis/2mb
- Execute ozone sh key delete /${VOLUME}/ecbucket/2mb
diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/lib.resource
b/hadoop-ozone/dist/src/main/smoketest/ec/lib.resource
index f6a84f9e06..f01ec191f5 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ec/lib.resource
+++ b/hadoop-ozone/dist/src/main/smoketest/ec/lib.resource
@@ -21,8 +21,6 @@ Resource ../ozone-lib/shell.robot
*** Variables ***
${SCM} scm
-${PREFIX} ${EMPTY}
-${VOLUME} vol${PREFIX}
*** Keywords ***
Prepare For Tests
@@ -31,13 +29,3 @@ Prepare For Tests
Execute dd if=/dev/urandom of=/tmp/3mb bs=1048576 count=3
Execute dd if=/dev/urandom of=/tmp/100mb bs=1048576 count=100
Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user
testuser testuser.keytab
-
-# xcompat/test.sh creates unified test data files in /tmp for client containers
-Prepare Data For Xcompat Tests
- Execute ozone sh volume create /${VOLUME}
- Execute ozone sh bucket create /${VOLUME}/default
- Execute ozone sh bucket create --replication 3 --type RATIS
/${VOLUME}/ratis
- Execute ozone sh bucket create --replication rs-3-2-1024k
--type EC /${VOLUME}/ecbucket
- Execute ozone sh key put /${VOLUME}/default/3mb /tmp/3mb
- Execute ozone sh key put /${VOLUME}/ratis/3mb /tmp/3mb
- Execute ozone sh key put /${VOLUME}/ecbucket/3mb /tmp/3mb
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
index 2b54e8bf33..d5762f912e 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
@@ -28,10 +28,10 @@ Bucket Exists
[Return] ${TRUE}
Compare Key With Local File
- [arguments] ${key} ${file}
+ [arguments] ${key} ${file} ${cmd}=sh key get
${postfix} = Generate Random String 5 [NUMBERS]
${tmpfile} = Set Variable /tmp/tempkey-${postfix}
- Execute ozone sh key get ${key} ${tmpfile}
+ Execute ozone ${cmd} ${key} ${tmpfile}
${rc} = Run And Return Rc diff -q ${file} ${tmpfile}
Execute rm -f ${tmpfile}
${result} = Set Variable If ${rc} == 0 ${TRUE} ${FALSE}
@@ -42,6 +42,11 @@ Key Should Match Local File
${matches} = Compare Key With Local File ${key} ${file}
Should Be True ${matches}
+File Should Match Local File
+ [arguments] ${key} ${file}
+ ${matches} = Compare Key With Local File ${key} ${file} fs -get
+ Should Be True ${matches}
+
Verify ACL
[arguments] ${object_type} ${object} ${type} ${name}
${acls}
${actual_acls} = Execute ozone sh ${object_type} getacl
${object} | jq -r '.[] | select(.type == "${type}") | select(.name ==
"${name}") | .aclList[]' | xargs
@@ -70,6 +75,11 @@ Create Key
Should not contain ${output} Failed
Log Uploaded ${file} to ${key}
+Assert Unsupported
+ [arguments] ${cmd}
+ ${result} = Execute and checkrc ${cmd} 255
+ Should Contain ${result} NOT_SUPPORTED_OPERATION
+
Verify Bucket Empty Replication Config
[arguments] ${bucket}
${result} = Execute ozone sh bucket info ${bucket}
| jq -r '.replicationConfig'
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
