This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c955729 HDDS-4194. Create a script to check AWS S3 compatibility
(#1383)
c955729 is described below
commit c955729c3c134d1a3e28eca7850c44f001d486ad
Author: Elek, Márton <[email protected]>
AuthorDate: Thu Sep 24 21:43:58 2020 +0200
HDDS-4194. Create a script to check AWS S3 compatibility (#1383)
---
.../src/main/smoketest/s3/MultipartUpload.robot | 130 ++++++++++-----------
.../dist/src/main/smoketest/s3/commonawslib.robot | 5 +
.../dist/src/main/smoketest/s3/objectcopy.robot | 23 ++--
.../dist/src/main/smoketest/s3/objectdelete.robot | 28 ++---
.../src/main/smoketest/s3/objectmultidelete.robot | 24 ++--
.../dist/src/main/smoketest/s3/objectputget.robot | 40 +++----
.../src/main/smoketest/s3/s3_compatbility_check.sh | 47 ++++++++
7 files changed, 173 insertions(+), 124 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
index 8f86a67..e05a706 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
@@ -21,13 +21,13 @@ Resource ../commonlib.robot
Resource commonawslib.robot
Test Timeout 5 minutes
Suite Setup Setup s3 tests
+Test Setup Generate random prefix
*** Keywords ***
Create Random file
[arguments] ${size_in_megabytes}
Execute dd if=/dev/urandom of=/tmp/part1 bs=1048576
count=${size_in_megabytes}
-
*** Variables ***
${ENDPOINT_URL} http://s3g:9878
${BUCKET} generated
@@ -35,16 +35,16 @@ ${BUCKET} generated
*** Test Cases ***
Test Multipart Upload
- ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key multipartKey
+ ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/multipartKey
${uploadID} = Execute and checkrc echo '${result}' | jq -r
'.UploadId' 0
Should contain ${result} ${BUCKET}
- Should contain ${result} multipartKey
+ Should contain ${result}
${PREFIX}/multipartKey
Should contain ${result} UploadId
# initiate again
- ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key multipartKey
+ ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/multipartKey
${nextUploadID} = Execute and checkrc echo '${result}' | jq -r
'.UploadId' 0
Should contain ${result} ${BUCKET}
- Should contain ${result} multipartKey
+ Should contain ${result}
${PREFIX}/multipartKey
Should contain ${result} UploadId
Should Not Be Equal ${uploadID} ${nextUploadID}
@@ -54,149 +54,149 @@ Test Multipart Upload
# multipart upload, uploading each part as 5MB file, exception is for last part
Run Keyword Create Random file 5
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id
${nextUploadID}
Should contain ${result} ETag
# override part
Run Keyword Create Random file 5
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id
${nextUploadID}
Should contain ${result} ETag
Test Multipart Upload Complete
- ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key multipartKey1
+ ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/multipartKey1
${uploadID} = Execute and checkrc echo '${result}' | jq -r
'.UploadId' 0
Should contain ${result} ${BUCKET}
- Should contain ${result} multipartKey
+ Should contain ${result}
${PREFIX}/multipartKey
Should contain ${result} UploadId
#upload parts
Run Keyword Create Random file 5
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey1 --part-number 1 --body /tmp/part1 --upload-id
${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
Execute echo "Part2" > /tmp/part2
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey1 --part-number 2 --body /tmp/part2 --upload-id
${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
#complete multipart upload
- ${result} = Execute AWSS3APICli complete-multipart-upload
--upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey1
--multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
+ ${result} = Execute AWSS3APICli complete-multipart-upload
--upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey1
--multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
Should contain ${result} ${BUCKET}
- Should contain ${result} multipartKey1
+ Should contain ${result}
${PREFIX}/multipartKey1
Should contain ${result} ETag
#read file and check the key
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key multipartKey1 /tmp/multipartKey1.result
- Execute cat /tmp/part1
/tmp/part2 > /tmp/multipartKey1
- Compare files /tmp/multipartKey1
/tmp/multipartKey1.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/multipartKey1 /tmp/${PREFIX}-multipartKey1.result
+ Execute cat /tmp/part1
/tmp/part2 > /tmp/${PREFIX}-multipartKey1
+ Compare files /tmp/${PREFIX}-multipartKey1
/tmp/${PREFIX}-multipartKey1.result
Test Multipart Upload Complete Entity too small
- ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key multipartKey2
+ ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/multipartKey2
${uploadID} = Execute and checkrc echo '${result}' | jq -r
'.UploadId' 0
Should contain ${result} ${BUCKET}
- Should contain ${result} multipartKey
+ Should contain ${result}
${PREFIX}/multipartKey
Should contain ${result} UploadId
#upload parts
Execute echo "Part1" > /tmp/part1
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey2 --part-number 1 --body /tmp/part1 --upload-id
${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
Execute echo "Part2" > /tmp/part2
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey2 --part-number 2 --body /tmp/part2 --upload-id
${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
#complete multipart upload
- ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
multipartKey2 --multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255
+ ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
${PREFIX}/multipartKey2 --multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255
Should contain ${result} EntityTooSmall
Test Multipart Upload Complete Invalid part errors and complete mpu with few
parts
- ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key multipartKey3
+ ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/multipartKey3
${uploadID} = Execute and checkrc echo '${result}' | jq -r
'.UploadId' 0
Should contain ${result} ${BUCKET}
- Should contain ${result} multipartKey
+ Should contain ${result}
${PREFIX}/multipartKey
Should contain ${result} UploadId
#complete multipart upload when no parts uploaded
- ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
multipartKey3 --multipart-upload
'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255
+ ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
${PREFIX}/multipartKey3 --multipart-upload
'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255
Should contain ${result} InvalidPart
- ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
multipartKey3 --multipart-upload
'Parts=[{ETag=etag1,PartNumber=2},{ETag=etag2,PartNumber=1}]' 255
+ ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
${PREFIX}/multipartKey3 --multipart-upload
'Parts=[{ETag=etag1,PartNumber=2},{ETag=etag2,PartNumber=1}]' 255
Should contain ${result} InvalidPart
#upload parts
Run Keyword Create Random file 5
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey3 --part-number 1 --body /tmp/part1 --upload-id
${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key multipartKey3 --part-number 2 --body /tmp/part1 --upload-id ${uploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey3 --part-number 2 --body /tmp/part1 --upload-id
${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
Execute echo "Part3" > /tmp/part3
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key multipartKey3 --part-number 3 --body /tmp/part3 --upload-id ${uploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey3 --part-number 3 --body /tmp/part3 --upload-id
${uploadID}
${eTag3} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
#complete multipart upload
- ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
multipartKey3 --multipart-upload
'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255
+ ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
${PREFIX}/multipartKey3 --multipart-upload
'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255
Should contain ${result} InvalidPart
- ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
multipartKey3 --multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=etag2,PartNumber=2}]' 255
+ ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
${PREFIX}/multipartKey3 --multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=etag2,PartNumber=2}]' 255
Should contain ${result} InvalidPart
- ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
multipartKey3 --multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=4},{ETag=etag2,PartNumber=2}]' 255
+ ${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
${PREFIX}/multipartKey3 --multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=4},{ETag=etag2,PartNumber=2}]' 255
Should contain ${result} InvalidPartOrder
#complete multipart upload(merge with few parts)
- ${result} = Execute AWSS3APICli complete-multipart-upload
--upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3
--multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag3},PartNumber=3}]'
+ ${result} = Execute AWSS3APICli complete-multipart-upload
--upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3
--multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag3},PartNumber=3}]'
Should contain ${result} ${BUCKET}
- Should contain ${result} multipartKey3
+ Should contain ${result}
${PREFIX}/multipartKey3
Should contain ${result} ETag
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key multipartKey3 /tmp/multipartKey3.result
- Execute cat /tmp/part1 /tmp/part3 >
/tmp/multipartKey3
- Compare files /tmp/multipartKey3 /tmp/multipartKey3.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/multipartKey3 /tmp/${PREFIX}-multipartKey3.result
+ Execute cat /tmp/part1 /tmp/part3 >
/tmp/${PREFIX}-multipartKey3
+ Compare files /tmp/${PREFIX}-multipartKey3
/tmp/${PREFIX}-multipartKey3.result
Test abort Multipart upload
- ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key multipartKey4 --storage-class REDUCED_REDUNDANCY
+ ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/multipartKey4 --storage-class
REDUCED_REDUNDANCY
${uploadID} = Execute and checkrc echo '${result}' | jq -r
'.UploadId' 0
Should contain ${result} ${BUCKET}
- Should contain ${result} multipartKey
+ Should contain ${result}
${PREFIX}/multipartKey
Should contain ${result} UploadId
- ${result} = Execute AWSS3APICli and checkrc
abort-multipart-upload --bucket ${BUCKET} --key multipartKey4 --upload-id
${uploadID} 0
+ ${result} = Execute AWSS3APICli and checkrc
abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey4
--upload-id ${uploadID} 0
Test abort Multipart upload with invalid uploadId
- ${result} = Execute AWSS3APICli and checkrc
abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id
"random" 255
+ ${result} = Execute AWSS3APICli and checkrc
abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey5
--upload-id "random" 255
Upload part with Incorrect uploadID
Execute echo "Multipart upload" >
/tmp/testfile
- ${result} = Execute AWSS3APICli and checkrc upload-part
--bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile
--upload-id "random" 255
+ ${result} = Execute AWSS3APICli and checkrc upload-part
--bucket ${BUCKET} --key ${PREFIX}/multipartKey --part-number 1 --body
/tmp/testfile --upload-id "random" 255
Should contain ${result} NoSuchUpload
Test list parts
#initiate multipart upload
- ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key multipartKey5
+ ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/multipartKey5
${uploadID} = Execute and checkrc echo '${result}' | jq -r
'.UploadId' 0
Should contain ${result} ${BUCKET}
- Should contain ${result} multipartKey
+ Should contain ${result}
${PREFIX}/multipartKey
Should contain ${result} UploadId
#upload parts
Run Keyword Create Random file 5
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey5 --part-number 1 --body /tmp/part1 --upload-id
${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
Execute echo "Part2" > /tmp/part2
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey5 --part-number 2 --body /tmp/part2 --upload-id
${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
#list parts
- ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET}
--key multipartKey5 --upload-id ${uploadID}
+ ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET}
--key ${PREFIX}/multipartKey5 --upload-id ${uploadID}
${part1} = Execute and checkrc echo '${result}' | jq -r
'.Parts[0].ETag' 0
${part2} = Execute and checkrc echo '${result}' | jq -r
'.Parts[1].ETag' 0
Should Be equal ${part1} ${eTag1}
@@ -204,19 +204,19 @@ Test list parts
Should contain ${result} STANDARD
#list parts with max-items and next token
- ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET}
--key multipartKey5 --upload-id ${uploadID} --max-items 1
+ ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET}
--key ${PREFIX}/multipartKey5 --upload-id ${uploadID} --max-items 1
${part1} = Execute and checkrc echo '${result}' | jq -r
'.Parts[0].ETag' 0
${token} = Execute and checkrc echo '${result}' | jq -r
'.NextToken' 0
Should Be equal ${part1} ${eTag1}
Should contain ${result} STANDARD
- ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET}
--key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token
${token}
+ ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET}
--key ${PREFIX}/multipartKey5 --upload-id ${uploadID} --max-items 1
--starting-token ${token}
${part2} = Execute and checkrc echo '${result}' | jq -r
'.Parts[0].ETag' 0
Should Be equal ${part2} ${eTag2}
Should contain ${result} STANDARD
#finally abort it
- ${result} = Execute AWSS3APICli and checkrc
abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id
${uploadID} 0
+ ${result} = Execute AWSS3APICli and checkrc
abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey5
--upload-id ${uploadID} 0
Test Multipart Upload with the simplified aws s3 cp API
Create Random file 22
@@ -227,67 +227,67 @@ Test Multipart Upload with the simplified aws s3 cp API
Test Multipart Upload Put With Copy
Run Keyword Create Random file 5
- ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET}
--key copytest/source --body /tmp/part1
+ ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET}
--key ${PREFIX}/copytest/source --body /tmp/part1
- ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key copytest/destination
+ ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/copytest/destination
${uploadID} = Execute and checkrc echo '${result}' | jq -r
'.UploadId' 0
Should contain ${result} ${BUCKET}
Should contain ${result} UploadId
- ${result} = Execute AWSS3APICli upload-part-copy --bucket
${BUCKET} --key copytest/destination --upload-id ${uploadID} --part-number 1
--copy-source ${BUCKET}/copytest/source
+ ${result} = Execute AWSS3APICli upload-part-copy --bucket
${BUCKET} --key ${PREFIX}/copytest/destination --upload-id ${uploadID}
--part-number 1 --copy-source ${BUCKET}/${PREFIX}/copytest/source
Should contain ${result} ETag
Should contain ${result} LastModified
${eTag1} = Execute and checkrc echo '${result}' | jq -r
'.CopyPartResult.ETag' 0
- Execute AWSS3APICli complete-multipart-upload
--upload-id ${uploadID} --bucket ${BUCKET} --key copytest/destination
--multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1}]'
- Execute AWSS3APICli get-object --bucket ${BUCKET}
--key copytest/destination /tmp/part-result
+ Execute AWSS3APICli complete-multipart-upload
--upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/copytest/destination
--multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1}]'
+ Execute AWSS3APICli get-object --bucket ${BUCKET}
--key ${PREFIX}/copytest/destination /tmp/part-result
Compare files /tmp/part1
/tmp/part-result
Test Multipart Upload Put With Copy and range
Run Keyword Create Random file 10
- ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET}
--key copyrange/source --body /tmp/part1
+ ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET}
--key ${PREFIX}/copyrange/source --body /tmp/part1
- ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key copyrange/destination
+ ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/copyrange/destination
${uploadID} = Execute and checkrc echo '${result}' | jq -r
'.UploadId' 0
Should contain ${result} ${BUCKET}
Should contain ${result} UploadId
- ${result} = Execute AWSS3APICli upload-part-copy --bucket
${BUCKET} --key copyrange/destination --upload-id ${uploadID} --part-number 1
--copy-source ${BUCKET}/copyrange/source --copy-source-range bytes=0-10485757
+ ${result} = Execute AWSS3APICli upload-part-copy --bucket
${BUCKET} --key ${PREFIX}/copyrange/destination --upload-id ${uploadID}
--part-number 1 --copy-source ${BUCKET}/${PREFIX}/copyrange/source
--copy-source-range bytes=0-10485757
Should contain ${result} ETag
Should contain ${result} LastModified
${eTag1} = Execute and checkrc echo '${result}' | jq -r
'.CopyPartResult.ETag' 0
- ${result} = Execute AWSS3APICli upload-part-copy --bucket
${BUCKET} --key copyrange/destination --upload-id ${uploadID} --part-number 2
--copy-source ${BUCKET}/copyrange/source --copy-source-range
bytes=10485758-10485759
+ ${result} = Execute AWSS3APICli upload-part-copy --bucket
${BUCKET} --key ${PREFIX}/copyrange/destination --upload-id ${uploadID}
--part-number 2 --copy-source ${BUCKET}/${PREFIX}/copyrange/source
--copy-source-range bytes=10485758-10485759
Should contain ${result} ETag
Should contain ${result} LastModified
${eTag2} = Execute and checkrc echo '${result}' | jq -r
'.CopyPartResult.ETag' 0
- Execute AWSS3APICli complete-multipart-upload
--upload-id ${uploadID} --bucket ${BUCKET} --key copyrange/destination
--multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
- Execute AWSS3APICli get-object --bucket ${BUCKET}
--key copyrange/destination /tmp/part-result
+ Execute AWSS3APICli complete-multipart-upload
--upload-id ${uploadID} --bucket ${BUCKET} --key
${PREFIX}/copyrange/destination --multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
+ Execute AWSS3APICli get-object --bucket ${BUCKET}
--key ${PREFIX}/copyrange/destination /tmp/part-result
Compare files /tmp/part1
/tmp/part-result
Test Multipart Upload list
- ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key listtest/key1
+ ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/listtest/key1
${uploadID1} = Execute and checkrc echo '${result}' | jq -r
'.UploadId' 0
Should contain ${result} ${BUCKET}
- Should contain ${result} listtest/key1
+ Should contain ${result}
${PREFIX}/listtest/key1
Should contain ${result} UploadId
- ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key listtest/key2
+ ${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/listtest/key2
${uploadID2} = Execute and checkrc echo '${result}' | jq -r
'.UploadId' 0
Should contain ${result} ${BUCKET}
- Should contain ${result} listtest/key2
+ Should contain ${result}
${PREFIX}/listtest/key2
Should contain ${result} UploadId
- ${result} = Execute AWSS3APICli list-multipart-uploads
--bucket ${BUCKET} --prefix listtest
+ ${result} = Execute AWSS3APICli list-multipart-uploads
--bucket ${BUCKET} --prefix ${PREFIX}/listtest
Should contain ${result} ${uploadID1}
Should contain ${result} ${uploadID2}
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
index 74dba38..505e0d1 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
@@ -91,6 +91,7 @@ Create bucket with name
Should contain ${result}
${ENDPOINT_URL}/${bucket}
Setup s3 tests
+ Run Keyword Generate random prefix
Run Keyword Install aws cli
Run Keyword if '${OZONE_S3_SET_CREDENTIALS}' == 'true' Setup v4
headers
${BUCKET} = Run Keyword if '${BUCKET}'
== 'generated' Create bucket
@@ -109,3 +110,7 @@ Create link
[arguments] ${bucket}
Execute ozone sh bucket link
o3://${OM_SERVICE_ID}/legacy/source-bucket o3://${OM_SERVICE_ID}/s3v/${bucket}
[return] ${bucket}
+
+Generate random prefix
+ ${random} = Generate Random String 5 [NUMBERS]
+ Set Suite Variable ${PREFIX} ${random}
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
index c1b4953..588ead7 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
@@ -27,41 +27,38 @@ ${ENDPOINT_URL} http://s3g:9878
${BUCKET} generated
${DESTBUCKET} generated1
-
*** Keywords ***
Create Dest Bucket
-
${postfix} = Generate Random String 5 [NUMBERS]
Set Suite Variable ${DESTBUCKET} destbucket-${postfix}
Execute AWSS3APICli create-bucket --bucket ${DESTBUCKET}
-
*** Test Cases ***
Copy Object Happy Scenario
Run Keyword if '${DESTBUCKET}' == 'generated1' Create Dest Bucket
Execute date > /tmp/copyfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key copyobject/f1 --body /tmp/copyfile
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix copyobject/
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/copyobject/f1 --body /tmp/copyfile
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/copyobject/
Should contain ${result} f1
- ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix copyobject/
+ ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/f1
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix ${PREFIX}/copyobject/
Should contain ${result} f1
#copying again will not throw error
- ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix copyobject/
+ ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/f1
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix ${PREFIX}/copyobject/
Should contain ${result} f1
Copy Object Where Bucket is not available
- ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket dfdfdfdfdfnonexistent --key copyobject/f1 --copy-source
${BUCKET}/copyobject/f1 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket dfdfdfdfdfnonexistent --key ${PREFIX}/copyobject/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/f1 255
Should contain ${result}
NoSuchBucket
- ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket ${DESTBUCKET} --key copyobject/f1 --copy-source
dfdfdfdfdfnonexistent/copyobject/f1 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source
dfdfdfdfdfnonexistent/${PREFIX}/copyobject/f1 255
Should contain ${result}
NoSuchBucket
Copy Object Where both source and dest are same with change to storageclass
- ${result} = Execute AWSS3APICli copy-object
--storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1
--copy-source ${DESTBUCKET}/copyobject/f1
+ ${result} = Execute AWSS3APICli copy-object
--storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key
${PREFIX}/copyobject/f1 --copy-source ${DESTBUCKET}/${PREFIX}/copyobject/f1
Should contain ${result} ETag
Copy Object Where Key not available
- ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket ${DESTBUCKET} --key copyobject/f1 --copy-source
${BUCKET}/nonnonexistentkey 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source
${BUCKET}/nonnonexistentkey 255
Should contain ${result} NoSuchKey
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
index b3faf7e..ca3a5a4 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
@@ -29,11 +29,11 @@ ${BUCKET} generated
*** Test Cases ***
Delete file with s3api
Execute date > /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key deletetestapi/f1 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix deletetestapi/
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapi/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/deletetestapi/
Should contain ${result} f1
- ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key deletetestapi/f1
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix deletetestapi/
+ ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapi/f1
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/deletetestapi/
Should not contain ${result} f1
#In case of HTTP 500, the error code is printed out to the console.
Should not contain ${result} 500
@@ -47,24 +47,24 @@ Delete file with s3api, file doesn't exist
Delete dir with s3api
Execute date > /tmp/testfile
- ${result} = Execute AWSS3Cli cp /tmp/testfile
s3://${BUCKET}/deletetestapidir/f1
- ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/deletetestapidir/
+ ${result} = Execute AWSS3Cli cp /tmp/testfile
s3://${BUCKET}/${PREFIX}/deletetestapidir/f1
+ ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapidir/
Should contain ${result} f1
- ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key deletetestapidir/
- ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/deletetestapidir/
+ ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapidir/
+ ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapidir/
Should contain ${result} f1
- ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key deletetestapidir/f1
+ ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapidir/f1
Delete file with s3api, file doesn't exist, prefix of a real file
Execute date > /tmp/testfile
- ${result} = Execute AWSS3Cli cp /tmp/testfile
s3://${BUCKET}/deletetestapiprefix/filefile
- ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/deletetestapiprefix/
+ ${result} = Execute AWSS3Cli cp /tmp/testfile
s3://${BUCKET}/${PREFIX}/deletetestapiprefix/filefile
+ ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapiprefix/
Should contain ${result} filefile
- ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key deletetestapiprefix/file
- ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/deletetestapiprefix/
+ ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapiprefix/file
+ ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapiprefix/
Should contain ${result} filefile
- ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key deletetestapiprefix/filefile
+ ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapiprefix/filefile
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
index 6e22d4c..cc49129 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
@@ -30,20 +30,20 @@ ${BUCKET} generated
Delete file with multi delete
Execute date > /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key multidelete/f1 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key multidelete/f2 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key multidelete/f3 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix multidelete/
- Should contain ${result}
multidelete/f1
- Should contain ${result}
multidelete/f2
- Should contain ${result}
multidelete/f3
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/multidelete/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/multidelete/f2 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/multidelete/f3 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/multidelete/
+ Should contain ${result}
${PREFIX}/multidelete/f1
+ Should contain ${result}
${PREFIX}/multidelete/f2
+ Should contain ${result}
${PREFIX}/multidelete/f3
Should contain ${result} STANDARD
Should not contain ${result}
REDUCED_REDUNDANCY
- ${result} = Execute AWSS3APICli delete-objects --bucket
${BUCKET} --delete
'Objects=[{Key=multidelete/f1},{Key=multidelete/f2},{Key=multidelete/f4}]'
+ ${result} = Execute AWSS3APICli delete-objects --bucket
${BUCKET} --delete
'Objects=[{Key=${PREFIX}/multidelete/f1},{Key=${PREFIX}/multidelete/f2},{Key=${PREFIX}/multidelete/f4}]'
Should not contain ${result} Error
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix multidelete/
- Should not contain ${result}
multidelete/f1
- Should not contain ${result}
multidelete/f2
- Should contain ${result}
multidelete/f3
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/multidelete/
+ Should not contain ${result}
${PREFIX}/multidelete/f1
+ Should not contain ${result}
${PREFIX}/multidelete/f2
+ Should contain ${result}
${PREFIX}/multidelete/f3
Should contain ${result} STANDARD
Should not contain ${result}
REDUCED_REDUNDANCY
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
index 40bcccb..e1896e2 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
@@ -31,22 +31,22 @@ ${BUCKET} generated
Put object to s3
Execute echo "Randomtext" >
/tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key putobject/f1 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix putobject/
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/putobject/
Should contain ${result} f1
Execute touch -f /tmp/zerobyte
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key putobject/zerobyte --body /tmp/zerobyte
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix putobject/
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/putobject/zerobyte --body /tmp/zerobyte
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/putobject/
Should contain ${result} zerobyte
#This test depends on the previous test case. Can't be executes alone
Get object from s3
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 /tmp/testfile.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 /tmp/testfile.result
Compare files /tmp/testfile /tmp/testfile.result
Get Partial object from s3 with both start and endoffset
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 --range bytes=0-4 /tmp/testfile1.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0-4 /tmp/testfile1.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 0-4/11
Should contain ${result}
AcceptRanges
@@ -54,7 +54,7 @@ Get Partial object from s3 with both start and endoffset
${actualData} = Execute cat
/tmp/testfile1.result
Should Be Equal ${expectedData}
${actualData}
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 --range bytes=2-4 /tmp/testfile1.result1
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=2-4 /tmp/testfile1.result1
Should contain ${result}
ContentRange
Should contain ${result}
bytes 2-4/11
Should contain ${result}
AcceptRanges
@@ -63,7 +63,7 @@ Get Partial object from s3 with both start and endoffset
Should Be Equal ${expectedData}
${actualData}
# end offset greater than file size and start with in file length
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 --range bytes=2-1000 /tmp/testfile1.result2
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=2-1000
/tmp/testfile1.result2
Should contain ${result}
ContentRange
Should contain ${result}
bytes 2-10/11
Should contain ${result}
AcceptRanges
@@ -72,12 +72,12 @@ Get Partial object from s3 with both start and endoffset
Should Be Equal ${expectedData}
${actualData}
Get Partial object from s3 with both start and endoffset(start offset and
endoffset is greater than file size)
- ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=10000-10000
/tmp/testfile2.result 255
+ ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range
bytes=10000-10000 /tmp/testfile2.result 255
Should contain ${result}
InvalidRange
Get Partial object from s3 with both start and endoffset(end offset is greater
than file size)
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 --range bytes=0-10000 /tmp/testfile2.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0-10000
/tmp/testfile2.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 0-10/11
Should contain ${result}
AcceptRanges
@@ -86,7 +86,7 @@ Get Partial object from s3 with both start and endoffset(end
offset is greater t
Should Be Equal ${expectedData}
${actualData}
Get Partial object from s3 with only start offset
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 --range bytes=0- /tmp/testfile3.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0- /tmp/testfile3.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 0-10/11
Should contain ${result}
AcceptRanges
@@ -95,7 +95,7 @@ Get Partial object from s3 with only start offset
Should Be Equal ${expectedData}
${actualData}
Get Partial object from s3 with both start and endoffset which are equal
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 --range bytes=0-0 /tmp/testfile4.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0-0 /tmp/testfile4.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 0-0/11
Should contain ${result}
AcceptRanges
@@ -103,7 +103,7 @@ Get Partial object from s3 with both start and endoffset
which are equal
${actualData} = Execute cat
/tmp/testfile4.result
Should Be Equal ${expectedData}
${actualData}
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 --range bytes=4-4 /tmp/testfile5.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=4-4 /tmp/testfile5.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 4-4/11
Should contain ${result}
AcceptRanges
@@ -112,7 +112,7 @@ Get Partial object from s3 with both start and endoffset
which are equal
Should Be Equal ${expectedData}
${actualData}
Get Partial object from s3 to get last n bytes
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 --range bytes=-4 /tmp/testfile6.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=-4 /tmp/testfile6.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 7-10/11
Should contain ${result}
AcceptRanges
@@ -121,7 +121,7 @@ Get Partial object from s3 to get last n bytes
Should Be Equal ${expectedData}
${actualData}
# if end is greater than file length, returns whole file
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 --range bytes=-10000 /tmp/testfile7.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=-10000
/tmp/testfile7.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 0-10/11
Should contain ${result}
AcceptRanges
@@ -130,14 +130,14 @@ Get Partial object from s3 to get last n bytes
Should Be Equal ${expectedData}
${actualData}
Incorrect values for end and start offset
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 --range bytes=-11-10000 /tmp/testfile8.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=-11-10000
/tmp/testfile8.result
Should not contain ${result}
ContentRange
Should contain ${result}
AcceptRanges
${expectedData} = Execute cat /tmp/testfile
${actualData} = Execute cat
/tmp/testfile8.result
Should Be Equal ${expectedData}
${actualData}
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key putobject/f1 --range bytes=11-8 /tmp/testfile9.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=11-8 /tmp/testfile9.result
Should not contain ${result}
ContentRange
Should contain ${result}
AcceptRanges
${expectedData} = Execute cat /tmp/testfile
@@ -145,11 +145,11 @@ Incorrect values for end and start offset
Should Be Equal ${expectedData}
${actualData}
Zero byte file
- ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-0
/tmp/testfile2.result 255
+ ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/zerobyte --range
bytes=0-0 /tmp/testfile2.result 255
Should contain ${result}
InvalidRange
- ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-1
/tmp/testfile2.result 255
+ ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/zerobyte --range
bytes=0-1 /tmp/testfile2.result 255
Should contain ${result}
InvalidRange
- ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-10000
/tmp/testfile2.result 255
+ ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/zerobyte --range
bytes=0-10000 /tmp/testfile2.result 255
Should contain ${result}
InvalidRange
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/s3_compatbility_check.sh
b/hadoop-ozone/dist/src/main/smoketest/s3/s3_compatbility_check.sh
new file mode 100755
index 0000000..3153bd7
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/s3_compatbility_check.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# shellcheck disable=SC2086
+
+set -e
+
+# This script helps to execute S3 robot test against real AWS s3 endpoint
+# To make sure that all of our defined tests cases copies the behavior of AWS
+
+: ${OZONE_TEST_S3_BUCKET1:?Please define test bucket}
+: ${OZONE_TEST_S3_BUCKET2:?Please define second test bucket}
+: ${OZONE_TEST_S3_REGION:?Please define the S3 region for test buckets}
+
+run_robot_test() {
+ TEST_NAME=$1
+ robot \
+ --nostatusrc \
+ -v ENDPOINT_URL:https://s3.$OZONE_TEST_S3_REGION.amazonaws.com \
+ -v BUCKET:$OZONE_TEST_S3_BUCKET1 \
+ -v DESTBUCKET:$OZONE_TEST_S3_BUCKET2 \
+ -v OZONE_S3_SET_CREDENTIALS:false \
+ -o results/$TEST_NAME.xml \
+ $TEST_NAME.robot
+}
+
+mkdir -p results
+
+run_robot_test objectputget
+run_robot_test objectdelete
+run_robot_test objectcopy
+run_robot_test objectmultidelete
+run_robot_test MultipartUpload
+
+rebot --outputdir results/ results/*.xml
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
