This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8587de7 HDDS-6096. S3 copy object fails for source key with special
char (#2912)
8587de7 is described below
commit 8587de72f109b0d7354cc098cfd4ce8070308dd4
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Mon Dec 13 20:28:11 2021 +0100
HDDS-6096. S3 copy object fails for source key with special char (#2912)
---
.../dist/src/main/smoketest/s3/objectcopy.robot | 20 +++++-----
.../dist/src/main/smoketest/s3/objectdelete.robot | 32 ++++++++--------
.../src/main/smoketest/s3/objectmultidelete.robot | 24 ++++++------
.../dist/src/main/smoketest/s3/objectputget.robot | 42 ++++++++++-----------
.../hadoop/ozone/security/AWSV4AuthValidator.java | 10 -----
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 19 ++++++++--
.../s3/signature/AuthorizationV4QueryParser.java | 5 ++-
.../ozone/s3/signature/StringToSignProducer.java | 5 +--
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 44 ++++++++++++++++++++++
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 15 ++++----
10 files changed, 131 insertions(+), 85 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
index 588ead7..21764d6 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
@@ -37,28 +37,28 @@ Create Dest Bucket
Copy Object Happy Scenario
Run Keyword if '${DESTBUCKET}' == 'generated1' Create Dest Bucket
Execute date > /tmp/copyfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/copyobject/f1 --body /tmp/copyfile
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/copyobject/
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/copyobject/key=value/f1 --body /tmp/copyfile
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1
- ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/f1
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix ${PREFIX}/copyobject/
+ ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/key=value/f1
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1
#copying again will not throw error
- ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/f1
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix ${PREFIX}/copyobject/
+ ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/key=value/f1
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1
Copy Object Where Bucket is not available
- ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket dfdfdfdfdfnonexistent --key ${PREFIX}/copyobject/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/f1 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket dfdfdfdfdfnonexistent --key ${PREFIX}/copyobject/key=value/f1
--copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1 255
Should contain ${result}
NoSuchBucket
- ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source
dfdfdfdfdfnonexistent/${PREFIX}/copyobject/f1 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
dfdfdfdfdfnonexistent/${PREFIX}/copyobject/key=value/f1 255
Should contain ${result}
NoSuchBucket
Copy Object Where both source and dest are same with change to storageclass
- ${result} = Execute AWSS3APICli copy-object
--storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key
${PREFIX}/copyobject/f1 --copy-source ${DESTBUCKET}/${PREFIX}/copyobject/f1
+ ${result} = Execute AWSS3APICli copy-object
--storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key
${PREFIX}/copyobject/key=value/f1 --copy-source
${DESTBUCKET}/${PREFIX}/copyobject/key=value/f1
Should contain ${result} ETag
Copy Object Where Key not available
- ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source
${BUCKET}/nonnonexistentkey 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object
--bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/nonnonexistentkey 255
Should contain ${result} NoSuchKey
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
index ce51731..73bf3da 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
@@ -29,12 +29,12 @@ ${BUCKET} generated
*** Test Cases ***
Delete file with s3api
Execute date > /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapi/f1 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/deletetestapi/
- Should contain ${result}
"${PREFIX}/deletetestapi/f1"
- ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapi/f1
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/deletetestapi/
- Should not contain ${result}
"${PREFIX}/deletetestapi/f1"
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapi/key=value/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/deletetestapi/key=value/
+ Should contain ${result}
"${PREFIX}/deletetestapi/key=value/f1"
+ ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapi/key=value/f1
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/deletetestapi/key=value/
+ Should not contain ${result}
"${PREFIX}/deletetestapi/key=value/f1"
Delete file with s3api, file doesn't exist
${result} = Execute AWSS3Cli ls s3://${BUCKET}/
@@ -45,24 +45,24 @@ Delete file with s3api, file doesn't exist
Delete dir with s3api
Execute date > /tmp/testfile
- ${result} = Execute AWSS3Cli cp /tmp/testfile
s3://${BUCKET}/${PREFIX}/deletetestapidir/f1
- ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapidir/
+ ${result} = Execute AWSS3Cli cp /tmp/testfile
s3://${BUCKET}/${PREFIX}/deletetestapidir/key=value/f1
+ ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapidir/key=value/
Should contain ${result} f1
- ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapidir/
- ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapidir/
+ ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapidir/key=value/
+ ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapidir/key=value/
Should contain ${result} f1
- ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapidir/f1
+ ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapidir/key=value/f1
Delete file with s3api, file doesn't exist, prefix of a real file
Execute date > /tmp/testfile
- ${result} = Execute AWSS3Cli cp /tmp/testfile
s3://${BUCKET}/${PREFIX}/deletetestapiprefix/filefile
- ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapiprefix/
+ ${result} = Execute AWSS3Cli cp /tmp/testfile
s3://${BUCKET}/${PREFIX}/deletetestapiprefix/key=value/filefile
+ ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapiprefix/key=value/
Should contain ${result} filefile
- ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapiprefix/file
- ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapiprefix/
+ ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapiprefix/key=value/file
+ ${result} = Execute AWSS3Cli ls
s3://${BUCKET}/${PREFIX}/deletetestapiprefix/key=value/
Should contain ${result} filefile
- ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapiprefix/filefile
+ ${result} = Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/deletetestapiprefix/key=value/filefile
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
index cc49129..37dc2d1 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
@@ -30,20 +30,20 @@ ${BUCKET} generated
Delete file with multi delete
Execute date > /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/multidelete/f1 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/multidelete/f2 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/multidelete/f3 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/multidelete/
- Should contain ${result}
${PREFIX}/multidelete/f1
- Should contain ${result}
${PREFIX}/multidelete/f2
- Should contain ${result}
${PREFIX}/multidelete/f3
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/multidelete/key=value/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/multidelete/key=value/f2 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/multidelete/key=value/f3 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/multidelete/key=value/
+ Should contain ${result}
${PREFIX}/multidelete/key=value/f1
+ Should contain ${result}
${PREFIX}/multidelete/key=value/f2
+ Should contain ${result}
${PREFIX}/multidelete/key=value/f3
Should contain ${result} STANDARD
Should not contain ${result}
REDUCED_REDUNDANCY
- ${result} = Execute AWSS3APICli delete-objects --bucket
${BUCKET} --delete
'Objects=[{Key=${PREFIX}/multidelete/f1},{Key=${PREFIX}/multidelete/f2},{Key=${PREFIX}/multidelete/f4}]'
+ ${result} = Execute AWSS3APICli delete-objects --bucket
${BUCKET} --delete
'Objects=[{Key=${PREFIX}/multidelete/key=value/f1},{Key=${PREFIX}/multidelete/key=value/f2},{Key=${PREFIX}/multidelete/key=value/f4}]'
Should not contain ${result} Error
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/multidelete/
- Should not contain ${result}
${PREFIX}/multidelete/f1
- Should not contain ${result}
${PREFIX}/multidelete/f2
- Should contain ${result}
${PREFIX}/multidelete/f3
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/multidelete/key=value/
+ Should not contain ${result}
${PREFIX}/multidelete/key=value/f1
+ Should not contain ${result}
${PREFIX}/multidelete/key=value/f2
+ Should contain ${result}
${PREFIX}/multidelete/key=value/f3
Should contain ${result} STANDARD
Should not contain ${result}
REDUCED_REDUNDANCY
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
index e1896e2..575e538 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
@@ -31,22 +31,22 @@ ${BUCKET} generated
Put object to s3
Execute echo "Randomtext" >
/tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/putobject/
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/putobject/key=value/
Should contain ${result} f1
Execute touch -f /tmp/zerobyte
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/putobject/zerobyte --body /tmp/zerobyte
- ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/putobject/
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --body /tmp/zerobyte
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/putobject/key=value/
Should contain ${result} zerobyte
#This test depends on the previous test case. Can't be executes alone
Get object from s3
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 /tmp/testfile.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 /tmp/testfile.result
Compare files /tmp/testfile /tmp/testfile.result
Get Partial object from s3 with both start and endoffset
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0-4 /tmp/testfile1.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0-4
/tmp/testfile1.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 0-4/11
Should contain ${result}
AcceptRanges
@@ -54,7 +54,7 @@ Get Partial object from s3 with both start and endoffset
${actualData} = Execute cat
/tmp/testfile1.result
Should Be Equal ${expectedData}
${actualData}
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=2-4 /tmp/testfile1.result1
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=2-4
/tmp/testfile1.result1
Should contain ${result}
ContentRange
Should contain ${result}
bytes 2-4/11
Should contain ${result}
AcceptRanges
@@ -63,7 +63,7 @@ Get Partial object from s3 with both start and endoffset
Should Be Equal ${expectedData}
${actualData}
# end offset greater than file size and start with in file length
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=2-1000
/tmp/testfile1.result2
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=2-1000
/tmp/testfile1.result2
Should contain ${result}
ContentRange
Should contain ${result}
bytes 2-10/11
Should contain ${result}
AcceptRanges
@@ -72,12 +72,12 @@ Get Partial object from s3 with both start and endoffset
Should Be Equal ${expectedData}
${actualData}
Get Partial object from s3 with both start and endoffset(start offset and
endoffset is greater than file size)
- ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range
bytes=10000-10000 /tmp/testfile2.result 255
+ ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range
bytes=10000-10000 /tmp/testfile2.result 255
Should contain ${result}
InvalidRange
Get Partial object from s3 with both start and endoffset(end offset is greater
than file size)
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0-10000
/tmp/testfile2.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0-10000
/tmp/testfile2.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 0-10/11
Should contain ${result}
AcceptRanges
@@ -86,7 +86,7 @@ Get Partial object from s3 with both start and endoffset(end
offset is greater t
Should Be Equal ${expectedData}
${actualData}
Get Partial object from s3 with only start offset
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0- /tmp/testfile3.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0-
/tmp/testfile3.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 0-10/11
Should contain ${result}
AcceptRanges
@@ -95,7 +95,7 @@ Get Partial object from s3 with only start offset
Should Be Equal ${expectedData}
${actualData}
Get Partial object from s3 with both start and endoffset which are equal
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0-0 /tmp/testfile4.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=0-0
/tmp/testfile4.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 0-0/11
Should contain ${result}
AcceptRanges
@@ -103,7 +103,7 @@ Get Partial object from s3 with both start and endoffset
which are equal
${actualData} = Execute cat
/tmp/testfile4.result
Should Be Equal ${expectedData}
${actualData}
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=4-4 /tmp/testfile5.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=4-4
/tmp/testfile5.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 4-4/11
Should contain ${result}
AcceptRanges
@@ -112,7 +112,7 @@ Get Partial object from s3 with both start and endoffset
which are equal
Should Be Equal ${expectedData}
${actualData}
Get Partial object from s3 to get last n bytes
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=-4 /tmp/testfile6.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=-4
/tmp/testfile6.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 7-10/11
Should contain ${result}
AcceptRanges
@@ -121,7 +121,7 @@ Get Partial object from s3 to get last n bytes
Should Be Equal ${expectedData}
${actualData}
# if end is greater than file length, returns whole file
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=-10000
/tmp/testfile7.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=-10000
/tmp/testfile7.result
Should contain ${result}
ContentRange
Should contain ${result}
bytes 0-10/11
Should contain ${result}
AcceptRanges
@@ -130,14 +130,14 @@ Get Partial object from s3 to get last n bytes
Should Be Equal ${expectedData}
${actualData}
Incorrect values for end and start offset
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=-11-10000
/tmp/testfile8.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=-11-10000
/tmp/testfile8.result
Should not contain ${result}
ContentRange
Should contain ${result}
AcceptRanges
${expectedData} = Execute cat /tmp/testfile
${actualData} = Execute cat
/tmp/testfile8.result
Should Be Equal ${expectedData}
${actualData}
- ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=11-8 /tmp/testfile9.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 --range bytes=11-8
/tmp/testfile9.result
Should not contain ${result}
ContentRange
Should contain ${result}
AcceptRanges
${expectedData} = Execute cat /tmp/testfile
@@ -145,11 +145,11 @@ Incorrect values for end and start offset
Should Be Equal ${expectedData}
${actualData}
Zero byte file
- ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/zerobyte --range
bytes=0-0 /tmp/testfile2.result 255
+ ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte
--range bytes=0-0 /tmp/testfile2.result 255
Should contain ${result}
InvalidRange
- ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/zerobyte --range
bytes=0-1 /tmp/testfile2.result 255
+ ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte
--range bytes=0-1 /tmp/testfile2.result 255
Should contain ${result}
InvalidRange
- ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/zerobyte --range
bytes=0-10000 /tmp/testfile2.result 255
- Should contain ${result}
InvalidRange
\ No newline at end of file
+ ${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte
--range bytes=0-10000 /tmp/testfile2.result 255
+ Should contain ${result}
InvalidRange
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java
index ca0bc18..1b526e7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java
@@ -24,8 +24,6 @@ import org.slf4j.LoggerFactory;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.security.MessageDigest;
@@ -45,14 +43,6 @@ final class AWSV4AuthValidator {
private AWSV4AuthValidator() {
}
- private static String urlDecode(String str) {
- try {
- return URLDecoder.decode(str, StandardCharsets.UTF_8.name());
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
- }
-
public static String hash(String payload) throws NoSuchAlgorithmException {
MessageDigest md = MessageDigest.getInstance("SHA-256");
md.update(payload.getBytes(StandardCharsets.UTF_8));
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 52b9c49..d2a45b1 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -41,6 +41,7 @@ import javax.ws.rs.core.StreamingOutput;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
import java.text.ParseException;
import java.time.Instant;
import java.time.ZoneId;
@@ -90,6 +91,7 @@ import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM
import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_DEFAULT;
import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_KEY;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ENTITY_TOO_SMALL;
+import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_REQUEST;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_UPLOAD;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED;
@@ -102,6 +104,8 @@ import static
org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_IF_UNMODIFIED
import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER_SUPPORTED_UNIT;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.urlDecode;
+
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -828,14 +832,21 @@ public class ObjectEndpoint extends EndpointBase {
}
int pos = header.indexOf('/');
if (pos == -1) {
- OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
- .INVALID_ARGUMENT, header);
+ OS3Exception ex = S3ErrorTable.newError(INVALID_ARGUMENT, header);
ex.setErrorMessage("Copy Source must mention the source bucket and " +
"key: sourcebucket/sourcekey");
throw ex;
}
- return Pair.of(header.substring(0, pos), header.substring(pos + 1));
+ try {
+ String bucket = header.substring(0, pos);
+ String key = urlDecode(header.substring(pos + 1));
+ return Pair.of(bucket, key);
+ } catch (UnsupportedEncodingException e) {
+ OS3Exception ex = S3ErrorTable.newError(INVALID_ARGUMENT, header);
+ ex.setErrorMessage("Copy Source header could not be url-decoded");
+ throw ex;
+ }
}
private static S3StorageType toS3StorageType(String storageType)
@@ -843,7 +854,7 @@ public class ObjectEndpoint extends EndpointBase {
try {
return S3StorageType.valueOf(storageType);
} catch (IllegalArgumentException ex) {
- throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT,
+ throw S3ErrorTable.newError(INVALID_ARGUMENT,
storageType);
}
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java
index fbf4594..40f9542 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.s3.signature;
import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
import java.time.ZonedDateTime;
import java.util.Map;
@@ -27,6 +26,8 @@ import
org.apache.hadoop.ozone.s3.signature.SignatureInfo.Version;
import com.google.common.annotations.VisibleForTesting;
import static java.time.temporal.ChronoUnit.SECONDS;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.urlDecode;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,7 +64,7 @@ public class AuthorizationV4QueryParser implements
SignatureParser {
Credential credential =
null;
try {
- credential = new Credential(URLDecoder.decode(rawCredential, "UTF-8"));
+ credential = new Credential(urlDecode(rawCredential));
} catch (UnsupportedEncodingException e) {
throw new IllegalArgumentException(
"X-Amz-Credential is not proper URL encoded");
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
index 3202a96..9dfa13e 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
@@ -23,7 +23,6 @@ import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
-import java.net.URLEncoder;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
@@ -42,6 +41,7 @@ import java.util.regex.Pattern;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import
org.apache.hadoop.ozone.s3.signature.AWSSignatureProcessor.LowerCaseKeyStringMap;
+import org.apache.hadoop.ozone.s3.util.S3Utils;
import org.apache.hadoop.util.StringUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -247,8 +247,7 @@ public final class StringToSignProducer {
private static String urlEncode(String str) {
try {
-
- return URLEncoder.encode(str, UTF_8.name())
+ return S3Utils.urlEncode(str)
.replaceAll("\\+", "%20")
.replaceAll("%7E", "~");
} catch (UnsupportedEncodingException e) {
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
new file mode 100644
index 0000000..c9e16d1
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3.util;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Utilities.
+ */
+public final class S3Utils {
+
+ public static String urlDecode(String str)
+ throws UnsupportedEncodingException {
+ return URLDecoder.decode(str, UTF_8.name());
+ }
+
+ public static String urlEncode(String str)
+ throws UnsupportedEncodingException {
+ return URLEncoder.encode(str, UTF_8.name());
+ }
+
+ private S3Utils() {
+ // no instances
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index 95b1244..cf44854 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -43,6 +43,7 @@ import org.mockito.Mockito;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.urlEncode;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.when;
@@ -53,9 +54,9 @@ import static org.mockito.Mockito.when;
public class TestObjectPut {
public static final String CONTENT = "0123456789";
private String bucketName = "b1";
- private String keyName = "key1";
+ private String keyName = "key=value/1";
private String destBucket = "b2";
- private String destkey = "key2";
+ private String destkey = "key=value/2";
private String nonexist = "nonexist";
private OzoneClient clientStub;
private ObjectEndpoint objectEndpoint;
@@ -152,7 +153,7 @@ public class TestObjectPut {
// Add copy header, and then call put
when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
- bucketName + "/" + keyName);
+ bucketName + "/" + urlEncode(keyName));
response = objectEndpoint.put(destBucket, destkey, CONTENT.length(), 1,
null, body);
@@ -178,7 +179,7 @@ public class TestObjectPut {
// source bucket not found
try {
when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
- nonexist + "/" + keyName);
+ nonexist + "/" + urlEncode(keyName));
objectEndpoint.put(destBucket, destkey, CONTENT.length(), 1, null,
body);
fail("test copy object failed");
@@ -189,7 +190,7 @@ public class TestObjectPut {
// dest bucket not found
try {
when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
- bucketName + "/" + keyName);
+ bucketName + "/" + urlEncode(keyName));
objectEndpoint.put(nonexist, destkey, CONTENT.length(), 1, null, body);
fail("test copy object failed");
} catch (OS3Exception ex) {
@@ -199,7 +200,7 @@ public class TestObjectPut {
//Both source and dest bucket not found
try {
when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
- nonexist + "/" + keyName);
+ nonexist + "/" + urlEncode(keyName));
objectEndpoint.put(nonexist, destkey, CONTENT.length(), 1, null, body);
fail("test copy object failed");
} catch (OS3Exception ex) {
@@ -209,7 +210,7 @@ public class TestObjectPut {
// source key not found
try {
when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
- bucketName + "/" + nonexist);
+ bucketName + "/" + urlEncode(nonexist));
objectEndpoint.put("nonexistent", keyName, CONTENT.length(), 1,
null, body);
fail("test copy object failed");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
