This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 062ea47faf HDDS-9229. Fix S3A compatibility - dfs -put for FSO bucket.
(#5422)
062ea47faf is described below
commit 062ea47faf8a7ad18368b93a7161cb88efc706ae
Author: ashishkumar50 <[email protected]>
AuthorDate: Thu Nov 2 07:28:33 2023 +0530
HDDS-9229. Fix S3A compatibility - dfs -put for FSO bucket. (#5422)
---
.../dist/src/main/smoketest/s3/objectcopys3a.robot | 49 ++++++++++++++++++++++
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 10 ++++-
2 files changed, 58 insertions(+), 1 deletion(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopys3a.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopys3a.robot
new file mode 100644
index 0000000000..fead57ca31
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopys3a.robot
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation S3 gateway test with aws cli
+Library OperatingSystem
+Library String
+Resource ../commonlib.robot
+Resource commonawslib.robot
+Test Timeout 5 minutes
+Suite Setup Setup s3 tests
+
+*** Variables ***
+${ENDPOINT_URL} http://s3g:9878
+${BUCKET} generated
+
+*** Test Cases ***
+Put object s3a simulation
+ Execute echo "Randomtext" >
/tmp/testfile
+ ${result} = Execute AWSS3APICli and checkrc head-object
--bucket ${BUCKET} --key ${PREFIX}/word.txt 255
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/word.txt/
+ Should Not contain ${result} word.txt
+
+ ${result} = Execute AWSS3APICli and checkrc head-object
--bucket ${BUCKET} --key ${PREFIX}/word.txt._COPYING_ 255
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/word.txt._COPYING_/
+ Should Not contain ${result}
word.txt._COPYING_
+
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/word.txt._COPYING_ --body /tmp/testfile
+ Execute AWSS3APICli and checkrc head-object
--bucket ${BUCKET} --key ${PREFIX}/word.txt._COPYING_ 0
+ Execute AWSS3APICli and checkrc head-object
--bucket ${BUCKET} --key ${PREFIX}/word.txt 255
+ ${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/word.txt/
+ Should Not contain ${result}
word.txt._COPYING_
+
+ ${result} = Execute AWSS3ApiCli copy-object --bucket
${BUCKET} --key ${PREFIX}/word.txt --copy-source
${BUCKET}/${PREFIX}/word.txt._COPYING_
+ Execute AWSS3APICli and checkrc head-object
--bucket ${BUCKET} --key ${PREFIX}/word.txt 0
+ Execute AWSS3APICli delete-object --bucket
${BUCKET} --key ${PREFIX}/word.txt._COPYING_
+ Execute AWSS3APICli and checkrc head-object
--bucket ${BUCKET} --key ${PREFIX}/word.txt._COPYING_ 255
\ No newline at end of file
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 4c0f055ecb..8e96973f82 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -118,6 +118,7 @@ public class BucketEndpoint extends EndpointBase {
Iterator<? extends OzoneKey> ozoneKeyIterator;
ContinueToken decodedToken =
ContinueToken.decodeFromString(continueToken);
+ OzoneBucket bucket;
try {
if (aclMarker != null) {
@@ -153,7 +154,7 @@ public class BucketEndpoint extends EndpointBase {
boolean shallow = listKeysShallowEnabled
&& OZONE_URI_DELIMITER.equals(delimiter);
- OzoneBucket bucket = getBucket(bucketName);
+ bucket = getBucket(bucketName);
ozoneKeyIterator = bucket.listKeys(prefix, prevKey, shallow);
} catch (OMException ex) {
@@ -206,6 +207,13 @@ public class BucketEndpoint extends EndpointBase {
int count = 0;
while (ozoneKeyIterator.hasNext()) {
OzoneKey next = ozoneKeyIterator.next();
+ if (bucket.getBucketLayout().isFileSystemOptimized() &&
+ StringUtils.isNotEmpty(prefix) &&
+ !next.getName().startsWith(prefix)) {
+ // prefix has delimiter but key don't have
+ // example prefix: dir1/ key: dir123
+ continue;
+ }
String relativeKeyName = next.getName().substring(prefix.length());
int depth = StringUtils.countMatches(relativeKeyName, delimiter);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
