This is an automated email from the ASF dual-hosted git repository.
Gargi-jais11 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b18ea9284c4 HDDS-15600. Fix ListObjects response for encoding-type and
empty delimiter (#10586).
b18ea9284c4 is described below
commit b18ea9284c4acd3753409888e77e2baf20931b27
Author: Gargi Jaiswal <[email protected]>
AuthorDate: Wed Jul 15 10:10:59 2026 +0530
HDDS-15600. Fix ListObjects response for encoding-type and empty delimiter
(#10586).
---
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 4 +--
.../hadoop/ozone/om/TestOmMetadataManager.java | 33 +++++++++++++++++++
.../ozone/s3/commontypes/ObjectKeyNameAdapter.java | 3 +-
.../ozone/s3/commontypes/RequestParameters.java | 11 +++++++
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 ++++---
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 11 +++++++
.../s3/commontypes/TestObjectKeyNameAdapter.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 38 ++++++++++++++++++++--
8 files changed, 103 insertions(+), 11 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 69b1ef67bfe..283bb493358 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -1095,11 +1095,11 @@ public ListKeysResult listKeys(String volumeName,
String bucketName,
} else {
// This allows us to seek directly to the first key with the right
prefix.
seekKey = getOzoneKey(volumeName, bucketName,
- StringUtils.isNotBlank(keyPrefix) ? keyPrefix : OM_KEY_PREFIX);
+ StringUtils.isNotEmpty(keyPrefix) ? keyPrefix : OM_KEY_PREFIX);
}
String seekPrefix;
- if (StringUtils.isNotBlank(keyPrefix)) {
+ if (StringUtils.isNotEmpty(keyPrefix)) {
seekPrefix = getOzoneKey(volumeName, bucketName, keyPrefix);
} else {
seekPrefix = getBucketKey(volumeName, bucketName) + OM_KEY_PREFIX;
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
index 17131e0c7db..ec241f9dcb3 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
@@ -1319,4 +1319,37 @@ public void testListKeysSpecialKeyNames() throws
Exception {
assertEquals(keyNames, listedKeys);
}
+
+ @Test
+ public void testListKeysWithWhitespaceAndNewlinePrefix() throws Exception {
+ String volumeName = "volumeA";
+ String bucketName = "bucketA";
+ OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager);
+ addBucketsToCache(volumeName, bucketName);
+
+ String spaceOnly = " ";
+ String spacePrefixed = " x";
+ String doubleSpacePrefixed = " y";
+ String newlinePrefixed = "\nbar";
+ String normalKey = "normal";
+
+ List<String> allKeys = Arrays.asList(
+ spaceOnly, spacePrefixed, doubleSpacePrefixed, newlinePrefixed,
normalKey);
+ for (int i = 0; i < allKeys.size(); i++) {
+ addKeysToOM(volumeName, bucketName, allKeys.get(i), i);
+ }
+
+ List<String> spacePrefixMatches = omMetadataManager.listKeys(volumeName,
bucketName,
+ null, spaceOnly, 100).getKeys().stream()
+ .map(OmKeyInfo::getKeyName)
+ .collect(Collectors.toList());
+ assertEquals(Arrays.asList(spaceOnly, doubleSpacePrefixed, spacePrefixed),
+ spacePrefixMatches);
+
+ List<String> newlinePrefixMatches = omMetadataManager.listKeys(volumeName,
bucketName,
+ null, "\n", 100).getKeys().stream()
+ .map(OmKeyInfo::getKeyName)
+ .collect(Collectors.toList());
+ assertEquals(Collections.singletonList(newlinePrefixed),
newlinePrefixMatches);
+ }
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/ObjectKeyNameAdapter.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/ObjectKeyNameAdapter.java
index 2f3ee0bdcf7..99b66381704 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/ObjectKeyNameAdapter.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/ObjectKeyNameAdapter.java
@@ -41,7 +41,8 @@ public EncodingTypeObject unmarshal(String s) {
public String marshal(EncodingTypeObject s)
throws UnsupportedEncodingException {
if (s.getEncodingType() != null && s.getEncodingType().equals("url")) {
- return StringUtils.replaceEach(S3Utils.urlEncode(s.getName()),
URL_ENCODE_SEARCH_CHARS, URL_ENCODE_REPLACE_CHARS);
+ return StringUtils.replaceEach(S3Utils.s3urlEncode(s.getName()),
+ URL_ENCODE_SEARCH_CHARS, URL_ENCODE_REPLACE_CHARS);
}
return s.getName();
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/RequestParameters.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/RequestParameters.java
index 85ff5fae535..127abe87121 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/RequestParameters.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/RequestParameters.java
@@ -47,6 +47,12 @@ default int getInt(String key, int defaultValue) {
}
}
+ /**
+ * @return true if the query parameter is present, even when its value is
+ * an empty string (eg. {@code delimiter=}).
+ */
+ boolean containsKey(String key);
+
/** Additional methods for tests. */
interface Mutable extends RequestParameters {
@@ -72,6 +78,11 @@ public String get(String key) {
return params.getFirst(key);
}
+ @Override
+ public boolean containsKey(String key) {
+ return params.containsKey(key);
+ }
+
@Override
public void set(String key, String value) {
params.putSingle(key, value);
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index fbe48cbc72c..438ed93670c 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -106,7 +106,8 @@ public Response get(
@Override
Response handleGetRequest(S3RequestContext context, String bucketName)
throws IOException, OS3Exception {
final String continueToken =
queryParams().get(QueryParams.CONTINUATION_TOKEN);
- final String delimiter = queryParams().get(QueryParams.DELIMITER);
+ final String delimiter = queryParams().containsKey(QueryParams.DELIMITER) ?
+ queryParams().get(QueryParams.DELIMITER) : null;
final String encodingType = queryParams().get(QueryParams.ENCODING_TYPE);
final String marker = queryParams().get(QueryParams.MARKER);
int maxKeys = queryParams().getInt(QueryParams.MAX_KEYS, 1000);
@@ -161,15 +162,16 @@ Response handleGetRequest(S3RequestContext context,
String bucketName) throws IO
if (encodingType != null && !encodingType.equals(ENCODING_TYPE)) {
throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, encodingType);
}
-
// If you specify the encoding-type request parameter,should return
- // encoded key name values in the following response elements:
- // Delimiter, Prefix, Key, and StartAfter.
+ // encoded key name values in the following response elements: Delimiter,
Prefix, Key, and StartAfter
//
// For detail refer:
//
https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#AmazonS3-ListObjectsV2-response-EncodingType
ListObjectResponse response = new ListObjectResponse();
- response.setDelimiter(EncodingTypeObject.createNullable(delimiter,
encodingType));
+ // AWS omits Delimiter from the response when the client passes delimiter=
or does not specify delimiter at all.
+ if (StringUtils.isNotEmpty(delimiter)) {
+ response.setDelimiter(EncodingTypeObject.createNullable(delimiter,
encodingType));
+ }
response.setName(bucketName);
response.setPrefix(EncodingTypeObject.createNullable(prefix,
encodingType));
response.setMarker(marker == null ? "" : marker);
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
index 29c556e4d31..4f9fe6c2a27 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
@@ -62,6 +62,17 @@ public static String urlEncode(String str)
return URLEncoder.encode(str, UTF_8.name());
}
+ /**
+ * Percent-encode a string for S3 {@code encoding-type=url} responses.
+ *
+ * <p>Unlike {@link URLEncoder} (application/x-www-form-urlencoded), AWS S3
+ * uses percent-encoding where spaces are {@code %20}, not {@code +}.
+ */
+ public static String s3urlEncode(String str)
+ throws UnsupportedEncodingException {
+ return urlEncode(str).replace("+", "%20");
+ }
+
private S3Utils() {
// no instances
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/commontypes/TestObjectKeyNameAdapter.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/commontypes/TestObjectKeyNameAdapter.java
index f2b10ac4e1b..c98d0a27981 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/commontypes/TestObjectKeyNameAdapter.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/commontypes/TestObjectKeyNameAdapter.java
@@ -31,7 +31,7 @@ public class TestObjectKeyNameAdapter {
public void testEncodeResult() throws Exception {
assertEquals("abc/", getAdapter()
.marshal(EncodingTypeObject.createNullable("abc/", ENCODING_TYPE)));
- assertEquals("a+b+c/", getAdapter()
+ assertEquals("a%20b%20c/", getAdapter()
.marshal(EncodingTypeObject.createNullable("a b c/", ENCODING_TYPE)));
assertEquals("a%2Bb%2Bc/", getAdapter()
.marshal(EncodingTypeObject.createNullable("a+b+c/", ENCODING_TYPE)));
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
index 5d0ad2de0a8..f78c2cc620c 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
@@ -36,6 +36,7 @@
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.s3.commontypes.EncodingTypeObject;
+import org.apache.hadoop.ozone.s3.commontypes.ObjectKeyNameAdapter;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams;
@@ -216,6 +217,7 @@ public void listWithPrefixAndEmptyStrDelimiter()
assertEquals(0, getBucketResponse.getCommonPrefixes().size());
assertEquals(4, getBucketResponse.getContents().size());
+ assertNull(getBucketResponse.getDelimiter());
assertEquals("dir1/",
getBucketResponse.getContents().get(0).getKey().getName());
assertEquals("dir1/dir2/",
@@ -403,7 +405,7 @@ public void testEncodingType() throws IOException,
OS3Exception {
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/";>
...
- <Prefix>data%3D</Prefix>
+ <Prefix>data%3D</Prefix>
<StartAfter>data%3D</StartAfter>
<Delimiter>%3D</Delimiter>
<EncodingType>url</EncodingType>
@@ -417,7 +419,7 @@ public void testEncodingType() throws IOException,
OS3Exception {
</CommonPrefixes>
</ListBucketResult>
- if encodingType == null , the = will not be encoded to "%3D"
+ if encodingType == null , the = will not be encoded to "%3D
* */
OzoneClient ozoneClient =
@@ -568,6 +570,38 @@ public void
testListObjectsRespectsConfiguredMaxKeysLimit() throws Exception {
assertEquals(Integer.parseInt(configuredMaxKeysLimit),
response.getContents().size());
}
+ @Test
+ public void testListObjectsUrlEncodingUsesPercentTwentyForSpaces()
+ throws Exception {
+ OzoneClient client = createClientWithKeys(
+ "foo+1/bar", "foo/bar/xyzzy", "quux ab/thud", "asdf+b");
+ BucketEndpoint endpoint =
newBucketEndpointBuilder().setClient(client).build();
+
+ endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/");
+ endpoint.queryParamsForTest().set(QueryParams.ENCODING_TYPE,
ENCODING_TYPE);
+ ListObjectResponse response = (ListObjectResponse)
endpoint.get("b1").getEntity();
+
+ ObjectKeyNameAdapter adapter = new ObjectKeyNameAdapter();
+ assertEquals("asdf%2Bb",
adapter.marshal(response.getContents().get(0).getKey()));
+ assertEquals(3, response.getCommonPrefixes().size());
+ assertEquals("foo%2B1/",
adapter.marshal(response.getCommonPrefixes().get(0).getPrefix()));
+ assertEquals("foo/",
adapter.marshal(response.getCommonPrefixes().get(1).getPrefix()));
+ assertEquals("quux%20ab/",
adapter.marshal(response.getCommonPrefixes().get(2).getPrefix()));
+ }
+
+ @Test
+ public void testListObjectsOmitsDelimiterWhenEmpty() throws Exception {
+ OzoneClient client = createClientWithKeys("bar", "baz", "cab", "foo");
+ BucketEndpoint endpoint =
newBucketEndpointBuilder().setClient(client).build();
+
+ endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "");
+ ListObjectResponse response = (ListObjectResponse)
endpoint.get("b1").getEntity();
+
+ assertNull(response.getDelimiter());
+ assertEquals(4, response.getContents().size());
+ assertEquals(0, response.getCommonPrefixes().size());
+ }
+
private void assertEncodingTypeObject(
String exceptName, String exceptEncodingType, EncodingTypeObject object)
{
assertEquals(exceptName, object.getName());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
