This is an automated email from the ASF dual-hosted git repository.
smengcl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1d6a5f48aa2 HDDS-15739. Fix ListObjectsV2 continuation-token: wrong
XML element name and empty-token handling (#10668)
1d6a5f48aa2 is described below
commit 1d6a5f48aa28c5461f5ae5d987e91a6e47bc7d2a
Author: KUAN-HAO HUANG <[email protected]>
AuthorDate: Fri Jul 10 11:03:44 2026 +0800
HDDS-15739. Fix ListObjectsV2 continuation-token: wrong XML element name
and empty-token handling (#10668)
---
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 10 ++-
.../ozone/s3/endpoint/ListObjectResponse.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 94 ++++++++++++++++++++++
3 files changed, 102 insertions(+), 4 deletions(-)
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 383dc753f19..fbe48cbc72c 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -114,7 +114,11 @@ Response handleGetRequest(S3RequestContext context, String
bucketName) throws IO
String startAfter = queryParams().get(QueryParams.START_AFTER);
Iterator<? extends OzoneKey> ozoneKeyIterator = null;
- ContinueToken decodedToken = ContinueToken.decodeFromString(continueToken);
+ // AWS S3 treats an empty continuation-token as no token: list from the
+ // start and echo the empty token back (see setContinueToken below).
+ ContinueToken decodedToken =
+ (continueToken == null || continueToken.isEmpty())
+ ? null : ContinueToken.decodeFromString(continueToken);
OzoneBucket bucket = null;
try {
@@ -127,7 +131,7 @@ Response handleGetRequest(S3RequestContext context, String
bucketName) throws IO
// If continuation token and start after both are provided, then we
// ignore start After
- String prevKey = continueToken != null ? decodedToken.getLastKey()
+ String prevKey = decodedToken != null ? decodedToken.getLastKey()
: startAfter;
// If shallow is true, only list immediate children
@@ -175,7 +179,7 @@ Response handleGetRequest(S3RequestContext context, String
bucketName) throws IO
response.setContinueToken(continueToken);
response.setStartAfter(EncodingTypeObject.createNullable(startAfter,
encodingType));
- String prevDir = continueToken != null ? decodedToken.getLastDir() : null;
+ String prevDir = decodedToken != null ? decodedToken.getLastDir() : null;
String lastKey = null;
int count = 0;
if (maxKeys > 0) {
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
index 027ac75bc4c..bc27e424703 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
@@ -69,7 +69,7 @@ public class ListObjectResponse {
@XmlElement(name = "NextMarker")
private String nextMarker;
- @XmlElement(name = "continueToken")
+ @XmlElement(name = "ContinuationToken")
private String continueToken;
@XmlElement(name = "Contents")
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
index d2a47c0c586..5d0ad2de0a8 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
@@ -28,7 +28,9 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
+import java.io.StringWriter;
import java.util.stream.IntStream;
+import javax.xml.bind.JAXB;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
@@ -572,6 +574,98 @@ private void assertEncodingTypeObject(
assertEquals(exceptEncodingType, object.getEncodingType());
}
+ /**
+ * An empty continuation token must be treated as no token: list from the
+ * start, not truncated, and echo the empty token back (AWS S3 semantics).
+ */
+ @Test
+ public void listWithEmptyContinuationToken() throws OS3Exception,
IOException {
+ OzoneClient ozoneClient = createClientWithKeys("bar", "baz", "foo",
"quxx");
+ BucketEndpoint endpoint =
newBucketEndpointBuilder().setClient(ozoneClient).build();
+
+ endpoint.queryParamsForTest().set(QueryParams.PREFIX, "");
+ endpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN, "");
+ ListObjectResponse response = (ListObjectResponse)
endpoint.get("b1").getEntity();
+
+ assertFalse(response.isTruncated());
+ assertEquals(4, response.getContents().size());
+ // Echoed back verbatim (empty), so botocore populates
ContinuationToken=''.
+ assertEquals("", response.getContinueToken());
+ }
+
+ /**
+ * A supplied continuation token must be echoed back in the response so that
+ * clients can read response['ContinuationToken'].
+ */
+ @Test
+ public void listEchoesContinuationToken() throws OS3Exception, IOException {
+ OzoneClient ozoneClient = createClientWithKeys("bar", "baz", "foo",
"quxx");
+ BucketEndpoint endpoint =
newBucketEndpointBuilder().setClient(ozoneClient).build();
+
+ endpoint.queryParamsForTest().set(QueryParams.PREFIX, "");
+ endpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, 1);
+ ListObjectResponse first = (ListObjectResponse)
endpoint.get("b1").getEntity();
+ assertTrue(first.isTruncated());
+ String token = first.getNextToken();
+ assertNotNull(token);
+
+ endpoint.queryParamsForTest().unset(QueryParams.MAX_KEYS);
+ endpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN, token);
+ ListObjectResponse second = (ListObjectResponse)
endpoint.get("b1").getEntity();
+
+ assertFalse(second.isTruncated());
+ // The request continuation token is echoed back verbatim.
+ assertEquals(token, second.getContinueToken());
+ assertEquals(3, second.getContents().size());
+ }
+
+ /**
+ * With both StartAfter and a continuation token, the token drives the
+ * listing position while both StartAfter and ContinuationToken are echoed.
+ */
+ @Test
+ public void listContinuationTokenWithStartAfter()
+ throws OS3Exception, IOException {
+ OzoneClient ozoneClient = createClientWithKeys("bar", "baz", "foo",
"quxx");
+ BucketEndpoint endpoint =
newBucketEndpointBuilder().setClient(ozoneClient).build();
+
+ endpoint.queryParamsForTest().set(QueryParams.PREFIX, "");
+ endpoint.queryParamsForTest().set(QueryParams.START_AFTER, "bar");
+ endpoint.queryParamsForTest().setInt(QueryParams.MAX_KEYS, 1);
+ ListObjectResponse first = (ListObjectResponse)
endpoint.get("b1").getEntity();
+ assertTrue(first.isTruncated());
+ String token = first.getNextToken();
+ assertNotNull(token);
+
+ endpoint.queryParamsForTest().unset(QueryParams.MAX_KEYS);
+ endpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN, token);
+ ListObjectResponse second = (ListObjectResponse)
endpoint.get("b1").getEntity();
+
+ assertFalse(second.isTruncated());
+ assertEquals(token, second.getContinueToken());
+ assertEquals("bar", second.getStartAfter().getName());
+ assertEquals(2, second.getContents().size());
+ }
+
+ /**
+ * The echoed request continuation token must be serialized as the AWS S3
+ * element name {@code ContinuationToken} (not {@code continueToken}).
+ */
+ @Test
+ public void continuationTokenXmlElementName() throws Exception {
+ ListObjectResponse response = new ListObjectResponse();
+ response.setContinueToken("token-value");
+
+ StringWriter writer = new StringWriter();
+ JAXB.marshal(response, writer);
+ String xml = writer.toString();
+
+
assertTrue(xml.contains("<ContinuationToken>token-value</ContinuationToken>"),
+ "expected <ContinuationToken> element, got: " + xml);
+ assertFalse(xml.contains("continueToken"),
+ "response must not use the non-AWS <continueToken> element");
+ }
+
private OzoneClient createClientWithKeys(String... keys) throws IOException {
OzoneClient client = new OzoneClientStub();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
