This is an automated email from the ASF dual-hosted git repository.
Russole pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new f5f2e14f3be HDDS-14760. Fix CompleteMultipartUpload rejecting chunked
request body as empty (#10461)
f5f2e14f3be is described below
commit f5f2e14f3be9cbf4e57edfebfa719a3a6498b06d
Author: KUAN-HAO HUANG <[email protected]>
AuthorDate: Thu Jul 2 18:51:37 2026 +0800
HDDS-14760. Fix CompleteMultipartUpload rejecting chunked request body as
empty (#10461)
---
.../src/main/smoketest/s3/MultipartUpload.robot | 34 +++++++++++-
.../src/main/smoketest/s3/presigned_url_helper.py | 48 +++++++++++++++++
...CompleteMultipartUploadRequestUnmarshaller.java | 12 ++++-
...CompleteMultipartUploadRequestUnmarshaller.java | 63 ++++++++++++++++++++++
4 files changed, 154 insertions(+), 3 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
index 18315fb12fb..5e799a7c772 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
@@ -18,6 +18,7 @@ Documentation S3 gateway test with aws cli
Library OperatingSystem
Library String
Library DateTime
+Library ./presigned_url_helper.py
Resource ../commonlib.robot
Resource commonawslib.robot
Resource mpu_lib.robot
@@ -61,6 +62,38 @@ Test Multipart Upload With Adjusted Length
Perform Multipart Upload ${BUCKET}
multipart/adjusted_length_${PREFIX} /tmp/part1 /tmp/part2
Verify Multipart Upload ${BUCKET}
multipart/adjusted_length_${PREFIX} /tmp/part1 /tmp/part2
+Test Multipart Upload Complete With Chunked Transfer Encoding
+ [Documentation] Regression test for HDDS-14760. When
CompleteMultipartUpload
+ ... is sent with chunked transfer encoding (no
Content-Length, as
+ ... e.g. the AWS C++ SDK does with Expect: 100-continue),
it must
+ ... not be rejected with "You must specify at least one
part".
+ ${access_key} = Execute aws configure get aws_access_key_id
+ ${secret_key} = Execute aws configure get aws_secret_access_key
+ ${key} = Set Variable ${PREFIX}/chunkedCompleteKey
+ ${uploadID} = Set Variable ${EMPTY}
+ ${uploadID} = Initiate MPU ${BUCKET} ${key}
+ ${eTag1} = Upload MPU part ${BUCKET} ${key} ${uploadID}
1 /tmp/part1
+ ${eTag2} = Upload MPU part ${BUCKET} ${key} ${uploadID}
2 /tmp/part2
+ ${body} = Catenate SEPARATOR=
+ ... <CompleteMultipartUpload>
+ ...
<Part><PartNumber>1</PartNumber><ETag>${eTag1}</ETag></Part>
+ ...
<Part><PartNumber>2</PartNumber><ETag>${eTag2}</ETag></Part>
+ ... </CompleteMultipartUpload>
+ Create File /tmp/${PREFIX}-complete.xml ${body}
+ ${presigned_url} = Generate Presigned Complete Multipart Upload Url
${access_key} ${secret_key} ${BUCKET} ${key} ${uploadID}
us-east-1 3600 ${ENDPOINT_URL}
+ ${result} = Execute curl -sS -v -X POST -H "Transfer-Encoding:
chunked" -H "Content-Length:" -H "Content-Type: application/xml" --data-binary
@/tmp/${PREFIX}-complete.xml "${presigned_url}" 2>&1
+ Should Contain ${result} > Transfer-Encoding: chunked
+ Should Not Contain ${result} > Content-Length:
+ # A success response carries <CompleteMultipartUploadResult>/<ETag>; the
+ # error response would instead contain the "must specify at least one part"
+ # message (the bucket/key alone are not success discriminators, since the
+ # key also appears in the <Resource> element of an error response).
+ Should Not Contain ${result} must specify at least one part
+ Should Contain ${result} CompleteMultipartUploadResult
+ Should Contain ${result} ETag
+ [Teardown] Run Keywords Remove File
/tmp/${PREFIX}-complete.xml
+ ... AND Run Keyword And Ignore Error Abort MPU
${BUCKET} ${key} ${uploadID}
+
Overwrite Empty File
Execute touch ${TEMP_DIR}/empty
Execute AWSS3Cli cp ${TEMP_DIR}/empty
s3://${BUCKET}/empty_file_${PREFIX}
@@ -456,4 +489,3 @@ Test Multipart Upload Part with wrong Content-MD5 header
# Abort the multipart upload (cleanup)
Abort MPU ${BUCKET}
${PREFIX}/mpu/md5test/key2 ${uploadID}
-
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/presigned_url_helper.py
b/hadoop-ozone/dist/src/main/smoketest/s3/presigned_url_helper.py
index 8b5cef974f5..4a68968142a 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/presigned_url_helper.py
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/presigned_url_helper.py
@@ -67,6 +67,54 @@ def generate_presigned_put_object_url(
raise Exception(f"Failed to generate presigned URL: {str(e)}")
+def generate_presigned_complete_multipart_upload_url(
+ aws_access_key_id=None,
+ aws_secret_access_key=None,
+ bucket_name=None,
+ object_key=None,
+ upload_id=None,
+ region_name='us-east-1',
+ expiration=3600,
+ endpoint_url=None,
+):
+ """
+ Generate a presigned URL for CompleteMultipartUpload. The request body
+ (the XML list of parts) is not part of the signature (UNSIGNED-PAYLOAD),
+ so the caller can stream it, e.g. with chunked transfer encoding.
+ """
+ try:
+ import boto3
+
+ client_args = {
+ 'service_name': 's3',
+ 'region_name': region_name,
+ }
+
+ if aws_access_key_id and aws_secret_access_key:
+ client_args['aws_access_key_id'] = aws_access_key_id
+ client_args['aws_secret_access_key'] = aws_secret_access_key
+
+ if endpoint_url:
+ client_args['endpoint_url'] = endpoint_url
+
+ s3_client = boto3.client(**client_args)
+
+ presigned_url = s3_client.generate_presigned_url(
+ ClientMethod='complete_multipart_upload',
+ Params={
+ 'Bucket': bucket_name,
+ 'Key': object_key,
+ 'UploadId': upload_id,
+ },
+ ExpiresIn=expiration
+ )
+
+ return presigned_url
+
+ except Exception as e:
+ raise Exception(f"Failed to generate presigned URL: {str(e)}")
+
+
def compute_sha256_file(path):
"""Compute SHA256 hex digest for the entire file content at path."""
with open(path, 'rb') as f:
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequestUnmarshaller.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequestUnmarshaller.java
index afee2747677..462ba3ff14e 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequestUnmarshaller.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequestUnmarshaller.java
@@ -23,6 +23,7 @@
import java.io.IOException;
import java.io.InputStream;
+import java.io.PushbackInputStream;
import java.lang.annotation.Annotation;
import java.lang.reflect.Type;
import javax.inject.Singleton;
@@ -50,11 +51,18 @@ public CompleteMultipartUploadRequest readFrom(
MultivaluedMap<String, String> multivaluedMap,
InputStream inputStream) throws WebApplicationException {
try {
- if (inputStream.available() == 0) {
+ // Detect an empty request body by trying to read a single byte rather
+ // than relying on InputStream#available(), which may return 0 even when
+ // the body is not yet buffered (e.g. with Expect: 100-continue). See
+ // HDDS-14760.
+ PushbackInputStream pushbackStream = new
PushbackInputStream(inputStream);
+ int firstByte = pushbackStream.read();
+ if (firstByte == -1) {
throw wrapOS3Exception(newError(INVALID_REQUEST)
.withMessage("You must specify at least one part"));
}
- return super.readFrom(aClass, type, annotations, mediaType,
multivaluedMap, inputStream);
+ pushbackStream.unread(firstByte);
+ return super.readFrom(aClass, type, annotations, mediaType,
multivaluedMap, pushbackStream);
} catch (IOException e) {
throw wrapOS3Exception(newError(INVALID_REQUEST, e)
.withMessage(e.getMessage()));
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCompleteMultipartUploadRequestUnmarshaller.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCompleteMultipartUploadRequestUnmarshaller.java
index d3615bb5c36..243b84b464f 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCompleteMultipartUploadRequestUnmarshaller.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCompleteMultipartUploadRequestUnmarshaller.java
@@ -19,14 +19,22 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayInputStream;
+import java.io.FilterInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
+import javax.ws.rs.WebApplicationException;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.junit.jupiter.api.Test;
@@ -58,6 +66,59 @@ public void fromStreamWithNamespace() throws IOException {
checkContent(completeMultipartUploadRequest);
}
+ @Test
+ public void fromStreamWhereAvailableReturnsZero() throws IOException {
+ // Simulates a request body stream (e.g. with Expect: 100-continue) where
+ // InputStream#available() returns 0 even though the body has not been
+ // fully buffered yet. See HDDS-14760.
+ InputStream inputBody = new FilterInputStream(new ByteArrayInputStream(
+ ("<CompleteMultipartUpload xmlns=\"" + S3Consts.S3_XML_NAMESPACE +
"\">" +
+ "<Part><ETag>" + part1 + "</ETag><PartNumber>1" +
+ "</PartNumber></Part><Part><ETag>" + part2 +
+ "</ETag><PartNumber>2</PartNumber></Part>" +
+ "</CompleteMultipartUpload>").getBytes(UTF_8))) {
+ @Override
+ public int available() {
+ return 0;
+ }
+ };
+
+ //WHEN
+ CompleteMultipartUploadRequest completeMultipartUploadRequest =
+ new CompleteMultipartUploadRequestUnmarshaller()
+ .readFrom(null, null, null, null, null, inputBody);
+
+ //THEN
+ checkContent(completeMultipartUploadRequest);
+ }
+
+ @Test
+ public void emptyBodyIsRejectedAsInvalidRequest() {
+ InputStream emptyBody = new ByteArrayInputStream(new byte[0]);
+
+ WebApplicationException ex = assertThrows(WebApplicationException.class,
+ () -> new CompleteMultipartUploadRequestUnmarshaller()
+ .readFrom(null, null, null, null, null, emptyBody));
+
+ // Assert on the stable S3 error code rather than the human-readable
message.
+ OS3Exception cause = assertInstanceOf(OS3Exception.class, ex.getCause());
+ assertEquals(S3ErrorTable.INVALID_REQUEST.getCode(), cause.getCode());
+ }
+
+ @Test
+ public void wellFormedBodyWithZeroPartsReturnsEmptyList() throws IOException
{
+ // The fix lets a non-empty body through, so a well-formed request with no
+ // <Part> elements (a shape a chunked SDK can send) is parsed into an empty
+ // part list; rejecting the empty list is deferred to the endpoint.
+ ByteArrayInputStream inputBody = new ByteArrayInputStream(
+ "<CompleteMultipartUpload></CompleteMultipartUpload>".getBytes(UTF_8));
+
+ CompleteMultipartUploadRequest request = unmarshall(inputBody);
+
+ assertNotNull(request);
+ assertTrue(request.getPartList().isEmpty());
+ }
+
@Test
public void fromStreamWithoutNamespace() throws IOException {
//GIVEN
@@ -85,7 +146,9 @@ private void checkContent(CompleteMultipartUploadRequest
request) {
request.getPartList();
assertEquals(part1, parts.get(0).getETag());
+ assertEquals(1, parts.get(0).getPartNumber());
assertEquals(part2, parts.get(1).getETag());
+ assertEquals(2, parts.get(1).getPartNumber());
}
private CompleteMultipartUploadRequest unmarshall(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
