This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c3899facba HDDS-9123. Make S3G MPU invalid part Exception Reporting
More detailed (#5153)
c3899facba is described below
commit c3899facba0cfe520c36624b72feff8fc4c314bd
Author: XiChen <[email protected]>
AuthorDate: Wed Aug 16 17:44:46 2023 +0800
HDDS-9123. Make S3G MPU invalid part Exception Reporting More detailed
(#5153)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 8 ++++--
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 20 +++++++++++++++
.../client/rpc/TestOzoneRpcClientAbstract.java | 29 ++++++++++++++++++++++
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 5 ++++
5 files changed, 64 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 4d123e9c0d..6008ef8642 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -377,6 +377,10 @@ public final class OzoneConsts {
// For Multipart upload
public static final int OM_MULTIPART_MIN_SIZE = 5 * 1024 * 1024;
+ // refer to :
+ // https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
+ public static final int MAXIMUM_NUMBER_OF_PARTS_PER_UPLOAD = 10000;
+
// GRPC block token metadata header and context key
public static final String OZONE_BLOCK_TOKEN = "blocktoken";
public static final Context.Key<UserGroupInformation> UGI_CTX_KEY =
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 0624e84dae..15ff53c200 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -167,6 +167,7 @@ import static
org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY;
+import static
org.apache.hadoop.ozone.OzoneConsts.MAXIMUM_NUMBER_OF_PARTS_PER_UPLOAD;
import static org.apache.hadoop.ozone.OzoneConsts.OLD_QUOTA_DEFAULT;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_MAXIMUM_ACCESS_ID_LENGTH;
import static
org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ;
@@ -1738,8 +1739,11 @@ public class RpcClient implements ClientProtocol {
HddsClientUtils.verifyKeyName(keyName);
}
HddsClientUtils.checkNotNull(keyName, uploadID);
- Preconditions.checkArgument(partNumber > 0 && partNumber <= 10000, "Part "
+
- "number should be greater than zero and less than or equal to 10000");
+ if (partNumber <= 0 || partNumber > MAXIMUM_NUMBER_OF_PARTS_PER_UPLOAD) {
+ throw new OMException("Part number must be an integer between 1 and "
+ + MAXIMUM_NUMBER_OF_PARTS_PER_UPLOAD + ", inclusive",
+ OMException.ResultCodes.INVALID_PART);
+ }
Preconditions.checkArgument(size >= 0, "size should be greater than or " +
"equal to zero");
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
index 5ce2f8a1ed..f89aa85d15 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
@@ -406,6 +406,26 @@ public class TestOzoneClientMultipartUploadWithFSO {
() -> completeMultipartUpload(bucket, keyName, uploadID, partsMap));
}
+ @Test
+ public void testMultipartPartNumberExceedingAllowedRange() throws Exception {
+ String uploadID = initiateMultipartUpload(bucket, keyName,
+ RATIS, ONE);
+ byte[] data = "data".getBytes(UTF_8);
+
+ // Multipart part number must be an integer between 1 and 10000. So the
+ // part number 1, 5000, 10000 will succeed,
+ // the part number 0, 10001 will fail.
+ bucket.createMultipartKey(keyName, data.length, 1, uploadID);
+ bucket.createMultipartKey(keyName, data.length, 5000, uploadID);
+ bucket.createMultipartKey(keyName, data.length, 10000, uploadID);
+ OzoneTestUtils.expectOmException(OMException.ResultCodes.INVALID_PART,
+ () -> bucket.createMultipartKey(
+ keyName, data.length, 0, uploadID));
+ OzoneTestUtils.expectOmException(OMException.ResultCodes.INVALID_PART,
+ () -> bucket.createMultipartKey(
+ keyName, data.length, 10001, uploadID));
+ }
+
@Test
public void testCommitPartAfterCompleteUpload() throws Exception {
String parentDir = "a/b/c/d/";
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index 8cf2f9cea9..4bd07fbcad 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -2972,6 +2972,35 @@ public abstract class TestOzoneRpcClientAbstract {
() -> completeMultipartUpload(bucket, keyName, uploadID, partsMap));
}
+ @Test
+ public void testMultipartPartNumberExceedingAllowedRange() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String keyName = UUID.randomUUID().toString();
+ String sampleData = "sample Value";
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ OmMultipartInfo multipartInfo = bucket.initiateMultipartUpload(keyName);
+ assertNotNull(multipartInfo);
+ String uploadID = multipartInfo.getUploadID();
+
+ // Multipart part number must be an integer between 1 and 10000. So the
+ // part number 1, 5000, 10000 will succeed,
+ // the part number 0, 10001 will fail.
+ bucket.createMultipartKey(keyName, sampleData.length(), 1, uploadID);
+ bucket.createMultipartKey(keyName, sampleData.length(), 5000, uploadID);
+ bucket.createMultipartKey(keyName, sampleData.length(), 10000, uploadID);
+ OzoneTestUtils.expectOmException(ResultCodes.INVALID_PART, () ->
+ bucket.createMultipartKey(
+ keyName, sampleData.length(), 0, uploadID));
+ OzoneTestUtils.expectOmException(ResultCodes.INVALID_PART, () ->
+ bucket.createMultipartKey(
+ keyName, sampleData.length(), 10001, uploadID));
+ }
+
@Test
public void testAbortUploadFail() throws Exception {
String volumeName = UUID.randomUUID().toString();
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index ad2211cd37..62149a73d1 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -919,6 +919,11 @@ public class ObjectEndpoint extends EndpointBase {
throw newError(NO_SUCH_UPLOAD, uploadID, ex);
} else if (isAccessDenied(ex)) {
throw newError(S3ErrorTable.ACCESS_DENIED, bucket + "/" + key, ex);
+ } else if (ex.getResult() == ResultCodes.INVALID_PART) {
+ OS3Exception os3Exception = newError(
+ S3ErrorTable.INVALID_ARGUMENT, String.valueOf(partNumber), ex);
+ os3Exception.setErrorMessage(ex.getMessage());
+ throw os3Exception;
}
throw ex;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
