This is an automated email from the ASF dual-hosted git repository.
ChenSammi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1a4486a271b HDDS-16088. Reject lifecycle
AbortIncompleteMultipartUpload rules whose daysAfterInitiation exceeds the MPU
cleanup threshold (#10982)
1a4486a271b is described below
commit 1a4486a271b0d292ad8868f9664f17d03d0972d1
Author: Priyesh Karatha <[email protected]>
AuthorDate: Mon Aug 17 10:03:13 2026 +0530
HDDS-16088. Reject lifecycle AbortIncompleteMultipartUpload rules whose
daysAfterInitiation exceeds the MPU cleanup threshold (#10982)
---
.../common/src/main/resources/ozone-default.xml | 16 ++++--
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 4 +-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 4 +-
.../OMLifecycleConfigurationSetRequest.java | 50 ++++++++++++++++-
.../TestOMLifecycleConfigurationRequest.java | 45 +++++++++++++++-
.../TestOMLifecycleConfigurationSetRequest.java | 63 ++++++++++++++++++++++
6 files changed, 173 insertions(+), 9 deletions(-)
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 3e51a6d9d04..9528ca27fc2 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1546,9 +1546,19 @@
<value>30d</value>
<tag>OZONE, OM, PERFORMANCE</tag>
<description>
- Controls how long multipart upload is considered active. Specifically,
if a multipart info
- has been ongoing longer than the value of this config entry, that
multipart info is considered as
- expired (e.g. due to client crash). Unit could be defined with postfix
(ns,ms,s,m,h,d)
+ Controls how long a multipart upload is considered active before
MultipartUploadCleanupService
+ treats it as stale and aborts it. If a multipart upload has been ongoing
longer than this
+ threshold (e.g. due to a client crash), it will be cleaned up
automatically.
+ Unit can be specified with a postfix (ns, ms, s, m, h, d).
+
+ Interaction with S3 lifecycle AbortIncompleteMultipartUpload rules:
+ When a bucket lifecycle rule specifies an AbortIncompleteMultipartUpload
action, its
+ daysAfterInitiation value must be strictly less than this threshold. If
+ daysAfterInitiation is greater than or equal to this threshold,
MultipartUploadCleanupService
+ will abort the upload before the lifecycle rule fires, silently making
the rule ineffective.
+ Ozone enforces this constraint at lifecycle configuration creation time
and will reject
+ configurations that violate it. To use a longer daysAfterInitiation
value, increase this
+ threshold accordingly.
</description>
</property>
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index f95d7de9bb9..bf41354a1bc 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -2281,7 +2281,7 @@ public void
testGetLifecycleWithAbortIncompleteMultipartUpload() {
.withPrefix("")
.withStatus(BucketLifecycleConfiguration.ENABLED);
rule3.setAbortIncompleteMultipartUpload(
- new AbortIncompleteMultipartUpload().withDaysAfterInitiation(30));
+ new AbortIncompleteMultipartUpload().withDaysAfterInitiation(29));
rules.add(rule1);
rules.add(rule2);
@@ -2313,7 +2313,7 @@ public void
testGetLifecycleWithAbortIncompleteMultipartUpload() {
assertEquals("abort-incomplete-mpu-no-prefix", retrievedRule3.getId());
assertEquals("", retrievedRule3.getPrefix());
assertEquals(BucketLifecycleConfiguration.ENABLED,
retrievedRule3.getStatus());
- assertEquals(30,
retrievedRule3.getAbortIncompleteMultipartUpload().getDaysAfterInitiation());
+ assertEquals(29,
retrievedRule3.getAbortIncompleteMultipartUpload().getDaysAfterInitiation());
}
/**
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index ec80b9c4de0..d9be4078414 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -2538,7 +2538,7 @@ public void
testGetLifecycleWithAbortIncompleteMultipartUpload() {
.prefix("")
.status(ExpirationStatus.ENABLED)
.abortIncompleteMultipartUpload(AbortIncompleteMultipartUpload.builder()
- .daysAfterInitiation(30)
+ .daysAfterInitiation(29)
.build())
.build();
@@ -2583,7 +2583,7 @@ public void
testGetLifecycleWithAbortIncompleteMultipartUpload() {
assertEquals("abort-incomplete-mpu-no-filter", retrievedRule4.id());
assertEquals("", retrievedRule4.prefix());
assertEquals(ExpirationStatus.ENABLED, retrievedRule4.status());
- assertEquals(30,
retrievedRule4.abortIncompleteMultipartUpload().daysAfterInitiation());
+ assertEquals(29,
retrievedRule4.abortIncompleteMultipartUpload().daysAfterInitiation());
}
/**
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationSetRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationSetRequest.java
index 53380943788..2959c05320d 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationSetRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationSetRequest.java
@@ -23,12 +23,14 @@
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.OMAction;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.ResolvedBucket;
@@ -44,7 +46,9 @@
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import
org.apache.hadoop.ozone.om.response.lifecycle.OMLifecycleConfigurationSetResponse;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleAction;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleConfiguration;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleRule;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetLifecycleConfigurationRequest;
@@ -92,11 +96,13 @@ public OMRequest preExecute(OzoneManager ozoneManager)
throws IOException {
if (resolvedBucket.bucketLayout().toProto() !=
request.getLifecycleConfiguration().getBucketLayout()) {
throw new OMException("Bucket layout mismatch: requested lifecycle
configuration " +
- "has bucket layout " +
request.getLifecycleConfiguration().getBucketLayout() +
+ "has bucket layout " +
request.getLifecycleConfiguration().getBucketLayout() +
" but the actual bucket has layout " +
resolvedBucket.bucketLayout().toProto(),
OMException.ResultCodes.INVALID_REQUEST);
}
+ validateAbortMpuDaysAgainstCleanupThreshold(ozoneManager,
lifecycleConfiguration);
+
SetLifecycleConfigurationRequest.Builder newCreateRequest =
request.toBuilder();
@@ -203,6 +209,48 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
}
}
+ /**
+ * Rejects lifecycle configurations where an AbortIncompleteMultipartUpload
rule's
+ * daysAfterInitiation is greater than or equal to the cluster-wide MPU
expire threshold
+ * (ozone.om.open.mpu.expire.threshold). When that happens,
MultipartUploadCleanupService
+ * will reap the upload before the lifecycle rule ever fires, silently
making the rule
+ * ineffective. Failing fast here surfaces the misconfiguration to the
operator.
+ */
+ private static void validateAbortMpuDaysAgainstCleanupThreshold(
+ OzoneManager ozoneManager, LifecycleConfiguration config) throws
OMException {
+ long expireThresholdMillis =
ozoneManager.getConfiguration().getTimeDuration(
+ OMConfigKeys.OZONE_OM_MPU_EXPIRE_THRESHOLD,
+ OMConfigKeys.OZONE_OM_MPU_EXPIRE_THRESHOLD_DEFAULT,
+ TimeUnit.MILLISECONDS);
+
+ for (LifecycleRule rule : config.getRulesList()) {
+ if (!rule.getEnabled()) {
+ continue;
+ }
+ for (LifecycleAction action : rule.getActionList()) {
+ if (action.hasAbortIncompleteMultipartUpload()) {
+ int daysAfterInitiation =
action.getAbortIncompleteMultipartUpload().getDaysAfterInitiation();
+ long daysAfterInitiationMillis =
TimeUnit.DAYS.toMillis(daysAfterInitiation);
+ if (daysAfterInitiationMillis >= expireThresholdMillis) {
+ String expireThresholdConfig = ozoneManager.getConfiguration().get(
+ OMConfigKeys.OZONE_OM_MPU_EXPIRE_THRESHOLD,
+ OMConfigKeys.OZONE_OM_MPU_EXPIRE_THRESHOLD_DEFAULT);
+ throw new OMException(
+ "Invalid lifecycle configuration: rule '" + rule.getId() + "'
has an " +
+ "AbortIncompleteMultipartUpload action with
daysAfterInitiation=" + daysAfterInitiation +
+ " day(s), which is not less than the cluster MPU expire
threshold (" +
+ OMConfigKeys.OZONE_OM_MPU_EXPIRE_THRESHOLD + "=" +
expireThresholdConfig +
+ "). The MultipartUploadCleanupService will clean up the upload
before the " +
+ "lifecycle rule fires, making the rule ineffective. " +
+ "Set daysAfterInitiation to a value less than " +
expireThresholdConfig +
+ ", or increase " + OMConfigKeys.OZONE_OM_MPU_EXPIRE_THRESHOLD
+ ".",
+ OMException.ResultCodes.INVALID_REQUEST);
+ }
+ }
+ }
+ }
+ }
+
private void checkAclPermission(OzoneManager ozoneManager, String
volumeName, String bucketName)
throws IOException {
if (ozoneManager.getAccessAuthorizer().isNative()) {
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java
index ddc581c5029..69541c393c3 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java
@@ -38,6 +38,7 @@
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AbortIncompleteMultipartUpload;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketLayoutProto;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteLifecycleConfigurationRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleAction;
@@ -66,14 +67,16 @@ public class TestOMLifecycleConfigurationRequest {
protected OMMetrics omMetrics;
protected OMMetadataManager omMetadataManager;
protected AuditLogger auditLogger;
+ protected OzoneConfiguration ozoneConfiguration;
@BeforeEach
public void setup() throws Exception {
ozoneManager = mock(OzoneManager.class);
- OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ ozoneConfiguration = new OzoneConfiguration();
omMetrics = OMMetrics.create(ozoneConfiguration);
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
tempDir.getAbsolutePath());
omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration,
ozoneManager);
+ when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
when(ozoneManager.getMetrics()).thenReturn(omMetrics);
when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
when(ozoneManager.getMaxUserVolumeCount()).thenReturn(10L);
@@ -150,4 +153,44 @@ public OMRequest setLifecycleConfigurationRequest(String
volumeName,
.setClientId(UUID.randomUUID().toString())
.build();
}
+
+ /**
+ * Builds a SetLifecycleConfiguration request with a single
AbortIncompleteMultipartUpload rule.
+ */
+ public OMRequest setLifecycleConfigurationRequestWithAbortMpu(
+ String volumeName, String bucketName, int daysAfterInitiation) {
+ return setLifecycleConfigurationRequestWithAbortMpu(volumeName,
bucketName, daysAfterInitiation, true);
+ }
+
+ /**
+ * Builds a SetLifecycleConfiguration request with a single
AbortIncompleteMultipartUpload rule,
+ * with configurable enabled state.
+ */
+ public OMRequest setLifecycleConfigurationRequestWithAbortMpu(
+ String volumeName, String bucketName, int daysAfterInitiation, boolean
enabled) {
+ LifecycleConfiguration lcc = LifecycleConfiguration.newBuilder()
+ .setBucketLayout(BucketLayoutProto.OBJECT_STORE)
+ .setCreationTime(System.currentTimeMillis())
+ .setVolume(volumeName)
+ .setBucket(bucketName)
+ .addRules(LifecycleRule.newBuilder()
+ .setId(RandomStringUtils.randomAlphabetic(32))
+ .setEnabled(enabled)
+ .addAction(LifecycleAction.newBuilder()
+ .setAbortIncompleteMultipartUpload(
+ AbortIncompleteMultipartUpload.newBuilder()
+ .setDaysAfterInitiation(daysAfterInitiation)
+ .build()))
+ .setFilter(LifecycleFilter.newBuilder().setPrefix("prefix/")))
+ .build();
+
+ return OMRequest.newBuilder()
+ .setSetLifecycleConfigurationRequest(
+ SetLifecycleConfigurationRequest.newBuilder()
+ .setLifecycleConfiguration(lcc)
+ .build())
+ .setCmdType(Type.SetLifecycleConfiguration)
+ .setClientId(UUID.randomUUID().toString())
+ .build();
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
index 721a012f5a7..4029c6c36ba 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
@@ -34,7 +34,9 @@
import static org.mockito.Mockito.when;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.ResolvedBucket;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -387,4 +389,65 @@ public void
testAllowSetLifecycleConfigurationAfterFinalization() throws Excepti
assertEquals(request, result);
}
+
+ @Test
+ public void testPreExecuteRejectsAbortMpuDaysEqualToExpireThreshold() throws
Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ // Default threshold is 30d; requesting daysAfterInitiation == 30 must be
rejected.
+ long thresholdDays = TimeUnit.MILLISECONDS.toDays(
+ ozoneConfiguration.getTimeDuration(
+ OMConfigKeys.OZONE_OM_MPU_EXPIRE_THRESHOLD,
+ OMConfigKeys.OZONE_OM_MPU_EXPIRE_THRESHOLD_DEFAULT,
+ TimeUnit.MILLISECONDS));
+
+ OMRequest request = setLifecycleConfigurationRequestWithAbortMpu(
+ volumeName, bucketName, (int) thresholdDays);
+ OMLifecycleConfigurationSetRequest setRequest =
+ new OMLifecycleConfigurationSetRequest(request);
+
+ OMException ex = assertThrows(OMException.class, () ->
setRequest.preExecute(ozoneManager));
+ assertEquals(OMException.ResultCodes.INVALID_REQUEST, ex.getResult());
+ }
+
+ @Test
+ public void testPreExecuteRejectsAbortMpuDaysGreaterThanExpireThreshold()
throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ // Default threshold is 30d; requesting daysAfterInitiation == 60 must be
rejected.
+ OMRequest request =
setLifecycleConfigurationRequestWithAbortMpu(volumeName, bucketName, 60);
+ OMLifecycleConfigurationSetRequest setRequest =
+ new OMLifecycleConfigurationSetRequest(request);
+
+ OMException ex = assertThrows(OMException.class, () ->
setRequest.preExecute(ozoneManager));
+ assertEquals(OMException.ResultCodes.INVALID_REQUEST, ex.getResult());
+ }
+
+ @Test
+ public void testPreExecuteAcceptsAbortMpuDaysLessThanExpireThreshold()
throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ // Default threshold is 30d; requesting daysAfterInitiation == 7 must be
accepted.
+ OMRequest request =
setLifecycleConfigurationRequestWithAbortMpu(volumeName, bucketName, 7);
+ OMLifecycleConfigurationSetRequest setRequest =
+ new OMLifecycleConfigurationSetRequest(request);
+
+ // preExecute must succeed without throwing.
+ OMRequest result = setRequest.preExecute(ozoneManager);
+ assertNotNull(result);
+ }
+
+ @Test
+ public void testPreExecuteSkipsDisabledAbortMpuRule() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ // daysAfterInitiation == 60 would normally be rejected, but the rule is
disabled.
+ OMRequest request =
setLifecycleConfigurationRequestWithAbortMpu(volumeName, bucketName, 60, false);
+ OMLifecycleConfigurationSetRequest setRequest =
+ new OMLifecycleConfigurationSetRequest(request);
+
+ // Disabled rules must not trigger the threshold check.
+ OMRequest result = setRequest.preExecute(ozoneManager);
+ assertNotNull(result);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
