This is an automated email from the ASF dual-hosted git repository.
ChenSammi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 749da668af1 HDDS-16158. Fix sendDeleteKeysRequestAndClearList
Authorization and FSO Trash Handling (#10996)
749da668af1 is described below
commit 749da668af1097ec079fe90626984c1898850f76
Author: Priyesh Karatha <[email protected]>
AuthorDate: Thu Aug 13 17:26:26 2026 +0530
HDDS-16158. Fix sendDeleteKeysRequestAndClearList Authorization and FSO
Trash Handling (#10996)
---
.../ozone/om/service/KeyLifecycleService.java | 54 +++++++++-----
.../ozone/om/service/TestKeyLifecycleService.java | 85 ++++++++++++++++++++++
2 files changed, 122 insertions(+), 17 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
index 5310f5b4837..e551b9242e4 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
@@ -190,7 +190,6 @@ public KeyLifecycleService(OzoneManager ozoneManager,
StorageUnit.BYTES);
// always go to 90% of max limit for request as other header will be added
this.ratisByteLimit = (int) (limit * 0.9);
- this.ozoneTrash = ozoneManager.getOzoneTrash();
}
@Override
@@ -447,8 +446,8 @@ public BackgroundTaskResult call() {
// If trash is enabled, move files to trash, instead of send
delete requests.
// OBS bucket doesn't support trash.
if (bucket.getBucketLayout() == OBJECT_STORE) {
- sendDeleteKeysRequestAndClearList(bucket.getVolumeName(),
bucket.getBucketName(), expiredKeyList,
- false, scanStateBuilder, true);
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(),
bucket.getBucketName(),
+ bucket.getOwner(), expiredKeyList, false,
scanStateBuilder, true);
} else {
// handle keys first, then directories
handleAndClearFullList(bucket, expiredKeyList, false,
scanStateBuilder, true);
@@ -987,8 +986,8 @@ private void flushAndSaveState(OmBucketInfo bucket,
LimitedExpiredObjectList exp
boolean saved = false;
if (expiredKeyList != null && !expiredKeyList.isEmpty()) {
if (bucket.getBucketLayout() == OBJECT_STORE) {
- sendDeleteKeysRequestAndClearList(bucket.getVolumeName(),
bucket.getBucketName(), expiredKeyList,
- false, scanStateBuilder, false);
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(),
bucket.getBucketName(),
+ bucket.getOwner(), expiredKeyList, false, scanStateBuilder,
false);
} else {
handleAndClearFullList(bucket, expiredKeyList, false,
scanStateBuilder, false);
}
@@ -1348,12 +1347,12 @@ private void onSuccess(String bucketName) {
private void handleAndClearFullList(OmBucketInfo bucket,
LimitedExpiredObjectList keysList,
boolean dir, OmLifecycleScanState.Builder scanStateBuilder, boolean
scanFinished) {
- if (moveToTrashEnabled.get() && bucket.getBucketLayout() != OBJECT_STORE
&& ozoneTrash != null) {
+ if (moveToTrashEnabled.get() && bucket.getBucketLayout() != OBJECT_STORE
&& getEffectiveOzoneTrash() != null) {
moveToTrash(bucket, keysList, dir);
sendSaveScanStateRequest(scanStateBuilder, scanFinished);
} else {
- sendDeleteKeysRequestAndClearList(bucket.getVolumeName(),
bucket.getBucketName(), keysList, dir,
- scanStateBuilder, scanFinished);
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(),
bucket.getBucketName(),
+ bucket.getOwner(), keysList, dir, scanStateBuilder, scanFinished);
}
}
@@ -1393,8 +1392,9 @@ private void
sendSaveScanStateRequest(OmLifecycleScanState.Builder scanStateBuil
}
}
- private void sendDeleteKeysRequestAndClearList(String volume, String
bucket, LimitedExpiredObjectList keysList,
- boolean dir, OmLifecycleScanState.Builder scanStateBuilder, boolean
scanFinished) {
+ private void sendDeleteKeysRequestAndClearList(String volume, String
bucket, String bucketOwner,
+ LimitedExpiredObjectList keysList, boolean dir,
+ OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
try {
if (getInjector(1) != null) {
try {
@@ -1404,6 +1404,7 @@ private void sendDeleteKeysRequestAndClearList(String
volume, String bucket, Lim
}
}
+ UserGroupInformation ugi =
UserGroupInformation.createRemoteUser(bucketOwner);
int batchSize = keyDeleteBatchSize;
int startIndex = 0;
for (int i = 0; i < keysList.size();) {
@@ -1442,16 +1443,32 @@ private void sendDeleteKeysRequestAndClearList(String
volume, String bucket, Lim
LOG.debug("request size {} for {} keys",
deleteKeysRequest.getSerializedSize(), keyCount);
if (deleteKeysRequest.getSerializedSize() < ratisByteLimit) {
- // send request out
- OMRequest omRequest = OMRequest.newBuilder()
+ OMRequest omRequestRaw = OMRequest.newBuilder()
.setCmdType(OzoneManagerProtocolProtos.Type.DeleteKeys)
.setVersion(ClientVersion.CURRENT_VERSION)
.setClientId(clientId.toString())
.setDeleteKeysRequest(deleteKeysRequest)
.build();
long startTime = System.nanoTime();
- final OzoneManagerProtocolProtos.OMResponse response =
OzoneManagerRatisUtils.submitRequest(
- getOzoneManager(), omRequest, clientId,
callId.getAndIncrement());
+ final OzoneManagerProtocolProtos.OMResponse response;
+ try {
+ final OMClientRequest omClientRequest =
OzoneManagerRatisUtils.createClientRequest(
+ omRequestRaw, getOzoneManager());
+ response = ugi.doAs(new
PrivilegedExceptionAction<OzoneManagerProtocolProtos.OMResponse>() {
+ @Override
+ public OzoneManagerProtocolProtos.OMResponse run() throws
Exception {
+ // perform preExecute as ratis submit does not perform
preExecute
+ OMRequest omRequest =
omClientRequest.preExecute(getOzoneManager());
+ return OzoneManagerRatisUtils.submitRequest(
+ getOzoneManager(), omRequest, clientId,
callId.getAndIncrement());
+ }
+ });
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new ServiceException(e);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
long endTime = System.nanoTime();
LOG.debug("DeleteKeys request with {} keys cost {} ns", keyCount,
endTime - startTime);
long deletedCount = keyCount;
@@ -1670,9 +1687,12 @@ public void setListMaxSize(int size) {
this.listMaxSize = size;
}
- @VisibleForTesting
- public void setMpuAbortLimitPerTask(int limit) {
- this.mpuAbortLimitPerTask = limit;
+ // Returns the test-injected OzoneTrash if set, otherwise the live instance
+ // from OzoneManager. This is needed because startTrashEmptier() runs after
+ // keyManager.start() in all OzoneManager startup paths, so the field cannot
+ // be populated eagerly in the constructor.
+ private OzoneTrash getEffectiveOzoneTrash() {
+ return ozoneTrash != null ? ozoneTrash : ozoneManager.getOzoneTrash();
}
@VisibleForTesting
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
index 89746fb6c82..c80ae4e5881 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
@@ -22,7 +22,13 @@
import static org.apache.hadoop.fs.ozone.OzoneTrashPolicy.CURRENT;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
+import static
org.apache.hadoop.hdds.security.SecurityConfig.OZONE_TEST_AUTHORIZATION_ENABLED;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
import static org.apache.hadoop.ozone.OzoneConsts.ETAG;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE;
@@ -3440,6 +3446,85 @@ public void testPartCountLimitedListBoundaryBehavior() {
assertEquals(0, overList.getPartCount());
}
+ /**
+ * Tests when security is enabled.
+ */
+ @Nested
+ @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+ class WithAclsEnabled {
+
+ @BeforeAll
+ void setup(@TempDir File testDir) throws Exception {
+ scmBlockTestingClient = new ScmBlockLocationTestingClient(null, null, 0);
+ createConfig(testDir);
+ conf.setBoolean(OZONE_TEST_AUTHORIZATION_ENABLED, true);
+ conf.setBoolean(OZONE_ACL_ENABLED, true);
+ conf.set(OZONE_ACL_AUTHORIZER_CLASS, OZONE_ACL_AUTHORIZER_CLASS_NATIVE);
+ conf.setStrings(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
+ createSubject();
+ keyDeletingService.suspend();
+ directoryDeletingService.suspend();
+ }
+
+ @AfterAll
+ void cleanup() {
+ if (om != null) {
+ om.stop();
+ om.join();
+ }
+ }
+
+ @Test
+ void testLifecycleDeleteSucceedsWithAclsEnabled()
+ throws IOException, TimeoutException, InterruptedException {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialKeyCount = getKeyCount(OBJECT_STORE);
+
+ List<OmKeyArgs> keyList = createKeys(volumeName, bucketName,
OBJECT_STORE,
+ KEY_COUNT, 1, "key", null);
+ assertEquals(KEY_COUNT, keyList.size());
+ GenericTestUtils.waitFor(
+ () -> getKeyCount(OBJECT_STORE) - initialKeyCount == KEY_COUNT,
+ WAIT_CHECK_INTERVAL, 1000);
+
+ ZonedDateTime date =
ZonedDateTime.now(ZoneOffset.UTC).plusSeconds(EXPIRE_SECONDS);
+ createLifecyclePolicy(volumeName, bucketName, OBJECT_STORE, "key", null,
date.toString(), true);
+
+ GenericTestUtils.waitFor(
+ () -> (getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT,
+ WAIT_CHECK_INTERVAL, 10000);
+ assertEquals(0, getKeyCount(OBJECT_STORE) - initialKeyCount,
+ "Keys should be deleted by lifecycle service but were not — "
+ + "possible UNAUTHORIZED from missing userInfo in DeleteKeys
request");
+ }
+
+ @Test
+ void testMoveToTrashWithAclsEnabled()
+ throws IOException, TimeoutException, InterruptedException {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ long initialRenamedKeyCount = metrics.getNumKeyRenamed().value();
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ String bucketOwner =
UserGroupInformation.getCurrentUser().getShortUserName() + "-test";
+ List<OmKeyArgs> keyList =
+ createKeys(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED,
bucketOwner, KEY_COUNT, 1, "key", null);
+ assertEquals(KEY_COUNT, keyList.size());
+ final float trashInterval = 0.5f;
+ conf.setFloat(FS_TRASH_INTERVAL_KEY, trashInterval);
+ FileSystem fs = SecurityUtil.doAsLoginUser(
+ (PrivilegedExceptionAction<FileSystem>) () -> new
TrashOzoneFileSystem(om));
+ keyLifecycleService.setOzoneTrash(new OzoneTrash(fs, conf, om));
+ ZonedDateTime date =
ZonedDateTime.now(ZoneOffset.UTC).plusSeconds(EXPIRE_SECONDS);
+ createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, "",
null, date.toString(), true);
+ GenericTestUtils.waitFor(
+ () -> (metrics.getNumKeyRenamed().value() - initialRenamedKeyCount)
== KEY_COUNT,
+ WAIT_CHECK_INTERVAL, 10000);
+ assertEquals(0, getDeletedKeyCount() - initialDeletedKeyCount);
+ }
+ }
+
private static void addSplitSchemaPart(OMMetadataManager omMetadataManager,
String uploadId, int partNumber) throws IOException {
OmKeyLocationInfo locationInfo = new OmKeyLocationInfo.Builder()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
