This is an automated email from the ASF dual-hosted git repository.
sumitagrawal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d3b63c681a HDDS-11492. Directory deletion get stuck having millions of
directory (#7254)
d3b63c681a is described below
commit d3b63c681a59c7b6321b1e2bd252614280faa605
Author: Sumit Agrawal <[email protected]>
AuthorDate: Wed Oct 2 09:59:48 2024 +0530
HDDS-11492. Directory deletion get stuck having millions of directory
(#7254)
---
.../om/service/AbstractKeyDeletingService.java | 2 +
.../ozone/om/service/DirectoryDeletingService.java | 2 +
.../om/service/TestDirectoryDeletingService.java | 52 ++++++++++++++++++++++
3 files changed, 56 insertions(+)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
index d5da77ca0a..7559cf9a72 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
@@ -422,6 +422,8 @@ public abstract class AbstractKeyDeletingService extends
BackgroundService
}
consumedSize += request.getSerializedSize();
purgePathRequestList.add(request);
+ // reduce remain count for self, sub-files, and sub-directories
+ remainNum = remainNum - 1;
remainNum = remainNum - request.getDeletedSubFilesCount();
remainNum = remainNum - request.getMarkDeletedSubDirsCount();
// Count up the purgeDeletedDir, subDirs and subFiles
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
index b300051599..09f4a8f8a3 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
@@ -209,6 +209,8 @@ public class DirectoryDeletingService extends
AbstractKeyDeletingService {
}
consumedSize += request.getSerializedSize();
purgePathRequestList.add(request);
+ // reduce remain count for self, sub-files, and sub-directories
+ remainNum = remainNum - 1;
remainNum = remainNum - request.getDeletedSubFilesCount();
remainNum = remainNum - request.getMarkDeletedSubDirsCount();
// Count up the purgeDeletedDir, subDirs and subFiles
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java
index 8dcb030d63..04e8efa7b7 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java
@@ -50,6 +50,7 @@ import org.junit.jupiter.api.io.TempDir;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -161,4 +162,55 @@ public class TestDirectoryDeletingService {
500, 60000);
assertThat(dirDeletingService.getRunCount().get()).isGreaterThanOrEqualTo(1);
}
+
+ @Test
+ public void testDeleteDirectoryFlatDirsHavingNoChilds() throws Exception {
+ OzoneConfiguration conf = createConfAndInitValues();
+ OmTestManagers omTestManagers
+ = new OmTestManagers(conf);
+ KeyManager keyManager = omTestManagers.getKeyManager();
+ writeClient = omTestManagers.getWriteClient();
+ om = omTestManagers.getOzoneManager();
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ om.getMetadataManager(), BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ String bucketKey = om.getMetadataManager().getBucketKey(volumeName,
bucketName);
+ OmBucketInfo bucketInfo =
om.getMetadataManager().getBucketTable().get(bucketKey);
+
+ int dirCreatesCount = OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT * 2 + 100;
+ long parentId = 1;
+ OmDirectoryInfo baseDir = new OmDirectoryInfo.Builder().setName("dir_base")
+ .setCreationTime(Time.now()).setModificationTime(Time.now())
+ .setObjectID(parentId).setParentObjectID(bucketInfo.getObjectID())
+ .setUpdateID(0).build();
+ OMRequestTestUtils.addDirKeyToDirTable(true, baseDir, volumeName,
bucketName,
+ 1L, om.getMetadataManager());
+ for (int i = 0; i < dirCreatesCount; ++i) {
+ OmDirectoryInfo dir1 = new OmDirectoryInfo.Builder().setName("dir" + i)
+
.setCreationTime(Time.now()).setModificationTime(Time.now()).setParentObjectID(parentId)
+ .setObjectID(i + 100).setUpdateID(i).build();
+ OMRequestTestUtils.addDirKeyToDirTable(true, dir1, volumeName,
bucketName,
+ 1L, om.getMetadataManager());
+ }
+
+ DirectoryDeletingService dirDeletingService =
keyManager.getDirDeletingService();
+ long[] delDirCnt = new long[2];
+ delDirCnt[0] = dirDeletingService.getDeletedDirsCount();
+
+ OmKeyArgs delArgs = new OmKeyArgs.Builder()
+
.setVolumeName(volumeName).setBucketName(bucketName).setKeyName("dir_base")
+ .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE))
+ .setDataSize(0).setRecursive(true).build();
+ writeClient.deleteKey(delArgs);
+
+ // check if difference between each run should not cross the directory
deletion limit
+ // and wait till all dir is removed
+ GenericTestUtils.waitFor(() -> {
+ delDirCnt[1] = dirDeletingService.getDeletedDirsCount();
+ assertTrue(delDirCnt[1] - delDirCnt[0] <=
OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT,
+ "base: " + delDirCnt[0] + ", new: " + delDirCnt[1]);
+ delDirCnt[0] = delDirCnt[1];
+ return dirDeletingService.getDeletedDirsCount() >= dirCreatesCount;
+ }, 500, 300000);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
