This is an automated email from the ASF dual-hosted git repository.
sshenoy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5eead92666 HDDS-7592. Recursive delete of directory multiple time
impacts some directory not removed (#4068)
5eead92666 is described below
commit 5eead92666b374c4bb912f332488acb23f9a81df
Author: Sumit Agrawal <[email protected]>
AuthorDate: Thu Dec 15 11:41:37 2022 +0530
HDDS-7592. Recursive delete of directory multiple time impacts some
directory not removed (#4068)
---
.../java/org/apache/hadoop/ozone/om/OMMetadataManager.java | 10 ++++++++++
.../java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 5 +++++
.../om/response/key/OMDirectoriesPurgeResponseWithFSO.java | 4 +++-
.../ozone/om/response/key/OMKeyDeleteResponseWithFSO.java | 4 +++-
.../ozone/om/response/key/OMKeysDeleteResponseWithFSO.java | 4 +++-
.../hadoop/ozone/om/service/DirectoryDeletingService.java | 4 +++-
.../ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java | 7 ++++++-
7 files changed, 33 insertions(+), 5 deletions(-)
diff --git
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index 469170475d..53080aa4a9 100644
---
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -448,6 +448,16 @@ public interface OMMetadataManager extends
DBStoreHAManager {
String getOzonePathKey(long volumeId, long bucketId,
long parentObjectId, String pathComponentName);
+ /**
+ * Given ozone path key, component id, return the corresponding
+ * DB path key for delete table.
+ *
+ * @param objectId - object Id
+ * @param pathKey - path key of component
+ * @return DB Delete directory key as String.
+ */
+ String getOzoneDeletePathKey(long objectId, String pathKey);
+
/**
* Returns DB key name of an open file in OM metadata store. Should be
* #open# prefix followed by actual leaf node name.
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index c499d1e386..247e657170 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -1452,6 +1452,11 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
return builder.toString();
}
+ @Override
+ public String getOzoneDeletePathKey(long objectId, String pathKey) {
+ return pathKey + OM_KEY_PREFIX + objectId;
+ }
+
@Override
public String getOpenFileName(long volumeId, long bucketId,
long parentID, String fileName,
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
index eed768fbb0..b89d56dc04 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
@@ -85,8 +85,10 @@ public class OMDirectoriesPurgeResponseWithFSO extends
OmKeyResponse {
OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key);
String ozoneDbKey = omMetadataManager.getOzonePathKey(volumeId,
bucketId, keyInfo.getParentObjectID(), keyInfo.getFileName());
+ String ozoneDeleteKey = omMetadataManager.getOzoneDeletePathKey(
+ key.getObjectID(), ozoneDbKey);
omMetadataManager.getDeletedDirTable().putWithBatch(batchOperation,
- ozoneDbKey, keyInfo);
+ ozoneDeleteKey, keyInfo);
omMetadataManager.getDirectoryTable().deleteWithBatch(batchOperation,
ozoneDbKey);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java
index 22f1702836..e18bcaae1f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java
@@ -84,8 +84,10 @@ public class OMKeyDeleteResponseWithFSO extends
OMKeyDeleteResponse {
// Sets full absolute key name to OmKeyInfo, which is
// required for moving the sub-files to KeyDeletionService.
omKeyInfo.setKeyName(keyName);
+ String ozoneDeleteKey = omMetadataManager.getOzoneDeletePathKey(
+ omKeyInfo.getObjectID(), ozoneDbKey);
omMetadataManager.getDeletedDirTable().putWithBatch(
- batchOperation, ozoneDbKey, omKeyInfo);
+ batchOperation, ozoneDeleteKey, omKeyInfo);
} else {
Table<String, OmKeyInfo> keyTable =
omMetadataManager.getKeyTable(getBucketLayout());
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
index d0d98e1f54..612fa36618 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
@@ -70,8 +70,10 @@ public class OMKeysDeleteResponseWithFSO extends
OMKeysDeleteResponse {
omKeyInfo.getParentObjectID(), omKeyInfo.getFileName());
omMetadataManager.getDirectoryTable().deleteWithBatch(batchOperation,
ozoneDbKey);
+ String ozoneDeleteKey = omMetadataManager.getOzoneDeletePathKey(
+ omKeyInfo.getObjectID(), ozoneDbKey);
omMetadataManager.getDeletedDirTable().putWithBatch(
- batchOperation, ozoneDbKey, omKeyInfo);
+ batchOperation, ozoneDeleteKey, omKeyInfo);
}
// remove keys from FileTable and add to DeletedTable
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
index 89f7a5bb06..3a633f2551 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
@@ -252,7 +252,9 @@ public class DirectoryDeletingService extends
BackgroundService {
for (OmKeyInfo dirInfo : subDirs) {
String ozoneDbKey = omMetadataManager.getOzonePathKey(volumeId,
bucketId, dirInfo.getParentObjectID(), dirInfo.getFileName());
- subDirList.add(Pair.of(ozoneDbKey, dirInfo));
+ String ozoneDeleteKey = omMetadataManager.getOzoneDeletePathKey(
+ dirInfo.getObjectID(), ozoneDbKey);
+ subDirList.add(Pair.of(ozoneDeleteKey, dirInfo));
LOG.debug("Moved sub dir name: {}", dirInfo.getKeyName());
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
index 9d17220631..6127c50533 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
@@ -53,6 +53,7 @@ public class TestOMKeysDeleteResponseWithFSO
private List<OmKeyInfo> dirDeleteList = new ArrayList<>();
private List<String> dirDBKeys = new ArrayList<>();
+ private List<String> dirDelDBKeys = new ArrayList<>();
private long volId;
@Override
@@ -82,6 +83,8 @@ public class TestOMKeysDeleteResponseWithFSO
bucketName, dirInfo, dir);
dirDeleteList.add(dirKeyInfo);
dirDBKeys.add(dirOzoneDBKey);
+ dirDelDBKeys.add(omMetadataManager.getOzoneDeletePathKey(
+ dirKeyInfo.getObjectID(), dirOzoneDBKey));
// create set of keys directly under the bucket
String ozoneDBKey = "";
@@ -150,10 +153,12 @@ public class TestOMKeysDeleteResponseWithFSO
RepeatedOmKeyInfo repeatedOmKeyInfo =
omMetadataManager.getDeletedTable().get(dirDBKey);
Assert.assertNull(repeatedOmKeyInfo);
+ }
+ for (String dirDelDBKey : dirDelDBKeys) {
// dir added to the deleted dir table, for deep cleanups
OmKeyInfo omDirInfo =
- omMetadataManager.getDeletedDirTable().get(dirDBKey);
+ omMetadataManager.getDeletedDirTable().get(dirDelDBKey);
Assert.assertNotNull(omDirInfo);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
