This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-7593
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-7593 by this push:
new fd188d1ba1 HDDS-10650. Delete hsync key info from openFileTable while
deleting directory recursively. (#6495)
fd188d1ba1 is described below
commit fd188d1ba1dd8b1ac2e3295e4911b5ba78d6d729
Author: Ashish Kumar <[email protected]>
AuthorDate: Wed Apr 24 10:32:02 2024 +0530
HDDS-10650. Delete hsync key info from openFileTable while deleting
directory recursively. (#6495)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 84 ++++++++++++++++++++++
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 4 +-
.../om/request/file/OMRecoverLeaseRequest.java | 4 ++
.../om/request/key/OMAllocateBlockRequest.java | 4 ++
.../key/OMDirectoriesPurgeRequestWithFSO.java | 23 +++++-
.../ozone/om/request/key/OMKeyCommitRequest.java | 4 ++
.../ozone/om/request/key/OMKeyDeleteRequest.java | 9 ++-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 9 ++-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 22 +++---
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 16 +++--
.../key/OMDirectoriesPurgeResponseWithFSO.java | 11 ++-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 28 +++++---
.../response/key/OMKeyDeleteResponseWithFSO.java | 18 +++--
.../om/response/key/OMKeysDeleteResponse.java | 20 +++---
.../response/key/OMKeysDeleteResponseWithFSO.java | 13 ++--
.../om/response/key/TestOMKeysDeleteResponse.java | 2 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 2 +-
18 files changed, 216 insertions(+), 58 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 5d8eb43274..3ddbb41008 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -387,6 +387,7 @@ public final class OzoneConsts {
/** Metadata stored in OmKeyInfo. */
public static final String HSYNC_CLIENT_ID = "hsyncClientId";
public static final String LEASE_RECOVERY = "leaseRecovery";
+ public static final String DELETED_HSYNC_KEY = "deletedHsyncKey";
public static final String FORCE_LEASE_RECOVERY_ENV =
"OZONE.CLIENT.RECOVER.LEASE.FORCE";
//GDPR
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
index daa433f68f..cdc9487a05 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
@@ -25,7 +25,9 @@ import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
@@ -84,6 +86,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import org.apache.ozone.test.GenericTestUtils;
@@ -113,6 +116,11 @@ import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_LEASE_HARD_LIMIT;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -144,6 +152,10 @@ public class TestHSync {
private static final int FLUSH_SIZE = 2 * CHUNK_SIZE;
private static final int MAX_FLUSH_SIZE = 2 * FLUSH_SIZE;
private static final int BLOCK_SIZE = 2 * MAX_FLUSH_SIZE;
+ private static final int SERVICE_INTERVAL = 100;
+ private static final int EXPIRE_THRESHOLD_MS = 140;
+
+ private static OpenKeyCleanupService openKeyCleanupService;
@BeforeAll
public static void init() throws Exception {
@@ -155,9 +167,18 @@ public class TestHSync {
CONF.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10);
// Reduce KeyDeletingService interval
CONF.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100,
TimeUnit.MILLISECONDS);
+ CONF.setTimeDuration(OZONE_DIR_DELETING_SERVICE_INTERVAL, 100,
TimeUnit.MILLISECONDS);
CONF.setBoolean("ozone.client.incremental.chunk.list", true);
CONF.setBoolean("ozone.client.stream.putblock.piggybacking", true);
CONF.setBoolean(OZONE_CHUNK_LIST_INCREMENTAL, true);
+ CONF.setTimeDuration(OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL,
+ SERVICE_INTERVAL, TimeUnit.MILLISECONDS);
+ CONF.setTimeDuration(OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD,
+ EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS);
+ CONF.setTimeDuration(OZONE_OM_LEASE_HARD_LIMIT,
+ EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS);
+ CONF.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
+
ClientConfigForTesting.newBuilder(StorageUnit.BYTES)
.setBlockSize(BLOCK_SIZE)
.setChunkSize(CHUNK_SIZE)
@@ -183,6 +204,10 @@ public class TestHSync {
GenericTestUtils.setLogLevel(BlockOutputStream.LOG, Level.DEBUG);
GenericTestUtils.setLogLevel(BlockInputStream.LOG, Level.DEBUG);
GenericTestUtils.setLogLevel(KeyValueHandler.LOG, Level.DEBUG);
+
+ openKeyCleanupService =
+ (OpenKeyCleanupService)
cluster.getOzoneManager().getKeyManager().getOpenKeyCleanupService();
+ openKeyCleanupService.suspend();
}
@AfterAll
@@ -393,6 +418,65 @@ public class TestHSync {
}
}
+ @Test
+ public void testHSyncOpenKeyDeletionWhileDeleteDirectory() throws Exception {
+ // Verify that when directory is deleted recursively hsync related
openKeys should be deleted,
+
+ // Set the fs.defaultFS
+ final String rootPath = String.format("%s://%s/",
+ OZONE_OFS_URI_SCHEME, CONF.get(OZONE_OM_ADDRESS_KEY));
+ CONF.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+
+ final String dir = OZONE_ROOT + bucket.getVolumeName()
+ + OZONE_URI_DELIMITER + bucket.getName() + OZONE_URI_DELIMITER +
"dir1/dir2";
+ final Path key1 = new Path(dir, "hsync-key");
+
+ try (FileSystem fs = FileSystem.get(CONF)) {
+ // Create key1
+ try (FSDataOutputStream os = fs.create(key1, true)) {
+ os.write(1);
+ os.hsync();
+ // There should be 1 key in openFileTable
+ assertThat(1 ==
getOpenKeyInfo(BucketLayout.FILE_SYSTEM_OPTIMIZED).size());
+ // Delete directory recursively
+ fs.delete(new Path(OZONE_ROOT + bucket.getVolumeName() +
OZONE_URI_DELIMITER +
+ bucket.getName() + OZONE_URI_DELIMITER + "dir1/"), true);
+
+ // Verify if DELETED_HSYNC_KEY metadata is added to openKey
+ GenericTestUtils.waitFor(() -> {
+ List<OmKeyInfo> omKeyInfo =
getOpenKeyInfo(BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ return omKeyInfo.size() > 0 &&
omKeyInfo.get(0).getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY);
+ }, 1000, 12000);
+
+ // Resume openKeyCleanupService
+ openKeyCleanupService.resume();
+
+ // Verify entry from openKey gets deleted eventually
+ GenericTestUtils.waitFor(() ->
+ 0 == getOpenKeyInfo(BucketLayout.FILE_SYSTEM_OPTIMIZED).size(),
1000, 12000);
+ } catch (OMException ex) {
+ assertEquals(OMException.ResultCodes.DIRECTORY_NOT_FOUND,
ex.getResult());
+ } finally {
+ openKeyCleanupService.suspend();
+ }
+ }
+ }
+
+ private List<OmKeyInfo> getOpenKeyInfo(BucketLayout bucketLayout) {
+ List<OmKeyInfo> omKeyInfo = new ArrayList<>();
+
+ Table<String, OmKeyInfo> openFileTable =
+
cluster.getOzoneManager().getMetadataManager().getOpenKeyTable(bucketLayout);
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ iterator = openFileTable.iterator()) {
+ while (iterator.hasNext()) {
+ omKeyInfo.add(iterator.next().getValue());
+ }
+ } catch (Exception e) {
+ }
+ return omKeyInfo;
+ }
+
@Test
public void testUncommittedBlocks() throws Exception {
// Set the fs.defaultFS
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index afc9eae859..5e68359193 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -1838,8 +1838,10 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
.filter(id -> id.equals(clientIdString))
.isPresent();
- if (!isHsync && openKeyInfo.getCreationTime() <=
expiredCreationTimestamp) {
+ if ((!isHsync && openKeyInfo.getCreationTime() <=
expiredCreationTimestamp) ||
+
(openKeyInfo.getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY))) {
// add non-hsync'ed keys
+ // also add hsync keys which are already deleted from keyTable
expiredKeys.addOpenKey(openKeyInfo, dbOpenKeyName);
num++;
} else if (isHsync && openKeyInfo.getModificationTime() <=
expiredLeaseTimestamp &&
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
index 6116ed81e8..be12886a68 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java
@@ -221,6 +221,10 @@ public class OMRecoverLeaseRequest extends OMKeyRequest {
throw new OMException("Open Key " + dbOpenFileKey + " not found in
openKeyTable", KEY_NOT_FOUND);
}
+ if (openKeyInfo.getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY)) {
+ throw new OMException("Open Key " + keyName + " is already deleted",
+ KEY_NOT_FOUND);
+ }
long openKeyModificationTime = openKeyInfo.getModificationTime();
if (openKeyInfo.getMetadata().containsKey(OzoneConsts.LEASE_RECOVERY)) {
LOG.debug("Key: " + keyName + " is already under recovery");
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index d47b2b7a57..2e3b63d6e4 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -208,6 +208,10 @@ public class OMAllocateBlockRequest extends OMKeyRequest {
throw new OMException("Open Key " + openKeyName + " is under lease
recovery",
KEY_UNDER_LEASE_RECOVERY);
}
+ if
(openKeyInfo.getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY)) {
+ throw new OMException("Open Key " + openKeyName + " is already
deleted",
+ KEY_NOT_FOUND);
+ }
List<OmKeyLocationInfo> newLocationList = Collections.singletonList(
OmKeyLocationInfo.getFromProtobuf(blockLocation));
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
index 505b628730..d6935ed683 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
@@ -21,9 +21,11 @@ package org.apache.hadoop.ozone.om.request.key;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -39,8 +41,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
-import java.util.List;
-
+import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY;
import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
/**
@@ -66,6 +67,7 @@ public class OMDirectoriesPurgeRequestWithFSO extends
OMKeyRequest {
Set<Pair<String, String>> lockSet = new HashSet<>();
Map<Pair<String, String>, OmBucketInfo> volBucketInfoMap = new HashMap<>();
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
+ Map<String, OmKeyInfo> openKeyInfoMap = new HashMap<>();
OMMetrics omMetrics = ozoneManager.getMetrics();
try {
@@ -110,6 +112,21 @@ public class OMDirectoriesPurgeRequestWithFSO extends
OMKeyRequest {
volumeName, bucketName);
lockSet.add(volBucketPair);
}
+
+ // If omKeyInfo has hsync metadata, delete its corresponding open
key as well
+ String dbOpenKey;
+ String hsyncClientId =
keyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID);
+ if (hsyncClientId != null) {
+ long parentId = keyInfo.getParentObjectID();
+ dbOpenKey = omMetadataManager.getOpenFileName(path.getVolumeId(),
path.getBucketId(),
+ parentId, keyInfo.getFileName(), hsyncClientId);
+ OmKeyInfo openKeyInfo =
omMetadataManager.getOpenKeyTable(getBucketLayout()).get(dbOpenKey);
+ if (openKeyInfo != null) {
+ openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true");
+ openKeyInfoMap.put(dbOpenKey, openKeyInfo);
+ }
+ }
+
omMetrics.decNumKeys();
OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager,
volumeName, bucketName);
@@ -142,7 +159,7 @@ public class OMDirectoriesPurgeRequestWithFSO extends
OMKeyRequest {
getOmRequest());
OMClientResponse omClientResponse = new OMDirectoriesPurgeResponseWithFSO(
omResponse.build(), purgeRequests, ozoneManager.isRatisEnabled(),
- getBucketLayout(), volBucketInfoMap, fromSnapshotInfo);
+ getBucketLayout(), volBucketInfoMap, fromSnapshotInfo,
openKeyInfoMap);
return omClientResponse;
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 830ab70827..2b238342ee 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -253,6 +253,10 @@ public class OMKeyCommitRequest extends OMKeyRequest {
throw new OMException("Failed to " + action + " key, as " + dbOpenKey +
" entry is not found in the OpenKey table", KEY_NOT_FOUND);
}
+ if (omKeyInfo.getMetadata().containsKey(OzoneConsts.DELETED_HSYNC_KEY)) {
+ throw new OMException("Open Key " + keyName + " is already deleted",
+ KEY_NOT_FOUND);
+ }
if (omKeyInfo.getMetadata().containsKey(OzoneConsts.LEASE_RECOVERY) &&
omKeyInfo.getMetadata().containsKey(OzoneConsts.HSYNC_CLIENT_ID)) {
if (!isRecovery) {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
index 61e5976f80..14dc824245 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.util.Time;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
@@ -161,6 +162,7 @@ public class OMKeyDeleteRequest extends OMKeyRequest {
long quotaReleased = sumBlockLengths(omKeyInfo);
omBucketInfo.incrUsedBytes(-quotaReleased);
omBucketInfo.incrUsedNamespace(-1L);
+ OmKeyInfo deletedOpenKeyInfo = null;
// If omKeyInfo has hsync metadata, delete its corresponding open key as
well
String dbOpenKey = null;
@@ -170,8 +172,9 @@ public class OMKeyDeleteRequest extends OMKeyRequest {
dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName,
keyName, hsyncClientId);
OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey);
if (openKeyInfo != null) {
- // Remove the open key by putting a tombstone entry
- openKeyTable.addCacheEntry(dbOpenKey, trxnLogIndex);
+ openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true");
+ openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex);
+ deletedOpenKeyInfo = openKeyInfo;
} else {
LOG.warn("Potentially inconsistent DB state: open key not found with
dbOpenKey '{}'", dbOpenKey);
}
@@ -180,7 +183,7 @@ public class OMKeyDeleteRequest extends OMKeyRequest {
omClientResponse = new OMKeyDeleteResponse(
omResponse.setDeleteKeyResponse(DeleteKeyResponse.newBuilder())
.build(), omKeyInfo, ozoneManager.isRatisEnabled(),
- omBucketInfo.copyObject(), dbOpenKey);
+ omBucketInfo.copyObject(), deletedOpenKeyInfo);
result = Result.SUCCESS;
} catch (IOException | InvalidPathException ex) {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java
index a0b2cfcbb1..b8b3efbc24 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java
@@ -51,6 +51,7 @@ import java.io.IOException;
import java.nio.file.InvalidPathException;
import java.util.Map;
+import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DIRECTORY_NOT_EMPTY;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
@@ -129,6 +130,7 @@ public class OMKeyDeleteRequestWithFSO extends
OMKeyDeleteRequest {
String ozonePathKey = omMetadataManager.getOzonePathKey(volumeId,
bucketId, omKeyInfo.getParentObjectID(),
omKeyInfo.getFileName());
+ OmKeyInfo deletedOpenKeyInfo = null;
if (keyStatus.isDirectory()) {
// Check if there are any sub path exists under the user requested path
@@ -165,8 +167,9 @@ public class OMKeyDeleteRequestWithFSO extends
OMKeyDeleteRequest {
dbOpenKey = omMetadataManager.getOpenFileName(volumeId, bucketId,
parentId, fileName, hsyncClientId);
OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey);
if (openKeyInfo != null) {
- // Remove the open key by putting a tombstone entry
- openKeyTable.addCacheEntry(dbOpenKey, trxnLogIndex);
+ openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true");
+ openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex);
+ deletedOpenKeyInfo = openKeyInfo;
} else {
LOG.warn("Potentially inconsistent DB state: open key not found with
dbOpenKey '{}'", dbOpenKey);
}
@@ -175,7 +178,7 @@ public class OMKeyDeleteRequestWithFSO extends
OMKeyDeleteRequest {
omClientResponse = new OMKeyDeleteResponseWithFSO(omResponse
.setDeleteKeyResponse(DeleteKeyResponse.newBuilder()).build(),
keyName, omKeyInfo, ozoneManager.isRatisEnabled(),
- omBucketInfo.copyObject(), keyStatus.isDirectory(), volumeId,
dbOpenKey);
+ omBucketInfo.copyObject(), keyStatus.isDirectory(), volumeId,
deletedOpenKeyInfo);
result = Result.SUCCESS;
} catch (IOException | InvalidPathException ex) {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
index be89da369c..61ed5ffb1c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
@@ -57,11 +57,13 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.InvalidPathException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.apache.hadoop.ozone.OzoneConsts.BUCKET;
+import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.DELETED_KEYS_LIST;
import static org.apache.hadoop.ozone.OzoneConsts.UNDELETED_KEYS_LIST;
import static org.apache.hadoop.ozone.OzoneConsts.VOLUME;
@@ -174,18 +176,18 @@ public class OMKeysDeleteRequest extends OMKeyRequest {
OmBucketInfo omBucketInfo =
getBucketInfo(omMetadataManager, volumeName, bucketName);
- List<String> dbOpenKeys = new ArrayList<>();
+ Map<String, OmKeyInfo> openKeyInfoMap = new HashMap<>();
// Mark all keys which can be deleted, in cache as deleted.
quotaReleased =
markKeysAsDeletedInCache(ozoneManager, trxnLogIndex, omKeyInfoList,
- dirList, omMetadataManager, quotaReleased, dbOpenKeys);
+ dirList, omMetadataManager, quotaReleased, openKeyInfoMap);
omBucketInfo.incrUsedBytes(-quotaReleased);
omBucketInfo.incrUsedNamespace(-1L * omKeyInfoList.size());
final long volumeId = omMetadataManager.getVolumeId(volumeName);
omClientResponse =
getOmClientResponse(ozoneManager, omKeyInfoList, dirList, omResponse,
- unDeletedKeys, deleteStatus, omBucketInfo, volumeId, dbOpenKeys);
+ unDeletedKeys, deleteStatus, omBucketInfo, volumeId,
openKeyInfoMap);
result = Result.SUCCESS;
@@ -260,7 +262,7 @@ public class OMKeysDeleteRequest extends OMKeyRequest {
List<OmKeyInfo> omKeyInfoList, List<OmKeyInfo> dirList,
OMResponse.Builder omResponse,
OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys,
- boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId,
List<String> dbOpenKeys) {
+ boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId,
Map<String, OmKeyInfo> openKeyInfoMap) {
OMClientResponse omClientResponse;
omClientResponse = new OMKeysDeleteResponse(omResponse
.setDeleteKeysResponse(
@@ -268,13 +270,13 @@ public class OMKeysDeleteRequest extends OMKeyRequest {
.setUnDeletedKeys(unDeletedKeys))
.setStatus(deleteStatus ? OK : PARTIAL_DELETE).setSuccess(deleteStatus)
.build(), omKeyInfoList, ozoneManager.isRatisEnabled(),
- omBucketInfo.copyObject(), dbOpenKeys);
+ omBucketInfo.copyObject(), openKeyInfoMap);
return omClientResponse;
}
protected long markKeysAsDeletedInCache(OzoneManager ozoneManager,
long trxnLogIndex, List<OmKeyInfo> omKeyInfoList, List<OmKeyInfo>
dirList,
- OMMetadataManager omMetadataManager, long quotaReleased, List<String>
dbOpenKeys)
+ OMMetadataManager omMetadataManager, long quotaReleased, Map<String,
OmKeyInfo> openKeyInfoMap)
throws IOException {
for (OmKeyInfo omKeyInfo : omKeyInfoList) {
String volumeName = omKeyInfo.getVolumeName();
@@ -294,10 +296,10 @@ public class OMKeysDeleteRequest extends OMKeyRequest {
String dbOpenKey = omMetadataManager.getOpenKey(volumeName,
bucketName, keyName, hsyncClientId);
OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey);
if (openKeyInfo != null) {
- // Remove the open key by putting a tombstone entry
- openKeyTable.addCacheEntry(dbOpenKey, trxnLogIndex);
- // Append to the list of open keys to be deleted. The list is not
expected to be large.
- dbOpenKeys.add(dbOpenKey);
+ openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true");
+ openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex);
+ // Add to the map of open keys to be deleted.
+ openKeyInfoMap.put(dbOpenKey, openKeyInfo);
} else {
LOG.warn("Potentially inconsistent DB state: open key not found with
dbOpenKey '{}'", dbOpenKey);
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
index b90fd15b26..85d9c8ff4c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
@@ -37,7 +37,9 @@ import jakarta.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
+import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.PARTIAL_DELETE;
@@ -88,7 +90,7 @@ public class OmKeysDeleteRequestWithFSO extends
OMKeysDeleteRequest {
OzoneManager ozoneManager, long trxnLogIndex,
List<OmKeyInfo> omKeyInfoList,
List<OmKeyInfo> dirList, OMMetadataManager omMetadataManager,
- long quotaReleased, List<String> dbOpenKeys) throws IOException {
+ long quotaReleased, Map<String, OmKeyInfo> openKeyInfoMap) throws
IOException {
// Mark all keys which can be deleted, in cache as deleted.
for (OmKeyInfo omKeyInfo : omKeyInfoList) {
@@ -113,10 +115,10 @@ public class OmKeysDeleteRequestWithFSO extends
OMKeysDeleteRequest {
String dbOpenKey = omMetadataManager.getOpenFileName(volumeId,
bucketId, parentId, fileName, hsyncClientId);
OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey);
if (openKeyInfo != null) {
- // Remove the open key by putting a tombstone entry
- openKeyTable.addCacheEntry(dbOpenKey, trxnLogIndex);
- // Append to the list of open keys to be deleted. The list is not
expected to be large.
- dbOpenKeys.add(dbOpenKey);
+ openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true");
+ openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex);
+ // Add to the map of open keys to be deleted.
+ openKeyInfoMap.put(dbOpenKey, openKeyInfo);
} else {
LOG.warn("Potentially inconsistent DB state: open key not found with
dbOpenKey '{}'", dbOpenKey);
}
@@ -146,7 +148,7 @@ public class OmKeysDeleteRequestWithFSO extends
OMKeysDeleteRequest {
List<OmKeyInfo> omKeyInfoList, List<OmKeyInfo> dirList,
OzoneManagerProtocolProtos.OMResponse.Builder omResponse,
OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys,
- boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId,
List<String> dbOpenKeys) {
+ boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId,
Map<String, OmKeyInfo> openKeyInfoMap) {
OMClientResponse omClientResponse;
omClientResponse = new OMKeysDeleteResponseWithFSO(omResponse
.setDeleteKeysResponse(
@@ -154,7 +156,7 @@ public class OmKeysDeleteRequestWithFSO extends
OMKeysDeleteRequest {
.setStatus(deleteStatus).setUnDeletedKeys(unDeletedKeys))
.setStatus(deleteStatus ? OK : PARTIAL_DELETE).setSuccess(deleteStatus)
.build(), omKeyInfoList, dirList, ozoneManager.isRatisEnabled(),
- omBucketInfo.copyObject(), volumeId, dbOpenKeys);
+ omBucketInfo.copyObject(), volumeId, openKeyInfoMap);
return omClientResponse;
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
index 848c5c3089..edb13f8cf9 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
@@ -62,17 +62,19 @@ public class OMDirectoriesPurgeResponseWithFSO extends
OmKeyResponse {
private boolean isRatisEnabled;
private Map<Pair<String, String>, OmBucketInfo> volBucketInfoMap;
private SnapshotInfo fromSnapshotInfo;
+ private Map<String, OmKeyInfo> openKeyInfoMap;
public OMDirectoriesPurgeResponseWithFSO(@Nonnull OMResponse omResponse,
@Nonnull List<OzoneManagerProtocolProtos.PurgePathRequest> paths,
boolean isRatisEnabled, @Nonnull BucketLayout bucketLayout,
Map<Pair<String, String>, OmBucketInfo> volBucketInfoMap,
- SnapshotInfo fromSnapshotInfo) {
+ SnapshotInfo fromSnapshotInfo, Map<String, OmKeyInfo> openKeyInfoMap) {
super(omResponse, bucketLayout);
this.paths = paths;
this.isRatisEnabled = isRatisEnabled;
this.volBucketInfoMap = volBucketInfoMap;
this.fromSnapshotInfo = fromSnapshotInfo;
+ this.openKeyInfoMap = openKeyInfoMap;
}
@Override
@@ -165,6 +167,13 @@ public class OMDirectoriesPurgeResponseWithFSO extends
OmKeyResponse {
deletedKey, repeatedOmKeyInfo);
}
+ if (!openKeyInfoMap.isEmpty()) {
+ for (Map.Entry<String, OmKeyInfo> entry : openKeyInfoMap.entrySet()) {
+ omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch(
+ batchOperation, entry.getKey(), entry.getValue());
+ }
+ }
+
// Delete the visited directory from deleted directory table
if (path.hasDeletedDir()) {
omMetadataManager.getDeletedDirTable().deleteWithBatch(batchOperation,
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java
index 0cb0d745d1..7cdd2dc36b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponse.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.om.response.key;
import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -45,15 +46,15 @@ public class OMKeyDeleteResponse extends
AbstractOMKeyDeleteResponse {
private OmKeyInfo omKeyInfo;
private OmBucketInfo omBucketInfo;
// If not null, this key will be deleted from OpenKeyTable
- private String dbOpenKey;
+ private OmKeyInfo deletedOpenKeyInfo;
public OMKeyDeleteResponse(@Nonnull OMResponse omResponse,
@Nonnull OmKeyInfo omKeyInfo, boolean isRatisEnabled,
- @Nonnull OmBucketInfo omBucketInfo, String dbOpenKey) {
+ @Nonnull OmBucketInfo omBucketInfo, OmKeyInfo deletedOpenKeyInfo) {
super(omResponse, isRatisEnabled, omBucketInfo.getBucketLayout());
this.omKeyInfo = omKeyInfo;
this.omBucketInfo = omBucketInfo;
- this.dbOpenKey = dbOpenKey;
+ this.deletedOpenKeyInfo = deletedOpenKeyInfo;
}
/**
@@ -83,10 +84,16 @@ public class OMKeyDeleteResponse extends
AbstractOMKeyDeleteResponse {
omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(),
omBucketInfo.getBucketName()), omBucketInfo);
- // Remove open key (necessary when the file is hsync'ed but not committed)
- if (dbOpenKey != null) {
- omMetadataManager.getOpenKeyTable(getBucketLayout()).deleteWithBatch(
- batchOperation, dbOpenKey);
+ // necessary when the file is hsync'ed but not committed
+ // Update metadata which will be used to cleanup openKey in
openKeyCleanupService
+ if (deletedOpenKeyInfo != null) {
+ String hsyncClientId =
deletedOpenKeyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID);
+ if (hsyncClientId != null) {
+ String dbOpenKey =
omMetadataManager.getOpenKey(deletedOpenKeyInfo.getVolumeName(),
+ deletedOpenKeyInfo.getBucketName(),
deletedOpenKeyInfo.getKeyName(), hsyncClientId);
+ omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch(
+ batchOperation, dbOpenKey, deletedOpenKeyInfo);
+ }
}
}
@@ -94,11 +101,12 @@ public class OMKeyDeleteResponse extends
AbstractOMKeyDeleteResponse {
return omKeyInfo;
}
+ protected OmKeyInfo getDeletedOpenKeyInfo() {
+ return deletedOpenKeyInfo;
+ }
+
protected OmBucketInfo getOmBucketInfo() {
return omBucketInfo;
}
- public String getDbOpenKey() {
- return dbOpenKey;
- }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java
index f52ea1b4ce..e428684a16 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseWithFSO.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.om.response.key;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -52,8 +53,8 @@ public class OMKeyDeleteResponseWithFSO extends
OMKeyDeleteResponse {
public OMKeyDeleteResponseWithFSO(@Nonnull OMResponse omResponse,
@Nonnull String keyName, @Nonnull OmKeyInfo omKeyInfo,
boolean isRatisEnabled, @Nonnull OmBucketInfo omBucketInfo,
- @Nonnull boolean isDeleteDirectory, @Nonnull long volumeId, String
dbOpenKey) {
- super(omResponse, omKeyInfo, isRatisEnabled, omBucketInfo, dbOpenKey);
+ @Nonnull boolean isDeleteDirectory, @Nonnull long volumeId, OmKeyInfo
deletedOpenKeyInfo) {
+ super(omResponse, omKeyInfo, isRatisEnabled, omBucketInfo,
deletedOpenKeyInfo);
this.keyName = keyName;
this.isDeleteDirectory = isDeleteDirectory;
this.volumeId = volumeId;
@@ -110,9 +111,16 @@ public class OMKeyDeleteResponseWithFSO extends
OMKeyDeleteResponse {
omMetadataManager.getBucketKey(getOmBucketInfo().getVolumeName(),
getOmBucketInfo().getBucketName()), getOmBucketInfo());
- if (getDbOpenKey() != null) {
- omMetadataManager.getOpenKeyTable(getBucketLayout()).deleteWithBatch(
- batchOperation, getDbOpenKey());
+ // Update metadata which will be used to cleanup openKey in
openKeyCleanupService
+ OmKeyInfo deletedOpenKeyInfo = getDeletedOpenKeyInfo();
+ if (deletedOpenKeyInfo != null) {
+ String hsyncClientId =
getDeletedOpenKeyInfo().getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID);
+ if (hsyncClientId != null) {
+ String dbOpenKey =
omMetadataManager.getOpenKey(deletedOpenKeyInfo.getVolumeName(),
+ deletedOpenKeyInfo.getBucketName(),
deletedOpenKeyInfo.getKeyName(), hsyncClientId);
+ omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch(
+ batchOperation, dbOpenKey, deletedOpenKeyInfo);
+ }
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java
index 23f3acd45a..8cf7f6b926 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java
@@ -29,7 +29,9 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRespo
import jakarta.annotation.Nonnull;
import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.BUCKET_TABLE;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE;
@@ -45,16 +47,16 @@ import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.
public class OMKeysDeleteResponse extends AbstractOMKeyDeleteResponse {
private List<OmKeyInfo> omKeyInfoList;
private OmBucketInfo omBucketInfo;
- private List<String> dbOpenKeys;
+ private Map<String, OmKeyInfo> openKeyInfoMap = new HashMap<>();
public OMKeysDeleteResponse(@Nonnull OMResponse omResponse,
@Nonnull List<OmKeyInfo> keyDeleteList,
boolean isRatisEnabled, @Nonnull OmBucketInfo omBucketInfo,
- @Nonnull List<String> dbOpenKeys) {
+ @Nonnull Map<String, OmKeyInfo> openKeyInfoMap) {
super(omResponse, isRatisEnabled);
this.omKeyInfoList = keyDeleteList;
this.omBucketInfo = omBucketInfo;
- this.dbOpenKeys = dbOpenKeys;
+ this.openKeyInfoMap = openKeyInfoMap;
}
/**
@@ -100,9 +102,11 @@ public class OMKeysDeleteResponse extends
AbstractOMKeyDeleteResponse {
omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(),
omBucketInfo.getBucketName()), omBucketInfo);
- for (String dbOpenKey : dbOpenKeys) {
- omMetadataManager.getOpenKeyTable(getBucketLayout()).deleteWithBatch(
- batchOperation, dbOpenKey);
+ if (!openKeyInfoMap.isEmpty()) {
+ for (Map.Entry<String, OmKeyInfo> entry : openKeyInfoMap.entrySet()) {
+ omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch(
+ batchOperation, entry.getKey(), entry.getValue());
+ }
}
}
@@ -114,7 +118,7 @@ public class OMKeysDeleteResponse extends
AbstractOMKeyDeleteResponse {
return omBucketInfo;
}
- public List<String> getDbOpenKeys() {
- return dbOpenKeys;
+ protected Map<String, OmKeyInfo> getOpenKeyInfoMap() {
+ return openKeyInfoMap;
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
index 3a662727b0..861204c393 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
@@ -29,6 +29,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import jakarta.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.BUCKET_TABLE;
import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_DIR_TABLE;
@@ -52,8 +53,8 @@ public class OMKeysDeleteResponseWithFSO extends
OMKeysDeleteResponse {
@Nonnull List<OmKeyInfo> keyDeleteList,
@Nonnull List<OmKeyInfo> dirDeleteList, boolean isRatisEnabled,
@Nonnull OmBucketInfo omBucketInfo, @Nonnull long volId,
- @Nonnull List<String> dbOpenKeys) {
- super(omResponse, keyDeleteList, isRatisEnabled, omBucketInfo, dbOpenKeys);
+ @Nonnull Map<String, OmKeyInfo> openKeyInfoMap) {
+ super(omResponse, keyDeleteList, isRatisEnabled, omBucketInfo,
openKeyInfoMap);
this.dirsList = dirDeleteList;
this.volumeId = volId;
}
@@ -95,9 +96,11 @@ public class OMKeysDeleteResponseWithFSO extends
OMKeysDeleteResponse {
omMetadataManager.getBucketKey(getOmBucketInfo().getVolumeName(),
getOmBucketInfo().getBucketName()), getOmBucketInfo());
- for (String dbOpenKey : getDbOpenKeys()) {
- omMetadataManager.getOpenKeyTable(getBucketLayout()).deleteWithBatch(
- batchOperation, dbOpenKey);
+ if (!getOpenKeyInfoMap().isEmpty()) {
+ for (Map.Entry<String, OmKeyInfo> entry :
getOpenKeyInfoMap().entrySet()) {
+ omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch(
+ batchOperation, entry.getKey(), entry.getValue());
+ }
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java
index 8d178bcd47..e7689e52ed 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java
@@ -104,7 +104,7 @@ public class TestOMKeysDeleteResponse extends
TestOMKeyResponse {
protected OMClientResponse getOmKeysDeleteResponse(OMResponse omResponse,
OmBucketInfo omBucketInfo) {
return new OMKeysDeleteResponse(
- omResponse, omKeyInfoList, true, omBucketInfo,
Collections.emptyList());
+ omResponse, omKeyInfoList, true, omBucketInfo, Collections.emptyMap());
}
@Test
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
index 98522814de..13e706757f 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
@@ -112,7 +112,7 @@ public class TestOMKeysDeleteResponseWithFSO
OmBucketInfo omBucketInfo) {
return new OMKeysDeleteResponseWithFSO(
omResponse, getOmKeyInfoList(), dirDeleteList, true, omBucketInfo,
- volId, Collections.emptyList());
+ volId, Collections.emptyMap());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
