This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a4be83f523 HDDS-11267. Skip duplicate deletion of block in datanode
(#7031)
a4be83f523 is described below
commit a4be83f5237288e12bdfd3efa640f5d55f87b2c6
Author: Arafat2198 <[email protected]>
AuthorDate: Thu Aug 8 12:26:09 2024 +0530
HDDS-11267. Skip duplicate deletion of block in datanode (#7031)
---
.../statemachine/background/BlockDeletingTask.java | 21 +++++++++++++++++----
1 file changed, 17 insertions(+), 4 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingTask.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingTask.java
index 60e5a58355..af500b2b6b 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingTask.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingTask.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.time.Duration;
import java.time.Instant;
+import java.util.HashSet;
+import java.util.Set;
import java.util.LinkedList;
import java.util.Objects;
import java.util.ArrayList;
@@ -424,14 +426,27 @@ public class BlockDeletingTask implements BackgroundTask {
List<DeletedBlocksTransaction> delBlocks, Handler handler,
Table<String, BlockData> blockDataTable, Container container)
throws IOException {
+
int blocksProcessed = 0;
int blocksDeleted = 0;
long bytesReleased = 0;
List<DeletedBlocksTransaction> deletedBlocksTxs = new ArrayList<>();
Instant startTime = Instant.now();
+ // Track deleted blocks to avoid duplicate deletion
+ Set<Long> deletedBlockSet = new HashSet<>();
+
for (DeletedBlocksTransaction entry : delBlocks) {
for (Long blkLong : entry.getLocalIDList()) {
+ // Increment blocksProcessed for every block processed
+ blocksProcessed++;
+
+ // Check if the block has already been deleted
+ if (deletedBlockSet.contains(blkLong)) {
+ LOG.debug("Skipping duplicate deletion for block {}", blkLong);
+ continue;
+ }
+
String blk = containerData.getBlockKey(blkLong);
BlockData blkInfo = blockDataTable.get(blk);
LOG.debug("Deleting block {}", blkLong);
@@ -442,8 +457,6 @@ public class BlockDeletingTask implements BackgroundTask {
LOG.error("Failed to delete files for unreferenced block {} of" +
" container {}", blkLong,
container.getContainerData().getContainerID(), e);
- } finally {
- blocksProcessed++;
}
continue;
}
@@ -453,14 +466,14 @@ public class BlockDeletingTask implements BackgroundTask {
handler.deleteBlock(container, blkInfo);
blocksDeleted++;
deleted = true;
+ // Track this block as deleted
+ deletedBlockSet.add(blkLong);
} catch (IOException e) {
// TODO: if deletion of certain block retries exceed the certain
// number of times, service should skip deleting it,
// otherwise invalid numPendingDeletionBlocks could accumulate
// beyond the limit and the following deletion will stop.
LOG.error("Failed to delete files for block {}", blkLong, e);
- } finally {
- blocksProcessed++;
}
if (deleted) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
