This is an automated email from the ASF dual-hosted git repository.
ljain pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 87b97fa HDDS-4447. SCMBlockDeletingService should handle
ContainerNotFoundException (#1643)
87b97fa is described below
commit 87b97fa6b4ba2c059af8d46eace9254df7fbf11a
Author: Aryan Gupta <[email protected]>
AuthorDate: Mon Dec 7 15:34:02 2020 +0530
HDDS-4447. SCMBlockDeletingService should handle ContainerNotFoundException
(#1643)
---
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 46 ++++++++++++++++------
.../hdds/scm/container/SCMContainerManager.java | 31 +++++++++------
2 files changed, 54 insertions(+), 23 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
index f13505a..ac53f2c 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
@@ -18,11 +18,12 @@
package org.apache.hadoop.hdds.scm.block;
import java.io.IOException;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
import java.util.UUID;
+import java.util.Set;
+import java.util.Map;
+import java.util.LinkedHashSet;
+import java.util.ArrayList;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
@@ -35,8 +36,9 @@ import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto.DeleteBlockTransactionResult;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import
org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler.DeleteBlockStatus;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
@@ -364,19 +366,27 @@ public class DeletedBlockLogImpl
? extends Table.KeyValue<Long, DeletedBlocksTransaction>> iter =
scmMetadataStore.getDeletedBlocksTXTable().iterator()) {
int numBlocksAdded = 0;
+ List<DeletedBlocksTransaction> txnsToBePurged =
+ new ArrayList<>();
while (iter.hasNext() && numBlocksAdded < blockDeletionLimit) {
- Table.KeyValue<Long, DeletedBlocksTransaction> keyValue =
- iter.next();
+ Table.KeyValue<Long, DeletedBlocksTransaction> keyValue =
iter.next();
DeletedBlocksTransaction txn = keyValue.getValue();
final ContainerID id = ContainerID.valueof(txn.getContainerID());
- if (txn.getCount() > -1 && txn.getCount() <= maxRetry
- && !containerManager.getContainer(id).isOpen()) {
- numBlocksAdded += txn.getLocalIDCount();
- getTransaction(txn, transactions);
- transactionToDNsCommitMap
- .putIfAbsent(txn.getTxID(), new LinkedHashSet<>());
+ try {
+ if (txn.getCount() > -1 && txn.getCount() <= maxRetry
+ && !containerManager.getContainer(id).isOpen()) {
+ numBlocksAdded += txn.getLocalIDCount();
+ getTransaction(txn, transactions);
+ transactionToDNsCommitMap
+ .putIfAbsent(txn.getTxID(), new LinkedHashSet<>());
+ }
+ } catch (ContainerNotFoundException ex) {
+ LOG.warn("Container: " + id + " was not found for the transaction:
"
+ + txn);
+ txnsToBePurged.add(txn);
}
}
+ purgeTransactions(txnsToBePurged);
}
return transactions;
} finally {
@@ -384,6 +394,18 @@ public class DeletedBlockLogImpl
}
}
+ public void purgeTransactions(List<DeletedBlocksTransaction> txnsToBePurged)
+ throws IOException {
+ try (BatchOperation batch = scmMetadataStore.getBatchHandler()
+ .initBatchOperation()) {
+ for (int i = 0; i < txnsToBePurged.size(); i++) {
+ scmMetadataStore.getDeletedBlocksTXTable()
+ .deleteWithBatch(batch, txnsToBePurged.get(i).getTxID());
+ }
+ scmMetadataStore.getBatchHandler().commitBatchOperation(batch);
+ }
+ }
+
@Override
public void onMessage(DeleteBlockStatus deleteBlockStatus,
EventPublisher publisher) {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
index 19a5ab2..7934417 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
@@ -407,18 +407,27 @@ public class SCMContainerManager implements
ContainerManager {
ContainerID containerIdObject = new ContainerID(containerID);
ContainerInfo containerInfo =
containerStore.get(containerIdObject);
- ContainerInfo containerInfoInMem = containerStateManager
- .getContainer(containerIdObject);
- if (containerInfo == null || containerInfoInMem == null) {
- throw new SCMException("Failed to increment number of deleted " +
- "blocks for container " + containerID + ", reason : " +
- "container doesn't exist.", FAILED_TO_FIND_CONTAINER);
+ try {
+ ContainerInfo containerInfoInMem = containerStateManager
+ .getContainer(containerIdObject);
+ if (containerInfo == null || containerInfoInMem == null) {
+ throw new SCMException("Failed to increment number of deleted " +
+ "blocks for container " + containerID + ", reason : " +
+ "container doesn't exist.", FAILED_TO_FIND_CONTAINER);
+ }
+ containerInfo.updateDeleteTransactionId(entry.getValue());
+ containerInfo.setNumberOfKeys(containerInfoInMem.getNumberOfKeys());
+ containerInfo.setUsedBytes(containerInfoInMem.getUsedBytes());
+ containerStore.putWithBatch(batchOperation, containerIdObject,
+ containerInfo);
+ } catch (ContainerNotFoundException ex) {
+ // Container is not present therefore we don't need to update
+ // transaction id for this container.
+ LOG.warn(
+ "Failed to update the transaction Id as container: " +
containerID
+ + " for transaction: " + entry.getValue()
+ + " does not exists");
}
- containerInfo.updateDeleteTransactionId(entry.getValue());
- containerInfo.setNumberOfKeys(containerInfoInMem.getNumberOfKeys());
- containerInfo.setUsedBytes(containerInfoInMem.getUsedBytes());
- containerStore.putWithBatch(batchOperation, containerIdObject,
- containerInfo);
}
batchHandler.commitBatchOperation(batchOperation);
containerStateManager.updateDeleteTransactionId(deleteTransactionMap);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
