jojochuang commented on a change in pull request #3063:
URL: https://github.com/apache/hadoop/pull/3063#discussion_r658381800
##########
File path:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
##########
@@ -4909,6 +4930,73 @@ public long getLastRedundancyMonitorTS() {
return lastRedundancyCycleTS.get();
}
+ /**
+ * Periodically deletes the marked block.
+ */
+ private class MarkedDeleteBlockScrubber implements Runnable {
+ private Iterator<BlockInfo> toDeleteIterator = null;
+ private boolean isSleep;
+
+ private void toRemove(long time) {
+ // Reentrant write lock, Release the lock when the remove is
+ // complete
+ if (checkToDeleteIterator()) {
+ namesystem.writeLock();
+ try {
+ while (toDeleteIterator.hasNext()) {
+ removeBlock(toDeleteIterator.next());
+ if (Time.now() - time > deleteBlockLockTimeMs) {
Review comment:
Use Time.monotonicNow()
##########
File path:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
##########
@@ -4909,6 +4930,73 @@ public long getLastRedundancyMonitorTS() {
return lastRedundancyCycleTS.get();
}
+ /**
+ * Periodically deletes the marked block.
+ */
+ private class MarkedDeleteBlockScrubber implements Runnable {
+ private Iterator<BlockInfo> toDeleteIterator = null;
+ private boolean isSleep;
+
+ private void toRemove(long time) {
+ // Reentrant write lock, Release the lock when the remove is
+ // complete
+ if (checkToDeleteIterator()) {
+ namesystem.writeLock();
+ try {
+ while (toDeleteIterator.hasNext()) {
+ removeBlock(toDeleteIterator.next());
+ if (Time.now() - time > deleteBlockLockTimeMs) {
+ isSleep = true;
+ break;
+ }
+ }
+ } finally {
+ namesystem.writeUnlock();
+ }
+ }
+ }
+
+ private boolean checkToDeleteIterator() {
+ return toDeleteIterator != null && toDeleteIterator.hasNext();
+ }
+
+ @Override
+ public void run() {
+ LOG.info("Start MarkedDeleteBlockScrubber thread");
+ while (namesystem.isRunning()) {
+ if (!markedDeleteQueue.isEmpty() || checkToDeleteIterator()) {
+ namesystem.writeLock();
+ try {
+ NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
+ metrics.setDeleteBlocksQueued(markedDeleteQueue.size());
+ isSleep = false;
+ long startTime = Time.now();
Review comment:
Replace Time.now() with Time.monotonicNow()
##########
File path:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
##########
@@ -3344,7 +3344,8 @@ boolean delete(String src, boolean recursive, boolean
logRetryCache)
getEditLog().logSync();
logAuditEvent(ret, operationName, src);
if (toRemovedBlocks != null) {
- removeBlocks(toRemovedBlocks); // Incremental deletion of blocks
+ blockManager.getMarkedDeleteQueue().add(
+ toRemovedBlocks.getToDeleteList());
Review comment:
we remove blocks for several other operations as well. For example,
truncate, rename, deleteSnapshot. Do we plan to make them delete block
asynchronous as well?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
