hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.

2017-05-26 Thread shv 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 948d0ac72 -> cafee1109


HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn 
Sharp.

(cherry picked from commit 0d8a35bd6de5d2a5a9b816ca98f31975e94bd7c6)

Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cafee110
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cafee110
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cafee110

Branch: refs/heads/branch-2.7
Commit: cafee1109c7b4a1e3fe2a1b18f0495a410cc19ef
Parents: 948d0ac
Author: Kihwal Lee 
Authored: Thu Dec 1 12:20:30 2016 -0600
Committer: Konstantin V Shvachko 
Committed: Fri May 26 19:00:03 2017 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  2 +
 .../server/blockmanagement/BlockManager.java| 75 +++-
 2 files changed, 26 insertions(+), 51 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cafee110/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7e4994a..09ee6cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -114,6 +114,8 @@ Release 2.7.4 - UNRELEASED
 HDFS-9710. DN can be configured to send block receipt IBRs in batches.
 (Tsz-Wo Nicholas Sze. Backport HDFS-11837 by Vinitha Reddy Gankidi)
 
+HDFS-8674. Improve performance of postponed block scans. (Daryn Sharp)
+
   BUG FIXES

 HDFS-8307. Spurious DNS Queries from hdfs shell. (Andres Perez via 
aengineer)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cafee110/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index aeb38e2..da15397 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -27,6 +27,7 @@ import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -127,7 +128,6 @@ public class BlockManager {
   private volatile long underReplicatedBlocksCount = 0L;
   private volatile long scheduledReplicationBlocksCount = 0L;
   private final AtomicLong excessBlocksCount = new AtomicLong(0L);
-  private final AtomicLong postponedMisreplicatedBlocksCount = new 
AtomicLong(0L);
   private final long startupDelayBlockDeletionInMs;
   
   /** Used by metrics */
@@ -160,7 +160,7 @@ public class BlockManager {
   }
   /** Used by metrics */
   public long getPostponedMisreplicatedBlocksCount() {
-return postponedMisreplicatedBlocksCount.get();
+return postponedMisreplicatedBlocks.size();
   }
   /** Used by metrics */
   public int getPendingDataNodeMessageCount() {
@@ -196,7 +196,10 @@ public class BlockManager {
* notified of all block deletions that might have been pending
* when the failover happened.
*/
-  private final Set postponedMisreplicatedBlocks = Sets.newHashSet();
+  private final LinkedHashSet postponedMisreplicatedBlocks =
+  new LinkedHashSet();
+  private final int blocksPerPostpondedRescan;
+  private final ArrayList rescannedMisreplicatedBlocks;
 
   /**
* Maps a StorageID to the set of blocks that are "extra" for this
@@ -286,7 +289,10 @@ public class BlockManager {
 this.namesystem = namesystem;
 datanodeManager = new DatanodeManager(this, namesystem, conf);
 heartbeatManager = datanodeManager.getHeartbeatManager();
-
+blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE,
+datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan());
+rescannedMisreplicatedBlocks =
+new ArrayList(blocksPerPostpondedRescan);
 startupDelayBlockDeletionInMs = conf.getLong(
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 
1000L;
@@ -1319,9 +1325,7 @@ public class BlockManager {
 
 
   private void postponeBlock(Block blk) {
-if

[16/29] hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.

2016-12-05 Thread xgong 
HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn 
Sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96c57492
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96c57492
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96c57492

Branch: refs/heads/YARN-5734
Commit: 96c574927a600d15fab919df1fdc9e07887af6c5
Parents: e0fa492
Author: Kihwal Lee 
Authored: Thu Dec 1 12:11:27 2016 -0600
Committer: Kihwal Lee 
Committed: Thu Dec 1 12:11:27 2016 -0600

--
 .../server/blockmanagement/BlockManager.java| 79 ++--
 1 file changed, 24 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c57492/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 1b744e7..e60703b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -30,6 +30,7 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -43,8 +44,6 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
 import javax.management.ObjectName;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -101,7 +100,6 @@ import 
org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
 import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
 import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
 import org.apache.hadoop.hdfs.util.FoldedTreeSet;
-import org.apache.hadoop.hdfs.util.LightWeightHashSet;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.server.namenode.CacheManager;
 
@@ -184,7 +182,6 @@ public class BlockManager implements BlockStatsMXBean {
   /** flag indicating whether replication queues have been initialized */
   private boolean initializedReplQueues;
 
-  private final AtomicLong postponedMisreplicatedBlocksCount = new 
AtomicLong(0L);
   private final long startupDelayBlockDeletionInMs;
   private final BlockReportLeaseManager blockReportLeaseManager;
   private ObjectName mxBeanName;
@@ -219,7 +216,7 @@ public class BlockManager implements BlockStatsMXBean {
   }
   /** Used by metrics */
   public long getPostponedMisreplicatedBlocksCount() {
-return postponedMisreplicatedBlocksCount.get();
+return postponedMisreplicatedBlocks.size();
   }
   /** Used by metrics */
   public int getPendingDataNodeMessageCount() {
@@ -275,8 +272,10 @@ public class BlockManager implements BlockStatsMXBean {
* notified of all block deletions that might have been pending
* when the failover happened.
*/
-  private final LightWeightHashSet postponedMisreplicatedBlocks =
-  new LightWeightHashSet<>();
+  private final Set postponedMisreplicatedBlocks =
+  new LinkedHashSet();
+  private final int blocksPerPostpondedRescan;
+  private final ArrayList rescannedMisreplicatedBlocks;
 
   /**
* Maps a StorageID to the set of blocks that are "extra" for this
@@ -378,7 +377,10 @@ public class BlockManager implements BlockStatsMXBean {
 datanodeManager = new DatanodeManager(this, namesystem, conf);
 heartbeatManager = datanodeManager.getHeartbeatManager();
 this.blockIdManager = new BlockIdManager(this);
-
+blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE,
+datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan());
+rescannedMisreplicatedBlocks =
+new ArrayList(blocksPerPostpondedRescan);
 startupDelayBlockDeletionInMs = conf.getLong(
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 
1000L;
@@ -1613,9 +1615,7 @@ public class BlockManager implements BlockStatsMXBean {
 
 
   private void postponeBlock(Block blk) {
-if (postponedMisreplicatedBlocks.add(blk)) {
-  postponedMisreplicatedBlocksCount.incrementAndGet();
-}
+postponedMisreplicatedBlocks.add(blk);
   }

[21/24] hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.

2016-12-01 Thread asuresh 
HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn 
Sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96c57492
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96c57492
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96c57492

Branch: refs/heads/YARN-5085
Commit: 96c574927a600d15fab919df1fdc9e07887af6c5
Parents: e0fa492
Author: Kihwal Lee 
Authored: Thu Dec 1 12:11:27 2016 -0600
Committer: Kihwal Lee 
Committed: Thu Dec 1 12:11:27 2016 -0600

--
 .../server/blockmanagement/BlockManager.java| 79 ++--
 1 file changed, 24 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c57492/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 1b744e7..e60703b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -30,6 +30,7 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -43,8 +44,6 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
 import javax.management.ObjectName;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -101,7 +100,6 @@ import 
org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
 import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
 import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
 import org.apache.hadoop.hdfs.util.FoldedTreeSet;
-import org.apache.hadoop.hdfs.util.LightWeightHashSet;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.server.namenode.CacheManager;
 
@@ -184,7 +182,6 @@ public class BlockManager implements BlockStatsMXBean {
   /** flag indicating whether replication queues have been initialized */
   private boolean initializedReplQueues;
 
-  private final AtomicLong postponedMisreplicatedBlocksCount = new 
AtomicLong(0L);
   private final long startupDelayBlockDeletionInMs;
   private final BlockReportLeaseManager blockReportLeaseManager;
   private ObjectName mxBeanName;
@@ -219,7 +216,7 @@ public class BlockManager implements BlockStatsMXBean {
   }
   /** Used by metrics */
   public long getPostponedMisreplicatedBlocksCount() {
-return postponedMisreplicatedBlocksCount.get();
+return postponedMisreplicatedBlocks.size();
   }
   /** Used by metrics */
   public int getPendingDataNodeMessageCount() {
@@ -275,8 +272,10 @@ public class BlockManager implements BlockStatsMXBean {
* notified of all block deletions that might have been pending
* when the failover happened.
*/
-  private final LightWeightHashSet postponedMisreplicatedBlocks =
-  new LightWeightHashSet<>();
+  private final Set postponedMisreplicatedBlocks =
+  new LinkedHashSet();
+  private final int blocksPerPostpondedRescan;
+  private final ArrayList rescannedMisreplicatedBlocks;
 
   /**
* Maps a StorageID to the set of blocks that are "extra" for this
@@ -378,7 +377,10 @@ public class BlockManager implements BlockStatsMXBean {
 datanodeManager = new DatanodeManager(this, namesystem, conf);
 heartbeatManager = datanodeManager.getHeartbeatManager();
 this.blockIdManager = new BlockIdManager(this);
-
+blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE,
+datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan());
+rescannedMisreplicatedBlocks =
+new ArrayList(blocksPerPostpondedRescan);
 startupDelayBlockDeletionInMs = conf.getLong(
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 
1000L;
@@ -1613,9 +1615,7 @@ public class BlockManager implements BlockStatsMXBean {
 
 
   private void postponeBlock(Block blk) {
-if (postponedMisreplicatedBlocks.add(blk)) {
-  postponedMisreplicatedBlocksCount.incrementAndGet();
-}
+postponedMisreplicatedBlocks.add(blk);
   }

hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.

2016-12-01 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 afad13f8d -> dd4acebb4


HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn 
Sharp.

(cherry picked from commit 0d8a35bd6de5d2a5a9b816ca98f31975e94bd7c6)

Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd4acebb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd4acebb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd4acebb

Branch: refs/heads/branch-2.8
Commit: dd4acebb4146c7629ce914df6642c5d69b8172d9
Parents: afad13f
Author: Kihwal Lee 
Authored: Thu Dec 1 12:20:30 2016 -0600
Committer: Kihwal Lee 
Committed: Thu Dec 1 12:20:30 2016 -0600

--
 .../server/blockmanagement/BlockManager.java| 75 +++-
 1 file changed, 24 insertions(+), 51 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd4acebb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 0ba01aa..a929c43 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -29,6 +29,7 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -143,7 +144,6 @@ public class BlockManager implements BlockStatsMXBean {
   private boolean initializedReplQueues;
 
   private final AtomicLong excessBlocksCount = new AtomicLong(0L);
-  private final AtomicLong postponedMisreplicatedBlocksCount = new 
AtomicLong(0L);
   private final long startupDelayBlockDeletionInMs;
   private final BlockReportLeaseManager blockReportLeaseManager;
   private ObjectName mxBeanName;
@@ -178,7 +178,7 @@ public class BlockManager implements BlockStatsMXBean {
   }
   /** Used by metrics */
   public long getPostponedMisreplicatedBlocksCount() {
-return postponedMisreplicatedBlocksCount.get();
+return postponedMisreplicatedBlocks.size();
   }
   /** Used by metrics */
   public int getPendingDataNodeMessageCount() {
@@ -218,8 +218,10 @@ public class BlockManager implements BlockStatsMXBean {
* notified of all block deletions that might have been pending
* when the failover happened.
*/
-  private final LightWeightHashSet postponedMisreplicatedBlocks =
-  new LightWeightHashSet<>();
+  private final LinkedHashSet postponedMisreplicatedBlocks =
+  new LinkedHashSet();
+  private final int blocksPerPostpondedRescan;
+  private final ArrayList rescannedMisreplicatedBlocks;
 
   /**
* Maps a StorageID to the set of blocks that are "extra" for this
@@ -316,6 +318,10 @@ public class BlockManager implements BlockStatsMXBean {
 datanodeManager = new DatanodeManager(this, namesystem, conf);
 heartbeatManager = datanodeManager.getHeartbeatManager();
 
+blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE,
+datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan());
+rescannedMisreplicatedBlocks =
+new ArrayList(blocksPerPostpondedRescan);
 startupDelayBlockDeletionInMs = conf.getLong(
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 
1000L;
@@ -1424,9 +1430,7 @@ public class BlockManager implements BlockStatsMXBean {
 
 
   private void postponeBlock(Block blk) {
-if (postponedMisreplicatedBlocks.add(blk)) {
-  postponedMisreplicatedBlocksCount.incrementAndGet();
-}
+postponedMisreplicatedBlocks.add(blk);
   }
   
   
@@ -2050,39 +2054,14 @@ public class BlockManager implements BlockStatsMXBean {
 if (getPostponedMisreplicatedBlocksCount() == 0) {
   return;
 }
-long startTimeRescanPostponedMisReplicatedBlocks = Time.monotonicNow();
 namesystem.writeLock();
-long startPostponedMisReplicatedBlocksCount =
-getPostponedMisreplicatedBlocksCount();
+long startTime = Time.monotonicNow();
+long startSize = postponedMisreplicatedBlocks.size();
 try {
-  // blocksPerRescan is the configured number of blocks per rescan.
-  // Randomly

hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.

2016-12-01 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 8ffe86f78 -> 0d8a35bd6


HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn 
Sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d8a35bd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d8a35bd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d8a35bd

Branch: refs/heads/branch-2
Commit: 0d8a35bd6de5d2a5a9b816ca98f31975e94bd7c6
Parents: 8ffe86f
Author: Kihwal Lee 
Authored: Thu Dec 1 12:15:15 2016 -0600
Committer: Kihwal Lee 
Committed: Thu Dec 1 12:15:15 2016 -0600

--
 .../server/blockmanagement/BlockManager.java| 76 +++-
 1 file changed, 24 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d8a35bd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 954b297..f2805e2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -29,6 +29,7 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -170,7 +171,6 @@ public class BlockManager implements BlockStatsMXBean {
   private boolean initializedReplQueues;
 
   private final AtomicLong excessBlocksCount = new AtomicLong(0L);
-  private final AtomicLong postponedMisreplicatedBlocksCount = new 
AtomicLong(0L);
   private final long startupDelayBlockDeletionInMs;
   private final BlockReportLeaseManager blockReportLeaseManager;
   private ObjectName mxBeanName;
@@ -205,7 +205,7 @@ public class BlockManager implements BlockStatsMXBean {
   }
   /** Used by metrics */
   public long getPostponedMisreplicatedBlocksCount() {
-return postponedMisreplicatedBlocksCount.get();
+return postponedMisreplicatedBlocks.size();
   }
   /** Used by metrics */
   public int getPendingDataNodeMessageCount() {
@@ -245,8 +245,10 @@ public class BlockManager implements BlockStatsMXBean {
* notified of all block deletions that might have been pending
* when the failover happened.
*/
-  private final LightWeightHashSet postponedMisreplicatedBlocks =
-  new LightWeightHashSet<>();
+  private final LinkedHashSet postponedMisreplicatedBlocks =
+  new LinkedHashSet();
+  private final int blocksPerPostpondedRescan;
+  private final ArrayList rescannedMisreplicatedBlocks;
 
   /**
* Maps a StorageID to the set of blocks that are "extra" for this
@@ -345,7 +347,10 @@ public class BlockManager implements BlockStatsMXBean {
 datanodeManager = new DatanodeManager(this, namesystem, conf);
 heartbeatManager = datanodeManager.getHeartbeatManager();
 this.blockIdManager = new BlockIdManager(this);
-
+blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE,
+datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan());
+rescannedMisreplicatedBlocks =
+new ArrayList(blocksPerPostpondedRescan);
 startupDelayBlockDeletionInMs = conf.getLong(
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 
1000L;
@@ -1455,9 +1460,7 @@ public class BlockManager implements BlockStatsMXBean {
 
 
   private void postponeBlock(Block blk) {
-if (postponedMisreplicatedBlocks.add(blk)) {
-  postponedMisreplicatedBlocksCount.incrementAndGet();
-}
+postponedMisreplicatedBlocks.add(blk);
   }
   
   
@@ -2164,39 +2167,14 @@ public class BlockManager implements BlockStatsMXBean {
 if (getPostponedMisreplicatedBlocksCount() == 0) {
   return;
 }
-long startTimeRescanPostponedMisReplicatedBlocks = Time.monotonicNow();
 namesystem.writeLock();
-long startPostponedMisReplicatedBlocksCount =
-getPostponedMisreplicatedBlocksCount();
+long startTime = Time.monotonicNow();
+long startSize = postponedMisreplicatedBlocks.size();
 try {
-  // blocksPerRescan is the configured number of blocks per rescan.
-  // Randomly select blocksPerRescan consecutive blocks from the HashSet
-  // when the number of blocks remaining is larger than blocksPerRescan.
-  // The

hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.

2016-12-01 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/trunk e0fa49234 -> 96c574927


HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn 
Sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96c57492
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96c57492
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96c57492

Branch: refs/heads/trunk
Commit: 96c574927a600d15fab919df1fdc9e07887af6c5
Parents: e0fa492
Author: Kihwal Lee 
Authored: Thu Dec 1 12:11:27 2016 -0600
Committer: Kihwal Lee 
Committed: Thu Dec 1 12:11:27 2016 -0600

--
 .../server/blockmanagement/BlockManager.java| 79 ++--
 1 file changed, 24 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c57492/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 1b744e7..e60703b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -30,6 +30,7 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -43,8 +44,6 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
 import javax.management.ObjectName;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -101,7 +100,6 @@ import 
org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
 import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
 import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
 import org.apache.hadoop.hdfs.util.FoldedTreeSet;
-import org.apache.hadoop.hdfs.util.LightWeightHashSet;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.server.namenode.CacheManager;
 
@@ -184,7 +182,6 @@ public class BlockManager implements BlockStatsMXBean {
   /** flag indicating whether replication queues have been initialized */
   private boolean initializedReplQueues;
 
-  private final AtomicLong postponedMisreplicatedBlocksCount = new 
AtomicLong(0L);
   private final long startupDelayBlockDeletionInMs;
   private final BlockReportLeaseManager blockReportLeaseManager;
   private ObjectName mxBeanName;
@@ -219,7 +216,7 @@ public class BlockManager implements BlockStatsMXBean {
   }
   /** Used by metrics */
   public long getPostponedMisreplicatedBlocksCount() {
-return postponedMisreplicatedBlocksCount.get();
+return postponedMisreplicatedBlocks.size();
   }
   /** Used by metrics */
   public int getPendingDataNodeMessageCount() {
@@ -275,8 +272,10 @@ public class BlockManager implements BlockStatsMXBean {
* notified of all block deletions that might have been pending
* when the failover happened.
*/
-  private final LightWeightHashSet postponedMisreplicatedBlocks =
-  new LightWeightHashSet<>();
+  private final Set postponedMisreplicatedBlocks =
+  new LinkedHashSet();
+  private final int blocksPerPostpondedRescan;
+  private final ArrayList rescannedMisreplicatedBlocks;
 
   /**
* Maps a StorageID to the set of blocks that are "extra" for this
@@ -378,7 +377,10 @@ public class BlockManager implements BlockStatsMXBean {
 datanodeManager = new DatanodeManager(this, namesystem, conf);
 heartbeatManager = datanodeManager.getHeartbeatManager();
 this.blockIdManager = new BlockIdManager(this);
-
+blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE,
+datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan());
+rescannedMisreplicatedBlocks =
+new ArrayList(blocksPerPostpondedRescan);
 startupDelayBlockDeletionInMs = conf.getLong(
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
 DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 
1000L;
@@ -1613,9 +1615,7 @@ public class BlockManager implements BlockStatsMXBean {
 
 
   private void postponeBlock(Block blk) {
-if (postponedMisreplicatedBlocks.add(blk)) {
-