hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.
Repository: hadoop Updated Branches: refs/heads/branch-2.7 948d0ac72 -> cafee1109 HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp. (cherry picked from commit 0d8a35bd6de5d2a5a9b816ca98f31975e94bd7c6) Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cafee110 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cafee110 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cafee110 Branch: refs/heads/branch-2.7 Commit: cafee1109c7b4a1e3fe2a1b18f0495a410cc19ef Parents: 948d0ac Author: Kihwal LeeAuthored: Thu Dec 1 12:20:30 2016 -0600 Committer: Konstantin V Shvachko Committed: Fri May 26 19:00:03 2017 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../server/blockmanagement/BlockManager.java| 75 +++- 2 files changed, 26 insertions(+), 51 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cafee110/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7e4994a..09ee6cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -114,6 +114,8 @@ Release 2.7.4 - UNRELEASED HDFS-9710. DN can be configured to send block receipt IBRs in batches. (Tsz-Wo Nicholas Sze. Backport HDFS-11837 by Vinitha Reddy Gankidi) +HDFS-8674. Improve performance of postponed block scans. (Daryn Sharp) + BUG FIXES HDFS-8307. Spurious DNS Queries from hdfs shell. (Andres Perez via aengineer) http://git-wip-us.apache.org/repos/asf/hadoop/blob/cafee110/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index aeb38e2..da15397 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -127,7 +128,6 @@ public class BlockManager { private volatile long underReplicatedBlocksCount = 0L; private volatile long scheduledReplicationBlocksCount = 0L; private final AtomicLong excessBlocksCount = new AtomicLong(0L); - private final AtomicLong postponedMisreplicatedBlocksCount = new AtomicLong(0L); private final long startupDelayBlockDeletionInMs; /** Used by metrics */ @@ -160,7 +160,7 @@ public class BlockManager { } /** Used by metrics */ public long getPostponedMisreplicatedBlocksCount() { -return postponedMisreplicatedBlocksCount.get(); +return postponedMisreplicatedBlocks.size(); } /** Used by metrics */ public int getPendingDataNodeMessageCount() { @@ -196,7 +196,10 @@ public class BlockManager { * notified of all block deletions that might have been pending * when the failover happened. */ - private final Set postponedMisreplicatedBlocks = Sets.newHashSet(); + private final LinkedHashSet postponedMisreplicatedBlocks = + new LinkedHashSet(); + private final int blocksPerPostpondedRescan; + private final ArrayList rescannedMisreplicatedBlocks; /** * Maps a StorageID to the set of blocks that are "extra" for this @@ -286,7 +289,10 @@ public class BlockManager { this.namesystem = namesystem; datanodeManager = new DatanodeManager(this, namesystem, conf); heartbeatManager = datanodeManager.getHeartbeatManager(); - +blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE, +datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan()); +rescannedMisreplicatedBlocks = +new ArrayList(blocksPerPostpondedRescan); startupDelayBlockDeletionInMs = conf.getLong( DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; @@ -1319,9 +1325,7 @@ public class BlockManager { private void postponeBlock(Block blk) { -if
[16/29] hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.
HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96c57492 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96c57492 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96c57492 Branch: refs/heads/YARN-5734 Commit: 96c574927a600d15fab919df1fdc9e07887af6c5 Parents: e0fa492 Author: Kihwal LeeAuthored: Thu Dec 1 12:11:27 2016 -0600 Committer: Kihwal Lee Committed: Thu Dec 1 12:11:27 2016 -0600 -- .../server/blockmanagement/BlockManager.java| 79 ++-- 1 file changed, 24 insertions(+), 55 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c57492/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 1b744e7..e60703b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -30,6 +30,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -43,8 +44,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.FutureTask; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - import javax.management.ObjectName; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -101,7 +100,6 @@ import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.util.FoldedTreeSet; -import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.namenode.CacheManager; @@ -184,7 +182,6 @@ public class BlockManager implements BlockStatsMXBean { /** flag indicating whether replication queues have been initialized */ private boolean initializedReplQueues; - private final AtomicLong postponedMisreplicatedBlocksCount = new AtomicLong(0L); private final long startupDelayBlockDeletionInMs; private final BlockReportLeaseManager blockReportLeaseManager; private ObjectName mxBeanName; @@ -219,7 +216,7 @@ public class BlockManager implements BlockStatsMXBean { } /** Used by metrics */ public long getPostponedMisreplicatedBlocksCount() { -return postponedMisreplicatedBlocksCount.get(); +return postponedMisreplicatedBlocks.size(); } /** Used by metrics */ public int getPendingDataNodeMessageCount() { @@ -275,8 +272,10 @@ public class BlockManager implements BlockStatsMXBean { * notified of all block deletions that might have been pending * when the failover happened. */ - private final LightWeightHashSet postponedMisreplicatedBlocks = - new LightWeightHashSet<>(); + private final Set postponedMisreplicatedBlocks = + new LinkedHashSet(); + private final int blocksPerPostpondedRescan; + private final ArrayList rescannedMisreplicatedBlocks; /** * Maps a StorageID to the set of blocks that are "extra" for this @@ -378,7 +377,10 @@ public class BlockManager implements BlockStatsMXBean { datanodeManager = new DatanodeManager(this, namesystem, conf); heartbeatManager = datanodeManager.getHeartbeatManager(); this.blockIdManager = new BlockIdManager(this); - +blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE, +datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan()); +rescannedMisreplicatedBlocks = +new ArrayList(blocksPerPostpondedRescan); startupDelayBlockDeletionInMs = conf.getLong( DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; @@ -1613,9 +1615,7 @@ public class BlockManager implements BlockStatsMXBean { private void postponeBlock(Block blk) { -if (postponedMisreplicatedBlocks.add(blk)) { - postponedMisreplicatedBlocksCount.incrementAndGet(); -} +postponedMisreplicatedBlocks.add(blk); }
[21/24] hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.
HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96c57492 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96c57492 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96c57492 Branch: refs/heads/YARN-5085 Commit: 96c574927a600d15fab919df1fdc9e07887af6c5 Parents: e0fa492 Author: Kihwal LeeAuthored: Thu Dec 1 12:11:27 2016 -0600 Committer: Kihwal Lee Committed: Thu Dec 1 12:11:27 2016 -0600 -- .../server/blockmanagement/BlockManager.java| 79 ++-- 1 file changed, 24 insertions(+), 55 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c57492/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 1b744e7..e60703b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -30,6 +30,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -43,8 +44,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.FutureTask; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - import javax.management.ObjectName; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -101,7 +100,6 @@ import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.util.FoldedTreeSet; -import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.namenode.CacheManager; @@ -184,7 +182,6 @@ public class BlockManager implements BlockStatsMXBean { /** flag indicating whether replication queues have been initialized */ private boolean initializedReplQueues; - private final AtomicLong postponedMisreplicatedBlocksCount = new AtomicLong(0L); private final long startupDelayBlockDeletionInMs; private final BlockReportLeaseManager blockReportLeaseManager; private ObjectName mxBeanName; @@ -219,7 +216,7 @@ public class BlockManager implements BlockStatsMXBean { } /** Used by metrics */ public long getPostponedMisreplicatedBlocksCount() { -return postponedMisreplicatedBlocksCount.get(); +return postponedMisreplicatedBlocks.size(); } /** Used by metrics */ public int getPendingDataNodeMessageCount() { @@ -275,8 +272,10 @@ public class BlockManager implements BlockStatsMXBean { * notified of all block deletions that might have been pending * when the failover happened. */ - private final LightWeightHashSet postponedMisreplicatedBlocks = - new LightWeightHashSet<>(); + private final Set postponedMisreplicatedBlocks = + new LinkedHashSet(); + private final int blocksPerPostpondedRescan; + private final ArrayList rescannedMisreplicatedBlocks; /** * Maps a StorageID to the set of blocks that are "extra" for this @@ -378,7 +377,10 @@ public class BlockManager implements BlockStatsMXBean { datanodeManager = new DatanodeManager(this, namesystem, conf); heartbeatManager = datanodeManager.getHeartbeatManager(); this.blockIdManager = new BlockIdManager(this); - +blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE, +datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan()); +rescannedMisreplicatedBlocks = +new ArrayList(blocksPerPostpondedRescan); startupDelayBlockDeletionInMs = conf.getLong( DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; @@ -1613,9 +1615,7 @@ public class BlockManager implements BlockStatsMXBean { private void postponeBlock(Block blk) { -if (postponedMisreplicatedBlocks.add(blk)) { - postponedMisreplicatedBlocksCount.incrementAndGet(); -} +postponedMisreplicatedBlocks.add(blk); }
hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.
Repository: hadoop Updated Branches: refs/heads/branch-2.8 afad13f8d -> dd4acebb4 HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp. (cherry picked from commit 0d8a35bd6de5d2a5a9b816ca98f31975e94bd7c6) Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd4acebb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd4acebb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd4acebb Branch: refs/heads/branch-2.8 Commit: dd4acebb4146c7629ce914df6642c5d69b8172d9 Parents: afad13f Author: Kihwal LeeAuthored: Thu Dec 1 12:20:30 2016 -0600 Committer: Kihwal Lee Committed: Thu Dec 1 12:20:30 2016 -0600 -- .../server/blockmanagement/BlockManager.java| 75 +++- 1 file changed, 24 insertions(+), 51 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd4acebb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 0ba01aa..a929c43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -29,6 +29,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -143,7 +144,6 @@ public class BlockManager implements BlockStatsMXBean { private boolean initializedReplQueues; private final AtomicLong excessBlocksCount = new AtomicLong(0L); - private final AtomicLong postponedMisreplicatedBlocksCount = new AtomicLong(0L); private final long startupDelayBlockDeletionInMs; private final BlockReportLeaseManager blockReportLeaseManager; private ObjectName mxBeanName; @@ -178,7 +178,7 @@ public class BlockManager implements BlockStatsMXBean { } /** Used by metrics */ public long getPostponedMisreplicatedBlocksCount() { -return postponedMisreplicatedBlocksCount.get(); +return postponedMisreplicatedBlocks.size(); } /** Used by metrics */ public int getPendingDataNodeMessageCount() { @@ -218,8 +218,10 @@ public class BlockManager implements BlockStatsMXBean { * notified of all block deletions that might have been pending * when the failover happened. */ - private final LightWeightHashSet postponedMisreplicatedBlocks = - new LightWeightHashSet<>(); + private final LinkedHashSet postponedMisreplicatedBlocks = + new LinkedHashSet(); + private final int blocksPerPostpondedRescan; + private final ArrayList rescannedMisreplicatedBlocks; /** * Maps a StorageID to the set of blocks that are "extra" for this @@ -316,6 +318,10 @@ public class BlockManager implements BlockStatsMXBean { datanodeManager = new DatanodeManager(this, namesystem, conf); heartbeatManager = datanodeManager.getHeartbeatManager(); +blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE, +datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan()); +rescannedMisreplicatedBlocks = +new ArrayList(blocksPerPostpondedRescan); startupDelayBlockDeletionInMs = conf.getLong( DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; @@ -1424,9 +1430,7 @@ public class BlockManager implements BlockStatsMXBean { private void postponeBlock(Block blk) { -if (postponedMisreplicatedBlocks.add(blk)) { - postponedMisreplicatedBlocksCount.incrementAndGet(); -} +postponedMisreplicatedBlocks.add(blk); } @@ -2050,39 +2054,14 @@ public class BlockManager implements BlockStatsMXBean { if (getPostponedMisreplicatedBlocksCount() == 0) { return; } -long startTimeRescanPostponedMisReplicatedBlocks = Time.monotonicNow(); namesystem.writeLock(); -long startPostponedMisReplicatedBlocksCount = -getPostponedMisreplicatedBlocksCount(); +long startTime = Time.monotonicNow(); +long startSize = postponedMisreplicatedBlocks.size(); try { - // blocksPerRescan is the configured number of blocks per rescan. - // Randomly
hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.
Repository: hadoop Updated Branches: refs/heads/branch-2 8ffe86f78 -> 0d8a35bd6 HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d8a35bd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d8a35bd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d8a35bd Branch: refs/heads/branch-2 Commit: 0d8a35bd6de5d2a5a9b816ca98f31975e94bd7c6 Parents: 8ffe86f Author: Kihwal LeeAuthored: Thu Dec 1 12:15:15 2016 -0600 Committer: Kihwal Lee Committed: Thu Dec 1 12:15:15 2016 -0600 -- .../server/blockmanagement/BlockManager.java| 76 +++- 1 file changed, 24 insertions(+), 52 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d8a35bd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 954b297..f2805e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -29,6 +29,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -170,7 +171,6 @@ public class BlockManager implements BlockStatsMXBean { private boolean initializedReplQueues; private final AtomicLong excessBlocksCount = new AtomicLong(0L); - private final AtomicLong postponedMisreplicatedBlocksCount = new AtomicLong(0L); private final long startupDelayBlockDeletionInMs; private final BlockReportLeaseManager blockReportLeaseManager; private ObjectName mxBeanName; @@ -205,7 +205,7 @@ public class BlockManager implements BlockStatsMXBean { } /** Used by metrics */ public long getPostponedMisreplicatedBlocksCount() { -return postponedMisreplicatedBlocksCount.get(); +return postponedMisreplicatedBlocks.size(); } /** Used by metrics */ public int getPendingDataNodeMessageCount() { @@ -245,8 +245,10 @@ public class BlockManager implements BlockStatsMXBean { * notified of all block deletions that might have been pending * when the failover happened. */ - private final LightWeightHashSet postponedMisreplicatedBlocks = - new LightWeightHashSet<>(); + private final LinkedHashSet postponedMisreplicatedBlocks = + new LinkedHashSet(); + private final int blocksPerPostpondedRescan; + private final ArrayList rescannedMisreplicatedBlocks; /** * Maps a StorageID to the set of blocks that are "extra" for this @@ -345,7 +347,10 @@ public class BlockManager implements BlockStatsMXBean { datanodeManager = new DatanodeManager(this, namesystem, conf); heartbeatManager = datanodeManager.getHeartbeatManager(); this.blockIdManager = new BlockIdManager(this); - +blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE, +datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan()); +rescannedMisreplicatedBlocks = +new ArrayList(blocksPerPostpondedRescan); startupDelayBlockDeletionInMs = conf.getLong( DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; @@ -1455,9 +1460,7 @@ public class BlockManager implements BlockStatsMXBean { private void postponeBlock(Block blk) { -if (postponedMisreplicatedBlocks.add(blk)) { - postponedMisreplicatedBlocksCount.incrementAndGet(); -} +postponedMisreplicatedBlocks.add(blk); } @@ -2164,39 +2167,14 @@ public class BlockManager implements BlockStatsMXBean { if (getPostponedMisreplicatedBlocksCount() == 0) { return; } -long startTimeRescanPostponedMisReplicatedBlocks = Time.monotonicNow(); namesystem.writeLock(); -long startPostponedMisReplicatedBlocksCount = -getPostponedMisreplicatedBlocksCount(); +long startTime = Time.monotonicNow(); +long startSize = postponedMisreplicatedBlocks.size(); try { - // blocksPerRescan is the configured number of blocks per rescan. - // Randomly select blocksPerRescan consecutive blocks from the HashSet - // when the number of blocks remaining is larger than blocksPerRescan. - // The
hadoop git commit: HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp.
Repository: hadoop Updated Branches: refs/heads/trunk e0fa49234 -> 96c574927 HDFS-8674. Improve performance of postponed block scans. Contributed by Daryn Sharp. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96c57492 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96c57492 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96c57492 Branch: refs/heads/trunk Commit: 96c574927a600d15fab919df1fdc9e07887af6c5 Parents: e0fa492 Author: Kihwal LeeAuthored: Thu Dec 1 12:11:27 2016 -0600 Committer: Kihwal Lee Committed: Thu Dec 1 12:11:27 2016 -0600 -- .../server/blockmanagement/BlockManager.java| 79 ++-- 1 file changed, 24 insertions(+), 55 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/96c57492/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 1b744e7..e60703b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -30,6 +30,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -43,8 +44,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.FutureTask; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - import javax.management.ObjectName; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -101,7 +100,6 @@ import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.util.FoldedTreeSet; -import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.namenode.CacheManager; @@ -184,7 +182,6 @@ public class BlockManager implements BlockStatsMXBean { /** flag indicating whether replication queues have been initialized */ private boolean initializedReplQueues; - private final AtomicLong postponedMisreplicatedBlocksCount = new AtomicLong(0L); private final long startupDelayBlockDeletionInMs; private final BlockReportLeaseManager blockReportLeaseManager; private ObjectName mxBeanName; @@ -219,7 +216,7 @@ public class BlockManager implements BlockStatsMXBean { } /** Used by metrics */ public long getPostponedMisreplicatedBlocksCount() { -return postponedMisreplicatedBlocksCount.get(); +return postponedMisreplicatedBlocks.size(); } /** Used by metrics */ public int getPendingDataNodeMessageCount() { @@ -275,8 +272,10 @@ public class BlockManager implements BlockStatsMXBean { * notified of all block deletions that might have been pending * when the failover happened. */ - private final LightWeightHashSet postponedMisreplicatedBlocks = - new LightWeightHashSet<>(); + private final Set postponedMisreplicatedBlocks = + new LinkedHashSet(); + private final int blocksPerPostpondedRescan; + private final ArrayList rescannedMisreplicatedBlocks; /** * Maps a StorageID to the set of blocks that are "extra" for this @@ -378,7 +377,10 @@ public class BlockManager implements BlockStatsMXBean { datanodeManager = new DatanodeManager(this, namesystem, conf); heartbeatManager = datanodeManager.getHeartbeatManager(); this.blockIdManager = new BlockIdManager(this); - +blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE, +datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan()); +rescannedMisreplicatedBlocks = +new ArrayList(blocksPerPostpondedRescan); startupDelayBlockDeletionInMs = conf.getLong( DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; @@ -1613,9 +1615,7 @@ public class BlockManager implements BlockStatsMXBean { private void postponeBlock(Block blk) { -if (postponedMisreplicatedBlocks.add(blk)) { -