Repository: hadoop Updated Branches: refs/heads/branch-1 2dff54ebc -> a8eee1fd0
HDFS-7503. Namenode restart after large deletions can cause slow processReport. (Arpit Agarwal) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8eee1fd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8eee1fd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8eee1fd Branch: refs/heads/branch-1 Commit: a8eee1fd01ddad153388e269b0d1381433864a82 Parents: 2dff54e Author: arp <[email protected]> Authored: Wed Dec 10 23:51:45 2014 -0800 Committer: arp <[email protected]> Committed: Wed Dec 10 23:51:45 2014 -0800 ---------------------------------------------------------------------- CHANGES.txt | 3 +++ .../hadoop/hdfs/server/namenode/FSNamesystem.java | 12 +++++------- .../apache/hadoop/hdfs/server/namenode/NameNode.java | 12 +++++++++++- 3 files changed, 19 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8eee1fd/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 0bc2773..59bef6c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -256,6 +256,9 @@ Release 1.3.0 - unreleased MAPREDUCE-6170. TestUlimit failure on JDK8. (bc Wong via kasha) + HDFS-7503. Namenode restart after large deletions can cause slow + processReport (Arpit Agarwal) + Release 1.2.2 - unreleased INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8eee1fd/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 8a95a75..f9b9d7d 100644 --- a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3934,9 +3934,9 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt * The given node is reporting all its blocks. Use this info to * update the (machine-->blocklist) and (block-->machinelist) tables. */ - public synchronized void processReport(DatanodeID nodeID, - BlockListAsLongs newReport - ) throws IOException { + public synchronized Collection<Block> processReport( + DatanodeID nodeID, + BlockListAsLongs newReport) throws IOException { long startTime = now(); if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("BLOCK* processReport: " @@ -3961,7 +3961,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt NameNode.stateChangeLog.info("BLOCK* processReport: " + "discarded non-initial block report from " + nodeID.getName() + " because namenode still in startup phase"); - return; + return null; } // @@ -3980,9 +3980,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt addStoredBlock(b, node, null); } for (Block b : toInvalidate) { - NameNode.stateChangeLog.info("BLOCK* processReport: " - + b + " on " + node.getName() + " size " + b.getNumBytes() - + " does not belong to any file"); addToInvalidates(b, node); } long endTime = now(); @@ -3991,6 +3988,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt + nodeID.getName() + ", blocks: " + newReport.getNumberOfBlocks() + ", processing time: " + (endTime - startTime) + " msecs"); node.processedBlockReport(); + return toInvalidate; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8eee1fd/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java ---------------------------------------------------------------------- diff --git a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 268d2e3..be0089c 100644 --- a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -1130,7 +1130,17 @@ public class NameNode implements ClientProtocol, DatanodeProtocol, stateChangeLog.debug("*BLOCK* NameNode.blockReport: " +"from "+nodeReg.getName()+" "+blist.getNumberOfBlocks() +" blocks"); - namesystem.processReport(nodeReg, blist); + Collection<Block> blocksInvalidated = + namesystem.processReport(nodeReg, blist); + + final String node = nodeReg.toString(); + if (blocksInvalidated != null) { + for (Block b : blocksInvalidated) { + stateChangeLog.info("BLOCK* processReport: " + + b + " on " + node + " size " + b.getNumBytes() + + " does not belong to any file"); + } + } if (getFSImage().isUpgradeFinalized()) return DatanodeCommand.FINALIZE; return null;
