Author: arp
Date: Fri Jan 31 21:30:07 2014
New Revision: 1563261
URL: http://svn.apache.org/r1563261
Log:
HDFS-5153. Merging r1563254 from trunk to branch-2. (Arpit Agarwal)
Added:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
- copied unchanged from r1563254,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDnRespectsBlockReportSplitThreshold.java
- copied unchanged from r1563254,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDnRespectsBlockReportSplitThreshold.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestNNHandlesBlockReportPerStorage.java
- copied unchanged from r1563254,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestNNHandlesBlockReportPerStorage.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestNNHandlesCombinedBlockReport.java
- copied unchanged from r1563254,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestNNHandlesCombinedBlockReport.java
Removed:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1563261&r1=1563260&r2=1563261&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Fri Jan 31 21:30:07 2014
@@ -11,6 +11,9 @@ Release 2.4.0 - UNRELEASED
HDFS-5781. Use an array to record the mapping between FSEditLogOpCode and
the corresponding byte value. (jing9)
+ HDFS-5153. Datanode should send block reports for each storage in a
+ separate message. (Arpit Agarwal)
+
OPTIMIZATIONS
HDFS-5790. LeaseManager.findPath is very slow when many leases need
recovery
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1563261&r1=1563260&r2=1563261&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Fri Jan 31 21:30:07 2014
@@ -399,6 +399,8 @@ public class DFSConfigKeys extends Commo
public static final long DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT = 60 * 60
* 1000;
public static final String DFS_BLOCKREPORT_INITIAL_DELAY_KEY =
"dfs.blockreport.initialDelay";
public static final int DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT = 0;
+ public static final String DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY =
"dfs.blockreport.split.threshold";
+ public static final long DFS_BLOCKREPORT_SPLIT_THRESHOLD_DEFAULT = 1000 *
1000;
public static final String DFS_CACHEREPORT_INTERVAL_MSEC_KEY =
"dfs.cachereport.intervalMsec";
public static final long DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT = 10 *
1000;
public static final String DFS_BLOCK_INVALIDATE_LIMIT_KEY =
"dfs.block.invalidate.limit";
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1563261&r1=1563260&r2=1563261&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Fri Jan 31 21:30:07 2014
@@ -1621,15 +1621,19 @@ public class BlockManager {
/**
* The given storage is reporting all its blocks.
* Update the (storage-->block list) and (block-->storage list) maps.
+ *
+ * @return true if all known storages of the given DN have finished
reporting.
+ * @throws IOException
*/
- public void processReport(final DatanodeID nodeID,
+ public boolean processReport(final DatanodeID nodeID,
final DatanodeStorage storage, final String poolId,
final BlockListAsLongs newReport) throws IOException {
namesystem.writeLock();
final long startTime = Time.now(); //after acquiring write lock
final long endTime;
+ DatanodeDescriptor node;
try {
- final DatanodeDescriptor node = datanodeManager.getDatanode(nodeID);
+ node = datanodeManager.getDatanode(nodeID);
if (node == null || !node.isAlive) {
throw new IOException(
"ProcessReport from dead or unregistered node: " + nodeID);
@@ -1637,13 +1641,21 @@ public class BlockManager {
// To minimize startup time, we discard any second (or later) block
reports
// that we receive while still in startup phase.
- final DatanodeStorageInfo storageInfo = node.updateStorage(storage);
+ DatanodeStorageInfo storageInfo =
node.getStorageInfo(storage.getStorageID());
+
+ if (storageInfo == null) {
+ // We handle this for backwards compatibility.
+ storageInfo = node.updateStorage(storage);
+ LOG.warn("Unknown storageId " + storage.getStorageID() +
+ ", updating storageMap. This indicates a buggy " +
+ "DataNode that isn't heartbeating correctly.");
+ }
if (namesystem.isInStartupSafeMode()
&& storageInfo.getBlockReportCount() > 0) {
blockLog.info("BLOCK* processReport: "
+ "discarded non-initial block report from " + nodeID
+ " because namenode still in startup phase");
- return;
+ return !node.hasStaleStorages();
}
if (storageInfo.numBlocks() == 0) {
@@ -1660,7 +1672,7 @@ public class BlockManager {
storageInfo.receivedBlockReport();
if (staleBefore && !storageInfo.areBlockContentsStale()) {
LOG.info("BLOCK* processReport: Received first block report from "
- + node + " after starting up or becoming active. Its block "
+ + storage + " after starting up or becoming active. Its block "
+ "contents are no longer considered stale");
rescanPostponedMisreplicatedBlocks();
}
@@ -1675,9 +1687,10 @@ public class BlockManager {
if (metrics != null) {
metrics.addBlockReport((int) (endTime - startTime));
}
- blockLog.info("BLOCK* processReport: from "
- + nodeID + ", blocks: " + newReport.getNumberOfBlocks()
+ blockLog.info("BLOCK* processReport: from storage " +
storage.getStorageID()
+ + " node " + nodeID + ", blocks: " + newReport.getNumberOfBlocks()
+ ", processing time: " + (endTime - startTime) + " msecs");
+ return !node.hasStaleStorages();
}
/**
@@ -1832,7 +1845,7 @@ public class BlockManager {
Collection<BlockToMarkCorrupt> toCorrupt, // add to corrupt replicas list
Collection<StatefulBlockInfo> toUC) { // add to under-construction list
- final DatanodeStorageInfo storageInfo = dn.updateStorage(storage);
+ final DatanodeStorageInfo storageInfo =
dn.getStorageInfo(storage.getStorageID());
// place a delimiter in the list which separates blocks
// that have been reported from those that have not
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java?rev=1563261&r1=1563260&r2=1563261&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
Fri Jan 31 21:30:07 2014
@@ -259,6 +259,17 @@ public class DatanodeDescriptor extends
}
}
+ boolean hasStaleStorages() {
+ synchronized (storageMap) {
+ for (DatanodeStorageInfo storage : storageMap.values()) {
+ if (storage.areBlockContentsStale()) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
/**
* Remove block from the list of blocks belonging to the data-node. Remove
* data-node from the block.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java?rev=1563261&r1=1563260&r2=1563261&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
Fri Jan 31 21:30:07 2014
@@ -22,11 +22,9 @@ import static org.apache.hadoop.util.Tim
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketTimeoutException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+import com.google.common.base.Joiner;
import org.apache.commons.logging.Log;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
@@ -437,75 +435,100 @@ class BPServiceActor implements Runnable
/**
* Report the list blocks to the Namenode
+ * @return DatanodeCommands returned by the NN. May be null.
* @throws IOException
*/
- DatanodeCommand blockReport() throws IOException {
+ List<DatanodeCommand> blockReport() throws IOException {
// send block report if timer has expired.
- DatanodeCommand cmd = null;
- long startTime = now();
- if (startTime - lastBlockReport > dnConf.blockReportInterval) {
+ final long startTime = now();
+ if (startTime - lastBlockReport <= dnConf.blockReportInterval) {
+ return null;
+ }
+
+ ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
- // Flush any block information that precedes the block report. Otherwise
- // we have a chance that we will miss the delHint information
- // or we will report an RBW replica after the BlockReport already reports
- // a FINALIZED one.
- reportReceivedDeletedBlocks();
-
- // Send one block report per known storage.
-
- // Create block report
- long brCreateStartTime = now();
- long totalBlockCount = 0;
-
- Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
- dn.getFSDataset().getBlockReports(bpos.getBlockPoolId());
-
- // Send block report
- long brSendStartTime = now();
- StorageBlockReport[] reports =
- new StorageBlockReport[perVolumeBlockLists.size()];
-
- int i = 0;
- for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair :
perVolumeBlockLists.entrySet()) {
- DatanodeStorage dnStorage = kvPair.getKey();
- BlockListAsLongs blockList = kvPair.getValue();
- totalBlockCount += blockList.getNumberOfBlocks();
-
- reports[i++] =
- new StorageBlockReport(
- dnStorage, blockList.getBlockListAsLongs());
- }
-
- cmd = bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(),
reports);
-
- // Log the block report processing stats from Datanode perspective
- long brSendCost = now() - brSendStartTime;
- long brCreateCost = brSendStartTime - brCreateStartTime;
- dn.getMetrics().addBlockReport(brSendCost);
- LOG.info("BlockReport of " + totalBlockCount
- + " blocks took " + brCreateCost + " msec to generate and "
- + brSendCost + " msecs for RPC and NN processing");
-
- // If we have sent the first block report, then wait a random
- // time before we start the periodic block reports.
- if (resetBlockReportTime) {
- lastBlockReport = startTime -
DFSUtil.getRandom().nextInt((int)(dnConf.blockReportInterval));
- resetBlockReportTime = false;
- } else {
- /* say the last block report was at 8:20:14. The current report
- * should have started around 9:20:14 (default 1 hour interval).
- * If current time is :
- * 1) normal like 9:20:18, next report should be at 10:20:14
- * 2) unexpected like 11:35:43, next report should be at 12:20:14
- */
- lastBlockReport += (now() - lastBlockReport) /
- dnConf.blockReportInterval * dnConf.blockReportInterval;
+ // Flush any block information that precedes the block report. Otherwise
+ // we have a chance that we will miss the delHint information
+ // or we will report an RBW replica after the BlockReport already reports
+ // a FINALIZED one.
+ reportReceivedDeletedBlocks();
+ lastDeletedReport = startTime;
+
+ long brCreateStartTime = now();
+ Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
+ dn.getFSDataset().getBlockReports(bpos.getBlockPoolId());
+
+ // Convert the reports to the format expected by the NN.
+ int i = 0;
+ int totalBlockCount = 0;
+ StorageBlockReport reports[] =
+ new StorageBlockReport[perVolumeBlockLists.size()];
+
+ for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair :
perVolumeBlockLists.entrySet()) {
+ BlockListAsLongs blockList = kvPair.getValue();
+ reports[i++] = new StorageBlockReport(
+ kvPair.getKey(), blockList.getBlockListAsLongs());
+ totalBlockCount += blockList.getNumberOfBlocks();
+ }
+
+ // Send the reports to the NN.
+ int numReportsSent;
+ long brSendStartTime = now();
+ if (totalBlockCount < dnConf.blockReportSplitThreshold) {
+ // Below split threshold, send all reports in a single message.
+ numReportsSent = 1;
+ DatanodeCommand cmd =
+ bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(),
reports);
+ if (cmd != null) {
+ cmds.add(cmd);
+ }
+ } else {
+ // Send one block report per message.
+ numReportsSent = i;
+ for (StorageBlockReport report : reports) {
+ StorageBlockReport singleReport[] = { report };
+ DatanodeCommand cmd = bpNamenode.blockReport(
+ bpRegistration, bpos.getBlockPoolId(), singleReport);
+ if (cmd != null) {
+ cmds.add(cmd);
+ }
}
- LOG.info("sent block report, processed command:" + cmd);
}
- return cmd;
+
+ // Log the block report processing stats from Datanode perspective
+ long brSendCost = now() - brSendStartTime;
+ long brCreateCost = brSendStartTime - brCreateStartTime;
+ dn.getMetrics().addBlockReport(brSendCost);
+ LOG.info("Sent " + numReportsSent + " blockreports " + totalBlockCount +
+ " blocks total. Took " + brCreateCost +
+ " msec to generate and " + brSendCost +
+ " msecs for RPC and NN processing. " +
+ " Got back commands " +
+ (cmds.size() == 0 ? "none" : Joiner.on("; ").join(cmds)));
+
+ scheduleNextBlockReport(startTime);
+ return cmds.size() == 0 ? null : cmds;
+ }
+
+ private void scheduleNextBlockReport(long previousReportStartTime) {
+ // If we have sent the first set of block reports, then wait a random
+ // time before we start the periodic block reports.
+ if (resetBlockReportTime) {
+ lastBlockReport = previousReportStartTime -
+ DFSUtil.getRandom().nextInt((int)(dnConf.blockReportInterval));
+ resetBlockReportTime = false;
+ } else {
+ /* say the last block report was at 8:20:14. The current report
+ * should have started around 9:20:14 (default 1 hour interval).
+ * If current time is :
+ * 1) normal like 9:20:18, next report should be at 10:20:14
+ * 2) unexpected like 11:35:43, next report should be at 12:20:14
+ */
+ lastBlockReport += (now() - lastBlockReport) /
+ dnConf.blockReportInterval * dnConf.blockReportInterval;
+ }
}
-
+
DatanodeCommand cacheReport() throws IOException {
// If caching is disabled, do not send a cache report
if (dn.getFSDataset().getCacheCapacity() == 0) {
@@ -513,7 +536,7 @@ class BPServiceActor implements Runnable
}
// send cache report if timer has expired.
DatanodeCommand cmd = null;
- long startTime = Time.monotonicNow();
+ final long startTime = Time.monotonicNow();
if (startTime - lastCacheReport > dnConf.cacheReportInterval) {
if (LOG.isDebugEnabled()) {
LOG.debug("Sending cacheReport from service actor: " + this);
@@ -613,7 +636,7 @@ class BPServiceActor implements Runnable
//
while (shouldRun()) {
try {
- long startTime = now();
+ final long startTime = now();
//
// Every so often, send heartbeat or block-report
@@ -659,10 +682,10 @@ class BPServiceActor implements Runnable
lastDeletedReport = startTime;
}
- DatanodeCommand cmd = blockReport();
- processCommand(new DatanodeCommand[]{ cmd });
+ List<DatanodeCommand> cmds = blockReport();
+ processCommand(cmds == null ? null : cmds.toArray(new
DatanodeCommand[cmds.size()]));
- cmd = cacheReport();
+ DatanodeCommand cmd = cacheReport();
processCommand(new DatanodeCommand[]{ cmd });
// Now safe to start scanning the block pool.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java?rev=1563261&r1=1563260&r2=1563261&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
Fri Jan 31 21:30:07 2014
@@ -23,6 +23,8 @@ import static org.apache.hadoop.hdfs.DFS
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY;
+import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY;
+import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_SPLIT_THRESHOLD_DEFAULT;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_KEY;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
@@ -70,6 +72,7 @@ public class DNConf {
final long readaheadLength;
final long heartBeatInterval;
final long blockReportInterval;
+ final long blockReportSplitThreshold;
final long deleteReportInterval;
final long initialBlockReportDelay;
final long cacheReportInterval;
@@ -117,6 +120,8 @@ public class DNConf {
DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
this.blockReportInterval = conf.getLong(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT);
+ this.blockReportSplitThreshold =
conf.getLong(DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY,
+
DFS_BLOCKREPORT_SPLIT_THRESHOLD_DEFAULT);
this.cacheReportInterval = conf.getLong(DFS_CACHEREPORT_INTERVAL_MSEC_KEY,
DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT);
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1563261&r1=1563260&r2=1563261&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
Fri Jan 31 21:30:07 2014
@@ -978,13 +978,18 @@ class NameNodeRpcServer implements Namen
+ "from " + nodeReg + ", reports.length=" + reports.length);
}
final BlockManager bm = namesystem.getBlockManager();
+ boolean hasStaleStorages = true;
for(StorageBlockReport r : reports) {
final BlockListAsLongs blocks = new BlockListAsLongs(r.getBlocks());
- bm.processReport(nodeReg, r.getStorage(), poolId, blocks);
+ hasStaleStorages = bm.processReport(nodeReg, r.getStorage(), poolId,
blocks);
}
- if (nn.getFSImage().isUpgradeFinalized() && !nn.isStandbyState())
+ if (nn.getFSImage().isUpgradeFinalized() &&
+ !nn.isStandbyState() &&
+ !hasStaleStorages) {
return new FinalizeCommand(poolId);
+ }
+
return null;
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1563261&r1=1563260&r2=1563261&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
Fri Jan 31 21:30:07 2014
@@ -483,6 +483,20 @@
</property>
<property>
+ <name>dfs.blockreport.split.threshold</name>
+ <value>1000000</value>
+ <description>If the number of blocks on the DataNode is below this
+ threshold then it will send block reports for all Storage Directories
+ in a single message.
+
+ If the number of blocks exceeds this threshold then the DataNode will
+ send block reports for each Storage Directory in separate messages.
+
+ Set to zero to always split.
+ </description>
+</property>
+
+<property>
<name>dfs.datanode.directoryscan.interval</name>
<value>21600</value>
<description>Interval in seconds for Datanode to scan data directories and
