Author: cmccabe
Date: Tue Jul 15 18:17:35 2014
New Revision: 1610791
URL: http://svn.apache.org/r1610791
Log:
HDFS-5809. BlockPoolSliceScanner and high speed hdfs appending make datanode to
drop into infinite loop (cmccabe)
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1610790
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1610790
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1610791&r1=1610790&r2=1610791&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Jul 15 18:17:35 2014
@@ -56,6 +56,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6678. MiniDFSCluster may still be partially running after
initialization
fails. (cnauroth)
+ HDFS-5809. BlockPoolSliceScanner and high speed hdfs appending make
+ datanode to drop into infinite loop (cmccabe)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1610790
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java?rev=1610791&r1=1610790&r2=1610791&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
Tue Jul 15 18:17:35 2014
@@ -310,18 +310,11 @@ class BlockPoolSliceScanner {
}
}
- private synchronized void updateScanStatus(Block block,
+ private synchronized void updateScanStatus(BlockScanInfo info,
ScanType type,
boolean scanOk) {
- BlockScanInfo info = blockMap.get(block);
-
- if ( info != null ) {
- delBlockInfo(info);
- } else {
- // It might already be removed. Thats ok, it will be caught next time.
- info = new BlockScanInfo(block);
- }
-
+ delBlockInfo(info);
+
long now = Time.monotonicNow();
info.lastScanType = type;
info.lastScanTime = now;
@@ -334,8 +327,8 @@ class BlockPoolSliceScanner {
}
if (verificationLog != null) {
- verificationLog.append(now, block.getGenerationStamp(),
- block.getBlockId());
+ verificationLog.append(now, info.getGenerationStamp(),
+ info.getBlockId());
}
}
@@ -434,11 +427,13 @@ class BlockPoolSliceScanner {
totalTransientErrors++;
}
- updateScanStatus(block.getLocalBlock(), ScanType.VERIFICATION_SCAN,
true);
+ updateScanStatus((BlockScanInfo)block.getLocalBlock(),
+ ScanType.VERIFICATION_SCAN, true);
return;
} catch (IOException e) {
- updateScanStatus(block.getLocalBlock(), ScanType.VERIFICATION_SCAN,
false);
+ updateScanStatus((BlockScanInfo)block.getLocalBlock(),
+ ScanType.VERIFICATION_SCAN, false);
// If the block does not exists anymore, then its not an error
if (!dataset.contains(block)) {
@@ -497,7 +492,7 @@ class BlockPoolSliceScanner {
// Picks one block and verifies it
private void verifyFirstBlock() {
- Block block = null;
+ BlockScanInfo block = null;
synchronized (this) {
if (!blockInfoSet.isEmpty()) {
block = blockInfoSet.first();
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java?rev=1610791&r1=1610790&r2=1610791&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
Tue Jul 15 18:17:35 2014
@@ -116,7 +116,8 @@ public class DataNodeTestUtils {
public static void runBlockScannerForBlock(DataNode dn, ExtendedBlock b) {
BlockPoolSliceScanner bpScanner = getBlockPoolScanner(dn, b);
- bpScanner.verifyBlock(b);
+ bpScanner.verifyBlock(new ExtendedBlock(b.getBlockPoolId(),
+ new BlockPoolSliceScanner.BlockScanInfo(b.getLocalBlock())));
}
private static BlockPoolSliceScanner getBlockPoolScanner(DataNode dn,
