Author: arp
Date: Tue Dec 3 17:33:46 2013
New Revision: 1547492
URL: http://svn.apache.org/r1547492
Log:
Merging r1547121 through r1547473 from trunk to branch HDFS-2832
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
Propchange: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1547121-1547473
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1547492&r1=1547491&r2=1547492&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Dec 3 17:33:46 2013
@@ -421,7 +421,7 @@ Trunk (Unreleased)
HDFS-5562. TestCacheDirectives and TestFsDatasetCache should stub out
native mlock. (Colin McCabe and Akira Ajisaka via wang)
-Release 2.3.0 - UNRELEASED
+Release 2.4.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -643,7 +643,7 @@ Release 2.3.0 - UNRELEASED
HDFS-5533. Symlink delete/create should be treated as DELETE/CREATE in
snapshot diff
report. (Binglin Chang via jing9)
-Release 2.2.1 - UNRELEASED
+Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -4060,6 +4060,12 @@ Release 0.23.10 - UNRELEASED
HDFS-5526. Datanode cannot roll back to previous layout version (kihwal)
+ HDFS-5557. Write pipeline recovery for the last packet in the block may
+ cause rejection of valid replicas. (kihwal)
+
+ HDFS-5558. LeaseManager monitor thread can crash if the last block is
+ complete but another block is not. (kihwal)
+
Release 0.23.9 - 2013-07-08
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1547121-1547473
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1547492&r1=1547491&r2=1547492&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
Tue Dec 3 17:33:46 2013
@@ -844,7 +844,6 @@ public class DFSOutputStream extends FSO
// We also need to set lastAckedSeqno to the end-of-block Packet's
seqno, so that
// a client waiting on close() will be aware that the flush finished.
synchronized (dataQueue) {
- assert dataQueue.size() == 1;
Packet endOfBlockPacket = dataQueue.remove(); // remove the end
of block packet
assert endOfBlockPacket.lastPacketInBlock;
assert lastAckedSeqno == endOfBlockPacket.seqno - 1;
@@ -1056,7 +1055,7 @@ public class DFSOutputStream extends FSO
// set up the pipeline again with the remaining nodes
if (failPacket) { // for testing
- success = createBlockOutputStream(nodes, newGS-1, isRecovery);
+ success = createBlockOutputStream(nodes, newGS, isRecovery);
failPacket = false;
try {
// Give DNs time to send in bad reports. In real situations,
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java?rev=1547492&r1=1547491&r2=1547492&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
Tue Dec 3 17:33:46 2013
@@ -235,6 +235,8 @@ public class BlockInfoUnderConstruction
* @param genStamp The final generation stamp for the block.
*/
public void setGenerationStampAndVerifyReplicas(long genStamp) {
+ // Set the generation stamp for the block.
+ setGenerationStamp(genStamp);
if (replicas == null)
return;
@@ -244,12 +246,9 @@ public class BlockInfoUnderConstruction
if (genStamp != r.getGenerationStamp()) {
r.getExpectedStorageLocation().removeBlock(this);
NameNode.blockStateChangeLog.info("BLOCK* Removing stale replica "
- + "from location: " + r);
+ + "from location: " + r.getExpectedStorageLocation());
}
}
-
- // Set the generation stamp for the block.
- setGenerationStamp(genStamp);
}
/**
@@ -264,6 +263,8 @@ public class BlockInfoUnderConstruction
+ block.getBlockId() + ", expected id = " + getBlockId());
blockUCState = BlockUCState.COMMITTED;
this.set(getBlockId(), block.getNumBytes(), block.getGenerationStamp());
+ // Sort out invalid replicas.
+ setGenerationStampAndVerifyReplicas(block.getGenerationStamp());
}
/**
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1547492&r1=1547491&r2=1547492&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Tue Dec 3 17:33:46 2013
@@ -1555,13 +1555,15 @@ public class BlockManager {
* Besides the block in question, it provides the ReplicaState
* reported by the datanode in the block report.
*/
- private static class StatefulBlockInfo {
+ static class StatefulBlockInfo {
final BlockInfoUnderConstruction storedBlock;
+ final Block reportedBlock;
final ReplicaState reportedState;
StatefulBlockInfo(BlockInfoUnderConstruction storedBlock,
- ReplicaState reportedState) {
+ Block reportedBlock, ReplicaState reportedState) {
this.storedBlock = storedBlock;
+ this.reportedBlock = reportedBlock;
this.reportedState = reportedState;
}
}
@@ -1719,8 +1721,7 @@ public class BlockManager {
// Process the blocks on each queue
for (StatefulBlockInfo b : toUC) {
- addStoredBlockUnderConstruction(b.storedBlock, node,
- storage.getStorageID(), b.reportedState);
+ addStoredBlockUnderConstruction(b, node, storage.getStorageID());
}
for (Block b : toRemove) {
removeStoredBlock(b, node);
@@ -1950,7 +1951,7 @@ assert storedBlock.findDatanode(dn) < 0
if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
toUC.add(new StatefulBlockInfo(
- (BlockInfoUnderConstruction)storedBlock, reportedState));
+ (BlockInfoUnderConstruction)storedBlock, block, reportedState));
return storedBlock;
}
@@ -2120,18 +2121,18 @@ assert storedBlock.findDatanode(dn) < 0
return false;
}
}
-
- void addStoredBlockUnderConstruction(
- BlockInfoUnderConstruction block,
- DatanodeDescriptor node, String storageID,
- ReplicaState reportedState)
- throws IOException {
- block.addReplicaIfNotPresent(node.getStorageInfo(storageID), block,
reportedState);
- if (reportedState == ReplicaState.FINALIZED && block.findDatanode(node) <
0) {
+
+ void addStoredBlockUnderConstruction(StatefulBlockInfo ucBlock,
+ DatanodeDescriptor node, String storageID) throws IOException {
+ BlockInfoUnderConstruction block = ucBlock.storedBlock;
+ block.addReplicaIfNotPresent(node.getStorageInfo(storageID),
+ ucBlock.reportedBlock, ucBlock.reportedState);
+
+ if (ucBlock.reportedState == ReplicaState.FINALIZED &&
block.findDatanode(node) < 0) {
addStoredBlock(block, node, storageID, null, true);
}
- }
-
+ }
+
/**
* Faster version of
* {@link #addStoredBlock(BlockInfo, DatanodeDescriptor, String,
DatanodeDescriptor, boolean)}
@@ -2702,7 +2703,7 @@ assert storedBlock.findDatanode(dn) < 0
: "The block should be only in one of the lists.";
for (StatefulBlockInfo b : toUC) {
- addStoredBlockUnderConstruction(b.storedBlock, node, storageID,
b.reportedState);
+ addStoredBlockUnderConstruction(b, node, storageID);
}
long numBlocksLogged = 0;
for (BlockInfo b : toAdd) {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1547492&r1=1547491&r2=1547492&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Tue Dec 3 17:33:46 2013
@@ -2923,6 +2923,12 @@ public class FSNamesystem implements Nam
}
throw lee;
}
+ // Check the state of the penultimate block. It should be completed
+ // before attempting to complete the last one.
+ if (!checkFileProgress(pendingFile, false)) {
+ return false;
+ }
+
// commit the last block and complete it if it has minimum replicas
commitOrCompleteLastBlock(pendingFile, last);
@@ -2991,7 +2997,7 @@ public class FSNamesystem implements Nam
//
BlockInfo b = v.getPenultimateBlock();
if (b != null && !b.isComplete()) {
- LOG.info("BLOCK* checkFileProgress: " + b
+ LOG.warn("BLOCK* checkFileProgress: " + b
+ " has not reached minimal replication "
+ blockManager.minReplication);
return false;
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java?rev=1547492&r1=1547491&r2=1547492&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
Tue Dec 3 17:33:46 2013
@@ -139,16 +139,10 @@ public class TestClientProtocolForPipeli
Path file = new Path("dataprotocol1.dat");
Mockito.when(faultInjector.failPacket()).thenReturn(true);
- try {
- DFSTestUtil.createFile(fileSys, file, 1L, (short)numDataNodes, 0L);
- } catch (IOException e) {
- // completeFile() should fail.
- Assert.assertTrue(e.getMessage().startsWith("Unable to close file"));
- return;
- }
+ DFSTestUtil.createFile(fileSys, file, 68000000L, (short)numDataNodes,
0L);
- // At this point, NN let data corruption to happen.
- // Before failing test, try reading the file. It should fail.
+ // At this point, NN should have accepted only valid replicas.
+ // Read should succeed.
FSDataInputStream in = fileSys.open(file);
try {
int c = in.read();
@@ -158,8 +152,6 @@ public class TestClientProtocolForPipeli
Assert.fail("Block is missing because the file was closed with"
+ " corrupt replicas.");
}
- Assert.fail("The file was closed with corrupt replicas, but read still"
- + " works!");
} finally {
DFSClientFaultInjector.instance = oldInjector;
if (cluster != null) {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1547492&r1=1547491&r2=1547492&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
Tue Dec 3 17:33:46 2013
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.StatefulBlockInfo;
import org.apache.hadoop.hdfs.server.namenode.FSClusterStats;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
@@ -1119,8 +1120,9 @@ public class TestReplicationPolicy {
// Adding this block will increase its current replication, and that will
// remove it from the queue.
- bm.addStoredBlockUnderConstruction(info,
- TestReplicationPolicy.dataNodes[0], "STORAGE", ReplicaState.FINALIZED);
+ bm.addStoredBlockUnderConstruction(new StatefulBlockInfo(info, info,
+ ReplicaState.FINALIZED), TestReplicationPolicy.dataNodes[0],
+ "STORAGE");
// Choose 1 block from UnderReplicatedBlocks. Then it should pick 1 block
// from QUEUE_VERY_UNDER_REPLICATED.
