Author: arp
Date: Tue Sep 3 23:15:28 2013
New Revision: 1519882
URL: http://svn.apache.org/r1519882
Log:
Merge r1518851 from trunk (reconcile branch conflicts with the fix for
HDFS-5077)
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
Propchange: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1518851
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1519882&r1=1519881&r2=1519882&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Sep 3 23:15:28 2013
@@ -401,6 +401,9 @@ Release 2.1.1-beta - UNRELEASED
HDFS-5132. Deadlock in NameNode between SafeModeMonitor#run and
DatanodeManager#handleHeartbeat. (kihwal)
+ HDFS-5077. NPE in FSNamesystem.commitBlockSynchronization().
+ (Plamen Jeliazkov via shv)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1518851
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1519882&r1=1519881&r2=1519882&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Tue Sep 3 23:15:28 2013
@@ -3326,7 +3326,7 @@ public class FSNamesystem implements Nam
boolean isFileClosed(String src)
throws AccessControlException, UnresolvedLinkException,
StandbyException, IOException {
- FSPermissionChecker pc = getPermissionChecker();
+ FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.READ);
readLock();
try {
@@ -3784,18 +3784,39 @@ public class FSNamesystem implements Nam
// find the DatanodeDescriptor objects
// There should be no locations in the blockManager till now because
the
// file is underConstruction
- final DatanodeStorageInfo[] storages =
blockManager.getDatanodeManager()
- .getDatanodeStorageInfos(newtargets, newtargetstorages);
- if (closeFile && storages != null) {
+ ArrayList<DatanodeDescriptor> trimmedTargets =
+ new ArrayList<DatanodeDescriptor>(newtargets.length);
+ ArrayList<String> trimmedStorages =
+ new ArrayList<String>(newtargets.length);
+ if (newtargets.length > 0) {
+ for (int i = 0; i < newtargets.length; ++i) {
+ // try to get targetNode
+ DatanodeDescriptor targetNode =
+ blockManager.getDatanodeManager().getDatanode(newtargets[i]);
+ if (targetNode != null) {
+ trimmedTargets.add(targetNode);
+ trimmedStorages.add(newtargetstorages[i]);
+ } else if (LOG.isDebugEnabled()) {
+ LOG.debug("DatanodeDescriptor (=" + newtargets[i] + ") not
found");
+ }
+ }
+ }
+ if ((closeFile) && !trimmedTargets.isEmpty()) {
// the file is getting closed. Insert block locations into
blockManager.
// Otherwise fsck will report these blocks as MISSING, especially if
the
// blocksReceived from Datanodes take a long time to arrive.
- for (int i = 0; i < storages.length; i++) {
- storages[i].addBlock(storedBlock);
+ for (int i = 0; i < trimmedTargets.size(); i++) {
+ trimmedTargets.get(i).addBlock(
+ trimmedStorages.get(i), storedBlock);
}
}
+
// add pipeline locations into the INodeUnderConstruction
- pendingFile.setLastBlock(storedBlock, storages);
+ DatanodeStorageInfo[] trimmedStorageInfos =
+ blockManager.getDatanodeManager().getDatanodeStorageInfos(
+ trimmedTargets.toArray(new DatanodeID[trimmedTargets.size()]),
+ trimmedStorages.toArray(new String[trimmedStorages.size()]));
+ pendingFile.setLastBlock(storedBlock, trimmedStorageInfos);
}
if (closeFile) {
@@ -5823,7 +5844,7 @@ public class FSNamesystem implements Nam
* @throws IOException
*/
Collection<CorruptFileBlockInfo> listCorruptFileBlocks(String path,
- String[] cookieTab) throws IOException {
+ String[] cookieTab) throws IOException {
checkSuperuserPrivilege();
checkOperation(OperationCategory.READ);
readLock();
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1518851
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1518851
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1518851
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1518851
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1518851
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java?rev=1519882&r1=1519881&r2=1519882&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
Tue Sep 3 23:15:28 2013
@@ -169,4 +169,23 @@ public class TestCommitBlockSynchronizat
namesystemSpy.commitBlockSynchronization(
lastBlock, genStamp, length, true, false, newTargets, null);
}
+
+ @Test
+ public void testCommitBlockSynchronizationWithCloseAndNonExistantTarget()
+ throws IOException {
+ INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class);
+ Block block = new Block(blockId, length, genStamp);
+ FSNamesystem namesystemSpy = makeNameSystemSpy(block, file);
+ DatanodeID[] newTargets = new DatanodeID[]{
+ new DatanodeID("0.0.0.0", "nonexistantHost", "1", 0, 0, 0)};
+
+ ExtendedBlock lastBlock = new ExtendedBlock();
+ namesystemSpy.commitBlockSynchronization(
+ lastBlock, genStamp, length, true,
+ false, newTargets, null);
+
+ // Repeat the call to make sure it returns true
+ namesystemSpy.commitBlockSynchronization(
+ lastBlock, genStamp, length, true, false, newTargets, null);
+ }
}
