Author: atm
Date: Thu Jun 19 05:20:23 2014
New Revision: 1603713
URL: http://svn.apache.org/r1603713
Log:
HDFS-6563. NameNode cannot save fsimage in certain circumstances when snapshots
are in use. Contributed by Aaron T. Myers.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1603713&r1=1603712&r2=1603713&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Thu Jun 19 05:20:23 2014
@@ -415,6 +415,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6553. Add missing DeprecationDeltas for NFS Kerberos configurations
(Stephen Chu via brandonli)
+ HDFS-6563. NameNode cannot save fsimage in certain circumstances when
+ snapshots are in use. (atm)
+
BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS
HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via
umamahesh)
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java?rev=1603713&r1=1603712&r2=1603713&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
Thu Jun 19 05:20:23 2014
@@ -530,8 +530,10 @@ public final class FSImageFormatPBINode
INodeSection.INodeFile.Builder b = buildINodeFile(n,
parent.getSaverContext());
- for (Block block : n.getBlocks()) {
- b.addBlocks(PBHelper.convert(block));
+ if (n.getBlocks() != null) {
+ for (Block block : n.getBlocks()) {
+ b.addBlocks(PBHelper.convert(block));
+ }
}
FileUnderConstructionFeature uc = n.getFileUnderConstructionFeature();
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java?rev=1603713&r1=1603712&r2=1603713&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
Thu Jun 19 05:20:23 2014
@@ -159,7 +159,7 @@ public class FileWithSnapshotFeature imp
// resize the array.
final BlockInfo[] newBlocks;
if (n == 0) {
- newBlocks = null;
+ newBlocks = BlockInfo.EMPTY_ARRAY;
} else {
newBlocks = new BlockInfo[n];
System.arraycopy(oldBlocks, 0, newBlocks, 0, n);
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java?rev=1603713&r1=1603712&r2=1603713&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
Thu Jun 19 05:20:23 2014
@@ -28,12 +28,14 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -396,4 +398,39 @@ public class TestSnapshotBlocksMap {
assertEquals(1, blks.length);
assertEquals(BLOCKSIZE, blks[0].getNumBytes());
}
+
+ /**
+ * Make sure that a delete of a non-zero-length file which results in a
+ * zero-length file in a snapshot works.
+ */
+ @Test
+ public void testDeletionOfLaterBlocksWithZeroSizeFirstBlock() throws
Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final byte[] testData = "foo bar baz".getBytes();
+
+ // Create a zero-length file.
+ DFSTestUtil.createFile(hdfs, bar, 0, REPLICATION, 0L);
+ assertEquals(0,
fsdir.getINode4Write(bar.toString()).asFile().getBlocks().length);
+
+ // Create a snapshot that includes that file.
+ SnapshotTestHelper.createSnapshot(hdfs, foo, "s0");
+
+ // Extend that file.
+ FSDataOutputStream out = hdfs.append(bar);
+ out.write(testData);
+ out.close();
+ INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
+ BlockInfo[] blks = barNode.getBlocks();
+ assertEquals(1, blks.length);
+ assertEquals(testData.length, blks[0].getNumBytes());
+
+ // Delete the file.
+ hdfs.delete(bar, true);
+
+ // Now make sure that the NN can still save an fsimage successfully.
+ cluster.getNameNode().getRpcServer().setSafeMode(
+ SafeModeAction.SAFEMODE_ENTER, false);
+ cluster.getNameNode().getRpcServer().saveNamespace();
+ }
}
