HDFS-12638. Delete copy-on-truncate block along with the original block, when deleting a file being truncated. Contributed by Konstantin Shvachko.
(cherry picked from commit 60fd0d7fd73198fd610e59d1a4cd007c5fcc7205) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85eb32b1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85eb32b1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85eb32b1 Branch: refs/heads/branch-2.7 Commit: 85eb32b12d2ce7285433c77ea131dc7a964d6ef9 Parents: 7252e18 Author: Konstantin V Shvachko <[email protected]> Authored: Thu Nov 30 18:18:09 2017 -0800 Committer: Konstantin V Shvachko <[email protected]> Committed: Thu Nov 30 19:22:01 2017 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++ .../hadoop/hdfs/server/namenode/INode.java | 17 +++++++- .../hdfs/server/namenode/TestFileTruncate.java | 42 ++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/85eb32b1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 207611d..156a354 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -48,6 +48,12 @@ Release 2.7.5 - UNRELEASED HDFS-9003. ForkJoin thread pool leaks. (Kihwal Lee via jing9) + HDFS-12832. INode.getFullPathName may throw ArrayIndexOutOfBoundsException + lead to NameNode exit. (shv) + + HDFS-12638. Delete copy-on-truncate block along with the original block, + when deleting a file being truncated. (shv) + Release 2.7.4 - 2017-08-04 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/85eb32b1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index e16fc7d..b23b989 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -30,6 +30,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.Block; @@ -869,9 +871,22 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> { * {@link BlocksMapUpdateInfo#toDeleteList} * @param toDelete the to-be-deleted block */ - public void addDeleteBlock(Block toDelete) { + public void addDeleteBlock(BlockInfoContiguous toDelete) { assert toDelete != null : "toDelete is null"; toDeleteList.add(toDelete); + // If the file is being truncated + // the copy-on-truncate block should also be collected for deletion + if(!(toDelete instanceof BlockInfoContiguousUnderConstruction)) { + return; + } + Block truncateBlock = + ((BlockInfoContiguousUnderConstruction)toDelete).getTruncateBlock(); + if(truncateBlock == null || truncateBlock.equals(toDelete)) { + return; + } + assert truncateBlock instanceof BlockInfoContiguous : + "should be BlockInfoContiguous"; + addDeleteBlock((BlockInfoContiguous) truncateBlock); } public void removeDeleteBlock(Block block) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/85eb32b1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java index fbcc73f..74586cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java @@ -31,6 +31,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.util.concurrent.ThreadLocalRandom; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -58,6 +59,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; @@ -1163,6 +1165,46 @@ public class TestFileTruncate { fs.delete(parent, true); } + /** + * While rolling upgrade is in-progress the test truncates a file + * such that copy-on-truncate is triggered, then deletes the file, + * and makes sure that no blocks involved in truncate are hanging around. + */ + @Test + public void testTruncateWithRollingUpgrade() throws Exception { + final DFSAdmin dfsadmin = new DFSAdmin(cluster.getConfiguration(0)); + DistributedFileSystem dfs = cluster.getFileSystem(); + //start rolling upgrade + dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + int status = dfsadmin.run(new String[]{"-rollingUpgrade", "prepare"}); + assertEquals("could not prepare for rolling upgrade", 0, status); + dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + Path dir = new Path("/testTruncateWithRollingUpgrade"); + fs.mkdirs(dir); + final Path p = new Path(dir, "file"); + final byte[] data = new byte[3]; + ThreadLocalRandom.current().nextBytes(data); + writeContents(data, data.length, p); + + assertEquals("block num should 1", 1, + cluster.getNamesystem().getFSDirectory().getBlockManager() + .getTotalBlocks()); + + final boolean isReady = fs.truncate(p, 2); + assertFalse("should be copy-on-truncate", isReady); + assertEquals("block num should 2", 2, + cluster.getNamesystem().getFSDirectory().getBlockManager() + .getTotalBlocks()); + fs.delete(p, true); + + assertEquals("block num should 0", 0, + cluster.getNamesystem().getFSDirectory().getBlockManager() + .getTotalBlocks()); + status = dfsadmin.run(new String[]{"-rollingUpgrade", "finalize"}); + assertEquals("could not finalize rolling upgrade", 0, status); + } + static void writeContents(byte[] contents, int fileLength, Path p) throws IOException { FSDataOutputStream out = fs.create(p, true, BLOCK_SIZE, REPLICATION, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
