Repository: hadoop Updated Branches: refs/heads/trunk 3fa33b5c2 -> 0ac8fb4b3
HDFS-9490. MiniDFSCluster should change block generation stamp via FsDatasetTestUtils. (Tony Wu via lei) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ac8fb4b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ac8fb4b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ac8fb4b Branch: refs/heads/trunk Commit: 0ac8fb4b336dd03a037092c9712b962e7ed8f852 Parents: 3fa33b5 Author: Lei Xu <[email protected]> Authored: Fri Dec 4 10:24:55 2015 -0800 Committer: Lei Xu <[email protected]> Committed: Fri Dec 4 10:24:55 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/MiniDFSCluster.java | 8 +++----- .../hdfs/server/datanode/FsDatasetTestUtils.java | 9 +++++++++ .../fsdataset/impl/FsDatasetImplTestUtils.java | 14 ++++++++++++++ .../namenode/ha/TestPendingCorruptDnMessages.java | 2 +- 5 files changed, 30 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac8fb4b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 17cbe29..89094bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1710,6 +1710,9 @@ Release 2.8.0 - UNRELEASED HDFS-9485. Make BlockManager#removeFromExcessReplicateMap accept BlockInfo instead of Block. (Mingliang Liu via jing9) + HDFS-9490. MiniDFSCluster should change block generation stamp via + FsDatasetTestUtils. (Tony Wu via lei) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac8fb4b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 6baea25..3e25177 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -2141,12 +2141,10 @@ public class MiniDFSCluster { getMaterializedReplica(i, blk).truncateMeta(newSize); } - public boolean changeGenStampOfBlock(int dnIndex, ExtendedBlock blk, + public void changeGenStampOfBlock(int dnIndex, ExtendedBlock blk, long newGenStamp) throws IOException { - File blockFile = getBlockFile(dnIndex, blk); - File metaFile = FsDatasetUtil.findMetaFile(blockFile); - return metaFile.renameTo(new File(DatanodeUtil.getMetaName( - blockFile.getAbsolutePath(), newGenStamp))); + getFsDatasetTestUtils(dnIndex) + .changeStoredGenerationStamp(blk, newGenStamp); } /* http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac8fb4b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java index 07fb7ce..fd47705 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java @@ -242,4 +242,13 @@ public interface FsDatasetTestUtils { * Get the persistently stored generation stamp. */ long getStoredGenerationStamp(ExtendedBlock block) throws IOException; + + /** + * Change the persistently stored generation stamp. + * @param block the block whose generation stamp will be changed + * @param newGenStamp the new generation stamp + * @throws IOException + */ + void changeStoredGenerationStamp(ExtendedBlock block, long newGenStamp) + throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac8fb4b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java index 8fce163..320ae9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil; import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica; import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils; import org.apache.hadoop.hdfs.server.datanode.Replica; @@ -47,6 +48,7 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.channels.FileChannel; import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.Random; /** @@ -363,4 +365,16 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils { File[] files = FileUtil.listFiles(dir); return FsDatasetUtil.getGenerationStampFromFile(files, f); } + + @Override + public void changeStoredGenerationStamp( + ExtendedBlock block, long newGenStamp) throws IOException { + File blockFile = + dataset.getBlockFile(block.getBlockPoolId(), block.getBlockId()); + File metaFile = FsDatasetUtil.findMetaFile(blockFile); + File newMetaFile = new File( + DatanodeUtil.getMetaName(blockFile.getAbsolutePath(), newGenStamp)); + Files.move(metaFile.toPath(), newMetaFile.toPath(), + StandardCopyOption.ATOMIC_MOVE); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac8fb4b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java index 443500c..5f116d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java @@ -68,7 +68,7 @@ public class TestPendingCorruptDnMessages { // Change the gen stamp of the block on datanode to go back in time (gen // stamps start at 1000) ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, filePath); - assertTrue(cluster.changeGenStampOfBlock(0, block, 900)); + cluster.changeGenStampOfBlock(0, block, 900); // Run directory dsscanner to update Datanode's volumeMap DataNodeTestUtils.runDirectoryScanner(cluster.getDataNodes().get(0));
