HDFS-8777. Erasure Coding: add tests for taking snapshots on EC files. Contributed by Rakesh R.
Change-Id: Ia3ae582405e741ca8e90d9255ab9b95d085e5fa8 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90e14055 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90e14055 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90e14055 Branch: refs/heads/HDFS-7240 Commit: 90e14055168afdb93fa8089158c03a6a694e066c Parents: 2529464 Author: Zhe Zhang <z...@apache.org> Authored: Mon Nov 2 07:48:30 2015 -0800 Committer: Zhe Zhang <z...@apache.org> Committed: Mon Nov 2 07:48:30 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../TestErasureCodingPolicyWithSnapshot.java | 199 +++++++++++++++++++ 2 files changed, 202 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e14055/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0bbc60d..3b2d997 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -189,6 +189,9 @@ Trunk (Unreleased) HDFS-9323. Randomize the DFSStripedOutputStreamWithFailure tests. (szetszwo) + HDFS-8777. Erasure Coding: add tests for taking snapshots on EC files. + (Rakesh R via zhz) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e14055/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java new file mode 100644 index 0000000..515763c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FsShell; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; +import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; +import org.apache.hadoop.util.ToolRunner; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestErasureCodingPolicyWithSnapshot { + private MiniDFSCluster cluster; + private DistributedFileSystem fs; + private Configuration conf; + + private final static short GROUP_SIZE = StripedFileTestUtil.NUM_DATA_BLOCKS + + StripedFileTestUtil.NUM_PARITY_BLOCKS; + private final static int SUCCESS = 0; + private final ErasureCodingPolicy sysDefaultPolicy = ErasureCodingPolicyManager + .getSystemDefaultPolicy(); + + @Before + public void setupCluster() throws IOException { + conf = new HdfsConfiguration(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE).build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + } + + @After + public void shutdownCluster() throws IOException { + if (cluster != null) { + cluster.shutdown(); + } + } + + /** + * Test correctness of successive snapshot creation and deletion with erasure + * coding policies. Create snapshot of ecDir's parent directory. + */ + @Test(timeout = 120000) + public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception { + final int len = 1024; + final Path ecDirParent = new Path("/parent"); + final Path ecDir = new Path(ecDirParent, "ecdir"); + final Path ecFile = new Path(ecDir, "ecfile"); + fs.mkdirs(ecDir); + fs.allowSnapshot(ecDirParent); + // set erasure coding policy + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED); + String contents = DFSTestUtil.readFile(fs, ecFile); + final Path snap1 = fs.createSnapshot(ecDirParent, "snap1"); + final Path snap1ECDir = new Path(snap1, ecDir.getName()); + assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + fs.getErasureCodingPolicy(snap1ECDir)); + + // Now delete the dir which has erasure coding policy. Re-create the dir again, and + // take another snapshot + fs.delete(ecDir, true); + fs.mkdir(ecDir, FsPermission.getDirDefault()); + final Path snap2 = fs.createSnapshot(ecDirParent, "snap2"); + final Path snap2ECDir = new Path(snap2, ecDir.getName()); + assertNull("Expected null erasure coding policy", + fs.getErasureCodingPolicy(snap2ECDir)); + + // Make dir again with system default ec policy + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + final Path snap3 = fs.createSnapshot(ecDirParent, "snap3"); + final Path snap3ECDir = new Path(snap3, ecDir.getName()); + // Check that snap3's ECPolicy has the correct settings + ErasureCodingPolicy ezSnap3 = fs.getErasureCodingPolicy(snap3ECDir); + assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + ezSnap3); + + // Check that older snapshots still have the old ECPolicy settings + assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + fs.getErasureCodingPolicy(snap1ECDir)); + assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + fs.getErasureCodingPolicy(snap2ECDir)); + + // Verify contents of the snapshotted file + final Path snapshottedECFile = new Path(snap1.toString() + "/" + + ecDir.getName() + "/" + ecFile.getName()); + assertEquals("Contents of snapshotted file have changed unexpectedly", + contents, DFSTestUtil.readFile(fs, snapshottedECFile)); + + // Now delete the snapshots out of order and verify the EC policy + // correctness + fs.deleteSnapshot(ecDirParent, snap2.getName()); + assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + fs.getErasureCodingPolicy(snap1ECDir)); + assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + fs.getErasureCodingPolicy(snap3ECDir)); + fs.deleteSnapshot(ecDirParent, snap1.getName()); + assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + fs.getErasureCodingPolicy(snap3ECDir)); + } + + /** + * Test creation of snapshot on directory has erasure coding policy. + */ + @Test(timeout = 120000) + public void testSnapshotsOnErasureCodingDir() throws Exception { + final Path ecDir = new Path("/ecdir"); + fs.mkdirs(ecDir); + fs.allowSnapshot(ecDir); + + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + final Path snap1 = fs.createSnapshot(ecDir, "snap1"); + assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + fs.getErasureCodingPolicy(snap1)); + } + + /** + * Test verify erasure coding policy is present after restarting the NameNode. + */ + @Test(timeout = 120000) + public void testSnapshotsOnErasureCodingDirAfterNNRestart() throws Exception { + final Path ecDir = new Path("/ecdir"); + fs.mkdirs(ecDir); + fs.allowSnapshot(ecDir); + + // set erasure coding policy + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + final Path snap1 = fs.createSnapshot(ecDir, "snap1"); + ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1); + assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + ecSnap); + + // save namespace, restart namenode, and check ec policy correctness. + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + fs.saveNamespace(); + fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + cluster.restartNameNode(true); + + ErasureCodingPolicy ecSnap1 = fs.getErasureCodingPolicy(snap1); + assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + ecSnap1); + assertEquals("Got unexpected ecSchema", ecSnap.getSchema(), + ecSnap1.getSchema()); + } + + /** + * Test copy a snapshot will not preserve its erasure coding policy info. + */ + @Test(timeout = 120000) + public void testCopySnapshotWillNotPreserveErasureCodingPolicy() + throws Exception { + final int len = 1024; + final Path ecDir = new Path("/ecdir"); + final Path ecFile = new Path(ecDir, "ecFile"); + fs.mkdirs(ecDir); + fs.allowSnapshot(ecDir); + + // set erasure coding policy + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED); + final Path snap1 = fs.createSnapshot(ecDir, "snap1"); + + Path snap1Copy = new Path(ecDir.toString() + "-copy"); + final Path snap1CopyECDir = new Path("/ecdir-copy"); + String[] argv = new String[] { "-cp", "-px", snap1.toUri().toString(), + snap1Copy.toUri().toString() }; + int ret = ToolRunner.run(new FsShell(conf), argv); + assertEquals("cp -px is not working on a snapshot", SUCCESS, ret); + + assertNull("Got unexpected erasure coding policy", + fs.getErasureCodingPolicy(snap1CopyECDir)); + assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + fs.getErasureCodingPolicy(snap1)); + } +}
