Repository: hadoop Updated Branches: refs/heads/trunk a3ed6f40e -> 6c63cc7d3
HDFS-13315. Add a test for the issue reported in HDFS-11481 which is fixed by HDFS-10997. Contributed by Yongjun Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c63cc7d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c63cc7d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c63cc7d Branch: refs/heads/trunk Commit: 6c63cc7d304571578e6551170552182d30b8e8fa Parents: a3ed6f4 Author: Yongjun Zhang <yzh...@cloudera.com> Authored: Tue Mar 20 23:00:39 2018 -0700 Committer: Yongjun Zhang <yzh...@cloudera.com> Committed: Tue Mar 20 23:04:05 2018 -0700 ---------------------------------------------------------------------- .../hdfs/tools/snapshot/SnapshotDiff.java | 1 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 37 ++++++++++++++++++ .../apache/hadoop/hdfs/TestEncryptionZones.java | 41 ++++++++++++++++++++ .../snapshot/TestSnapshotDiffReport.java | 25 +----------- 4 files changed, 80 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c63cc7d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java index 3838ca1..1237099 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java @@ -106,6 +106,7 @@ public class SnapshotDiff extends Configured implements Tool { } catch (IOException e) { String[] content = e.getLocalizedMessage().split("\n"); System.err.println("snapshotDiff: " + content[0]); + e.printStackTrace(System.err); return 1; } return 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c63cc7d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 1411a7f..4f9f260 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -121,6 +121,9 @@ import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -2354,4 +2357,38 @@ public class DFSTestUtil { } return closedFiles; } + + /** + * Check the correctness of the snapshotDiff report. + * Make sure all items in the passed entries are in the snapshotDiff + * report. + */ + public static void verifySnapshotDiffReport(DistributedFileSystem fs, + Path dir, String from, String to, + DiffReportEntry... entries) throws IOException { + SnapshotDiffReport report = fs.getSnapshotDiffReport(dir, from, to); + // reverse the order of from and to + SnapshotDiffReport inverseReport = fs + .getSnapshotDiffReport(dir, to, from); + LOG.info(report.toString()); + LOG.info(inverseReport.toString() + "\n"); + + assertEquals(entries.length, report.getDiffList().size()); + assertEquals(entries.length, inverseReport.getDiffList().size()); + + for (DiffReportEntry entry : entries) { + if (entry.getType() == DiffType.MODIFY) { + assertTrue(report.getDiffList().contains(entry)); + assertTrue(inverseReport.getDiffList().contains(entry)); + } else if (entry.getType() == DiffType.DELETE) { + assertTrue(report.getDiffList().contains(entry)); + assertTrue(inverseReport.getDiffList().contains( + new DiffReportEntry(DiffType.CREATE, entry.getSourcePath()))); + } else if (entry.getType() == DiffType.CREATE) { + assertTrue(report.getDiffList().contains(entry)); + assertTrue(inverseReport.getDiffList().contains( + new DiffReportEntry(DiffType.DELETE, entry.getSourcePath()))); + } + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c63cc7d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 4e69b9d..c541434 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -75,6 +75,8 @@ import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.server.namenode.EncryptionFaultInjector; import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; @@ -148,6 +150,7 @@ import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; public class TestEncryptionZones { + static final Logger LOG = Logger.getLogger(TestEncryptionZones.class); protected Configuration conf; private FileSystemTestHelper fsHelper; @@ -1496,6 +1499,44 @@ public class TestEncryptionZones { } /** + * Check the correctness of the diff reports. + */ + private void verifyDiffReport(Path dir, String from, String to, + DiffReportEntry... entries) throws IOException { + DFSTestUtil.verifySnapshotDiffReport(fs, dir, from, to, entries); + } + + /** + * Test correctness of snapshotDiff for encryption zone. + * snapshtoDiff should work when the path parameter is prefixed with + * /.reserved/raw for path that's both snapshottable and encryption zone. + */ + @Test + public void testSnapshotDiffOnEncryptionZones() throws Exception { + final String TEST_KEY2 = "testkey2"; + DFSTestUtil.createKey(TEST_KEY2, cluster, conf); + + final int len = 8196; + final Path zone = new Path("/zone"); + final Path rawZone = new Path("/.reserved/raw/zone"); + final Path zoneFile = new Path(zone, "zoneFile"); + fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true); + dfsAdmin.allowSnapshot(zone); + dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH); + DFSTestUtil.createFile(fs, zoneFile, len, (short) 1, 0xFEED); + fs.createSnapshot(zone, "snap1"); + fsWrapper.delete(zoneFile, true); + fs.createSnapshot(zone, "snap2"); + verifyDiffReport(zone, "snap1", "snap2", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("zoneFile"))); + + verifyDiffReport(rawZone, "snap1", "snap2", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("zoneFile"))); + } + + /** * Verify symlinks can be created in encryption zones and that * they function properly when the target is in the same * or different ez. http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c63cc7d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java index 47677e0..4625988 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java @@ -194,30 +194,7 @@ public class TestSnapshotDiffReport { */ private void verifyDiffReport(Path dir, String from, String to, DiffReportEntry... entries) throws IOException { - SnapshotDiffReport report = hdfs.getSnapshotDiffReport(dir, from, to); - // reverse the order of from and to - SnapshotDiffReport inverseReport = hdfs - .getSnapshotDiffReport(dir, to, from); - LOG.info(report.toString()); - LOG.info(inverseReport.toString() + "\n"); - - assertEquals(entries.length, report.getDiffList().size()); - assertEquals(entries.length, inverseReport.getDiffList().size()); - - for (DiffReportEntry entry : entries) { - if (entry.getType() == DiffType.MODIFY) { - assertTrue(report.getDiffList().contains(entry)); - assertTrue(inverseReport.getDiffList().contains(entry)); - } else if (entry.getType() == DiffType.DELETE) { - assertTrue(report.getDiffList().contains(entry)); - assertTrue(inverseReport.getDiffList().contains( - new DiffReportEntry(DiffType.CREATE, entry.getSourcePath()))); - } else if (entry.getType() == DiffType.CREATE) { - assertTrue(report.getDiffList().contains(entry)); - assertTrue(inverseReport.getDiffList().contains( - new DiffReportEntry(DiffType.DELETE, entry.getSourcePath()))); - } - } + DFSTestUtil.verifySnapshotDiffReport(hdfs, dir, from, to, entries); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
