Repository: hadoop Updated Branches: refs/heads/branch-2 15242786b -> 6917c1916
HDFS-11881. NameNode consumes a lot of memory for snapshot diff report generation. Contributed by Manoj Govindassamy. (cherry picked from commit 16c8dbde574f49827fde5ee9add1861ee65d4645) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6917c191 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6917c191 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6917c191 Branch: refs/heads/branch-2 Commit: 6917c19167060df7879b0e0ead4db1ded62714b3 Parents: 1524278 Author: Wei-Chiu Chuang <weic...@apache.org> Authored: Thu Jun 29 06:38:41 2017 -0700 Committer: Wei-Chiu Chuang <weic...@apache.org> Committed: Thu Jun 29 06:40:24 2017 -0700 ---------------------------------------------------------------------- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 5 +- .../namenode/snapshot/SnapshotDiffInfo.java | 5 +- .../hadoop/hdfs/TestSnapshotCommands.java | 50 ++++++++++++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6917c191/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 769097b..3f27038 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -169,6 +169,7 @@ import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ChunkedArrayList; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.LimitInputStream; @@ -1292,7 +1293,7 @@ public class PBHelperClient { String toSnapshot = reportProto.getToSnapshot(); List<SnapshotDiffReportEntryProto> list = reportProto .getDiffReportEntriesList(); - List<DiffReportEntry> entries = new ArrayList<>(); + List<DiffReportEntry> entries = new ChunkedArrayList<>(); for (SnapshotDiffReportEntryProto entryProto : list) { DiffReportEntry entry = convert(entryProto); if (entry != null) @@ -2143,7 +2144,7 @@ public class PBHelperClient { return null; } List<DiffReportEntry> entries = report.getDiffList(); - List<SnapshotDiffReportEntryProto> entryProtos = new ArrayList<>(); + List<SnapshotDiffReportEntryProto> entryProtos = new ChunkedArrayList<>(); for (DiffReportEntry entry : entries) { SnapshotDiffReportEntryProto entryProto = convert(entry); if (entryProto != null) http://git-wip-us.apache.org/repos/asf/hadoop/blob/6917c191/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java index a576c57..fcd80ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.util.Diff.ListType; import com.google.common.base.Preconditions; import com.google.common.primitives.SignedBytes; +import org.apache.hadoop.util.ChunkedArrayList; /** * A class describing the difference between snapshots of a snapshottable @@ -186,7 +187,7 @@ class SnapshotDiffInfo { * @return A {@link SnapshotDiffReport} describing the difference */ public SnapshotDiffReport generateReport() { - List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>(); + List<DiffReportEntry> diffReportList = new ChunkedArrayList<>(); for (Map.Entry<INode,byte[][]> drEntry : diffMap.entrySet()) { INode node = drEntry.getKey(); byte[][] path = drEntry.getValue(); @@ -213,7 +214,7 @@ class SnapshotDiffInfo { */ private List<DiffReportEntry> generateReport(ChildrenDiff dirDiff, byte[][] parentPath, boolean fromEarlier, Map<Long, RenameEntry> renameMap) { - List<DiffReportEntry> list = new ArrayList<DiffReportEntry>(); + List<DiffReportEntry> list = new ChunkedArrayList<>(); List<INode> created = dirDiff.getList(ListType.CREATED); List<INode> deleted = dirDiff.getList(ListType.DELETED); byte[][] fullPath = new byte[parentPath.length + 1][]; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6917c191/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java index 1d5e071..7e1af2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff; +import org.apache.hadoop.util.ChunkedArrayList; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -195,4 +196,53 @@ public class TestSnapshotCommands { "Disallowing snaphot on " + path + " succeeded", config); fs.delete(new Path("/Fully/QPath"), true); } + + @Test (timeout=60000) + public void testSnapshotDiff()throws Exception { + Configuration config = new HdfsConfiguration(); + Path snapDirPath = new Path(fs.getUri().toString() + "/snap_dir"); + String snapDir = snapDirPath.toString(); + fs.mkdirs(snapDirPath); + + DFSTestUtil.DFSAdminRun("-allowSnapshot " + snapDirPath, 0, + "Allowing snaphot on " + snapDirPath + " succeeded", config); + DFSTestUtil.createFile(fs, new Path(snapDirPath, "file1"), + 1024, (short) 1, 100); + DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn1", config); + DFSTestUtil.createFile(fs, new Path(snapDirPath, "file2"), + 1024, (short) 1, 100); + DFSTestUtil.createFile(fs, new Path(snapDirPath, "file3"), + 1024, (short) 1, 100); + DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn2", config); + + // verify the snapshot diff using api and command line + SnapshotDiffReport report_s1_s2 = + fs.getSnapshotDiffReport(snapDirPath, "sn1", "sn2"); + DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir + + " sn1 sn2", 0, report_s1_s2.toString()); + DFSTestUtil.FsShellRun("-renameSnapshot " + snapDirPath + " sn2 sn3", + config); + + SnapshotDiffReport report_s1_s3 = + fs.getSnapshotDiffReport(snapDirPath, "sn1", "sn3"); + DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir + + " sn1 sn3", 0, report_s1_s3.toString()); + + // Creating 100 more files so as to force DiffReport generation + // backend ChunkedArrayList to create multiple chunks. + for (int i = 0; i < 100; i++) { + DFSTestUtil.createFile(fs, new Path(snapDirPath, "file_" + i), + 1, (short) 1, 100); + } + DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn4", config); + DFSTestUtil.toolRun(new SnapshotDiff(config), snapDir + + " sn1 sn4", 0, null); + + DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn1", config); + DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn3", config); + DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn4", config); + DFSTestUtil.DFSAdminRun("-disallowSnapshot " + snapDir, 0, + "Disallowing snaphot on " + snapDirPath + " succeeded", config); + fs.delete(new Path("/Fully/QPath"), true); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org