HDFS-14008. NN should log snapshotdiff report. Contributed by Pranay Singh.
Signed-off-by: Wei-Chiu Chuang <weic...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d98b881e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d98b881e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d98b881e Branch: refs/heads/HDDS-4 Commit: d98b881e9ab826eb7b70485d0de2a41ab7345334 Parents: 2ab611d Author: Pranay Singh <psi...@cloudera.com> Authored: Thu Nov 1 17:25:11 2018 -0700 Committer: Wei-Chiu Chuang <weic...@apache.org> Committed: Thu Nov 1 17:26:00 2018 -0700 ---------------------------------------------------------------------- .../hdfs/protocol/SnapshotDiffReport.java | 65 ++++++++++++++++++++ .../hdfs/server/namenode/FSNamesystem.java | 20 ++++++ .../snapshot/DirectorySnapshottableFeature.java | 5 ++ .../namenode/snapshot/SnapshotDiffInfo.java | 50 ++++++++++++++- 4 files changed, 139 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java index 8ee4ec7..7bc95c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java @@ -170,14 +170,75 @@ public class SnapshotDiffReport { /** end point of the diff */ private final String toSnapshot; + /** list of diff */ private final List<DiffReportEntry> diffList; + /** + * Records the stats related to Snapshot diff operation. + */ + public static class DiffStats { + // Total dirs processed + private long totalDirsProcessed; + + // Total dirs compared + private long totalDirsCompared; + + // Total files processed + private long totalFilesProcessed; + + // Total files compared + private long totalFilesCompared; + + // Total children listing time + private final long totalChildrenListingTime; + + public DiffStats(long totalDirsProcessed, long totalDirsCompared, + long totalFilesProcessed, long totalFilesCompared, + long totalChildrenListingTime) { + this.totalDirsCompared = totalDirsProcessed; + this.totalDirsProcessed = totalDirsCompared; + this.totalFilesCompared = totalFilesProcessed; + this.totalFilesProcessed = totalFilesCompared; + this.totalChildrenListingTime = totalChildrenListingTime; + } + + public long getTotalDirsProcessed() { + return this.totalDirsProcessed; + } + + public long getTotalDirsCompared() { + return this.totalDirsCompared; + } + + public long getTotalFilesProcessed() { + return this.totalFilesProcessed; + } + + public long getTotalFilesCompared() { + return this.totalFilesCompared; + } + + public long getTotalChildrenListingTime() { + return totalChildrenListingTime; + } + } + + /* Stats associated with the SnapshotDiff Report. */ + private final DiffStats diffStats; + public SnapshotDiffReport(String snapshotRoot, String fromSnapshot, String toSnapshot, List<DiffReportEntry> entryList) { + this(snapshotRoot, fromSnapshot, toSnapshot, new DiffStats(0, 0, 0, 0, 0), + entryList); + } + + public SnapshotDiffReport(String snapshotRoot, String fromSnapshot, + String toSnapshot, DiffStats dStat, List<DiffReportEntry> entryList) { this.snapshotRoot = snapshotRoot; this.fromSnapshot = fromSnapshot; this.toSnapshot = toSnapshot; + this.diffStats = dStat; this.diffList = entryList != null ? entryList : Collections .<DiffReportEntry> emptyList(); } @@ -197,6 +258,10 @@ public class SnapshotDiffReport { return toSnapshot; } + public DiffStats getStats() { + return this.diffStats; + } + /** @return {@link #diffList} */ public List<DiffReportEntry> getDiffList() { return diffList; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index d1904fa..8b1cdf6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -104,6 +104,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.server.namenode.metrics.ReplicatedBlocksMBean; import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; +import org.apache.hadoop.util.Time; import static org.apache.hadoop.util.Time.now; import static org.apache.hadoop.util.Time.monotonicNow; import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME; @@ -6624,6 +6625,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, */ SnapshotDiffReport getSnapshotDiffReport(String path, String fromSnapshot, String toSnapshot) throws IOException { + long begTime = Time.monotonicNow(); final String operationName = "computeSnapshotDiff"; SnapshotDiffReport diffs = null; checkOperation(OperationCategory.READ); @@ -6634,6 +6636,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, path : Snapshot.getSnapshotPath(path, toSnapshot); final FSPermissionChecker pc = getPermissionChecker(); readLock(); + long actualTime = Time.monotonicNow(); try { checkOperation(OperationCategory.READ); diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, pc, snapshotManager, @@ -6646,6 +6649,23 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, } finally { readUnlock(operationName); } + + if (diffs != null) { + SnapshotDiffReport.DiffStats dstat = diffs.getStats(); + LOG.info("SnapshotDiffReport '" + + fromSnapshot + + "' to '" + toSnapshot + "'. Total comparison dirs: " + + dstat.getTotalDirsCompared() + + "/" + dstat.getTotalDirsProcessed() + + ", files: " + + dstat.getTotalFilesCompared() + + "/" + dstat.getTotalFilesProcessed() + + ". Time snapChildrenListing: " + + dstat.getTotalChildrenListingTime() / 1000.0 + "s, actual: " + + ((Time.monotonicNow() - actualTime) / 1000.0) + "s, total: " + + ((Time.monotonicNow() - begTime) / 1000.0) + "s."); + } + logAuditEvent(success, operationName, fromSnapshotRoot, toSnapshotRoot, null); return diffs; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java index 15aa22a..073b88b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java @@ -389,9 +389,13 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature if (change) { diffReport.addDirDiff(dir, relativePath, diff); } + } else { + diffReport.incrementDirsProcessed(); } + long startTime = Time.monotonicNow(); ReadOnlyList<INode> children = dir.getChildrenList(earlierSnapshot .getId()); + diffReport.addChildrenListingTime(Time.monotonicNow() - startTime); for (INode child : children) { final byte[] name = child.getLocalNameBytes(); boolean toProcess = !diff.containsDeleted(name); @@ -418,6 +422,7 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature if (change) { diffReport.addFileDiff(file, relativePath); } + diffReport.incrementFilesProcessed(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java index 4c74afd..ab6f415 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java @@ -125,6 +125,21 @@ class SnapshotDiffInfo { private final Map<Long, RenameEntry> renameMap = new HashMap<Long, RenameEntry>(); + // Total directories compared + private long totalDirsCompared; + + // Total directories + private long totalDirsProcessed; + + // Total files compared + private long totalFilesCompared; + + // Total files + private long totalFilesProcessed; + + // Total children listing time + private long childrenListingTime; + SnapshotDiffInfo(INodeDirectory snapshotRootDir, INodeDirectory snapshotDiffScopeDir, Snapshot start, Snapshot end) { Preconditions.checkArgument(snapshotRootDir.isSnapshottable() && @@ -133,6 +148,10 @@ class SnapshotDiffInfo { this.snapshotDiffScopeDir = snapshotDiffScopeDir; this.from = start; this.to = end; + this.totalDirsCompared = 0; + this.totalDirsProcessed = 0; + this.totalFilesCompared = 0; + this.totalFilesProcessed = 0; } /** Add a dir-diff pair */ @@ -164,6 +183,29 @@ class SnapshotDiffInfo { return to; } + + void incrementDirsCompared() { + this.totalDirsCompared++; + incrementDirsProcessed(); + } + + void incrementDirsProcessed() { + this.totalDirsProcessed++; + } + + void incrementFilesCompared() { + this.totalFilesCompared++; + incrementFilesProcessed(); + } + + void incrementFilesProcessed() { + this.totalFilesProcessed++; + } + + public void addChildrenListingTime(long millis) { + this.childrenListingTime += millis; + } + private RenameEntry getEntry(long inodeId) { RenameEntry entry = renameMap.get(inodeId); if (entry == null) { @@ -203,9 +245,15 @@ class SnapshotDiffInfo { diffReportList.addAll(subList); } } + + SnapshotDiffReport.DiffStats dStats = new SnapshotDiffReport.DiffStats( + this.totalDirsCompared, this.totalDirsProcessed, + this.totalFilesCompared, this.totalFilesProcessed, + this.childrenListingTime); + return new SnapshotDiffReport(snapshotRoot.getFullPathName(), Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to), - diffReportList); + dStats, diffReportList); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
