Repository: hadoop Updated Branches: refs/heads/trunk 871d0d39f -> 346caa209
HDFS-13227. Add a method to calculate cumulative diff over multiple snapshots in DirectoryDiffList. Contributed by Shashikant Banerjee Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/346caa20 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/346caa20 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/346caa20 Branch: refs/heads/trunk Commit: 346caa209571dedf1331b2658d5702b45dd40bfe Parents: 871d0d3 Author: Tsz-Wo Nicholas Sze <[email protected]> Authored: Tue Mar 6 13:35:03 2018 -0800 Committer: Tsz-Wo Nicholas Sze <[email protected]> Committed: Tue Mar 6 13:35:03 2018 -0800 ---------------------------------------------------------------------- .../snapshot/AbstractINodeDiffList.java | 18 +++++----- .../snapshot/DirectoryWithSnapshotFeature.java | 36 +++++++++++++++----- 2 files changed, 38 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/346caa20/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java index 5163e5e..d115656 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java @@ -235,6 +235,12 @@ abstract class AbstractINodeDiffList<N extends INode, return diff == null ? Snapshot.CURRENT_STATE_ID : diff.getSnapshotId(); } + public final int getDiffIndexById(final int snapshotId) { + int diffIndex = diffs.binarySearch(snapshotId); + diffIndex = diffIndex < 0 ? (-diffIndex - 1) : diffIndex; + return diffIndex; + } + final int[] changedBetweenSnapshots(Snapshot from, Snapshot to) { if (diffs == null) { return null; @@ -247,10 +253,10 @@ abstract class AbstractINodeDiffList<N extends INode, } final int size = diffs.size(); - int earlierDiffIndex = diffs.binarySearch(earlier.getId()); + int earlierDiffIndex = getDiffIndexById(earlier.getId()); int laterDiffIndex = later == null ? size - : diffs.binarySearch(later.getId()); - if (-earlierDiffIndex - 1 == size) { + : getDiffIndexById(later.getId()); + if (earlierDiffIndex == size) { // if the earlierSnapshot is after the latest SnapshotDiff stored in // diffs, no modification happened after the earlierSnapshot return null; @@ -260,10 +266,6 @@ abstract class AbstractINodeDiffList<N extends INode, // before it, no modification happened before the laterSnapshot return null; } - earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1) - : earlierDiffIndex; - laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1) - : laterDiffIndex; return new int[]{earlierDiffIndex, laterDiffIndex}; } @@ -304,7 +306,7 @@ abstract class AbstractINodeDiffList<N extends INode, } return diff; } - + @Override public Iterator<D> iterator() { return diffs != null ? diffs.iterator() : Collections.emptyIterator(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/346caa20/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index b3ce602..19be8f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -228,12 +228,18 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { private List<INode> initChildren() { if (children == null) { final ChildrenDiff combined = new ChildrenDiff(); - for (DirectoryDiff d = DirectoryDiff.this; d != null; - d = d.getPosterior()) { + DirectoryDiffList directoryDiffList = + currentDir.getDirectoryWithSnapshotFeature().diffs; + final int diffIndex = + directoryDiffList.getDiffIndexById(getSnapshotId()); + List<DirectoryDiff> diffList = directoryDiffList + .getDiffListBetweenSnapshots(diffIndex, + directoryDiffList.asList().size(), currentDir); + for (DirectoryDiff d : diffList) { combined.combinePosterior(d.diff, null); } - children = combined.apply2Current(ReadOnlyList.Util.asList( - currentDir.getChildrenList(Snapshot.CURRENT_STATE_ID))); + children = combined.apply2Current(ReadOnlyList.Util + .asList(currentDir.getChildrenList(Snapshot.CURRENT_STATE_ID))); } return children; } @@ -383,6 +389,19 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { } return NO_SNAPSHOT_ID; } + + /** + * Returns the list of diffs between two indexes corresponding to two + * snapshots. + * @param fromIndex Index of the diff corresponding to the earlier snapshot + * @param toIndex Index of the diff corresponding to the later snapshot + * @param dir The Directory to which the diffList belongs + * @return list of directory diffs + */ + List<DirectoryDiff> getDiffListBetweenSnapshots(int fromIndex, int toIndex, + INodeDirectory dir) { + return asList().getMinListForRange(fromIndex, toIndex, dir); + } } private static Map<INode, INode> cloneDiffList(List<INode> diffList) { @@ -597,7 +616,7 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { return diff != null ? diff.getChild(name, true, currentINode) : currentINode.getChild(name, Snapshot.CURRENT_STATE_ID); } - + /** Used to record the modification of a symlink node */ public INode saveChild2Snapshot(INodeDirectory currentINode, final INode child, final int latestSnapshotId, final INode snapshotCopy) { @@ -678,9 +697,10 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { boolean dirMetadataChanged = false; INodeDirectoryAttributes dirCopy = null; - DiffList<DirectoryDiff> difflist = diffs.asList(); - for (int i = earlierDiffIndex; i < laterDiffIndex; i++) { - DirectoryDiff sdiff = difflist.get(i); + List<DirectoryDiff> difflist = diffs + .getDiffListBetweenSnapshots(earlierDiffIndex, laterDiffIndex, + currentINode); + for (DirectoryDiff sdiff : difflist) { diff.combinePosterior(sdiff.diff, null); if (!dirMetadataChanged && sdiff.snapshotINode != null) { if (dirCopy == null) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
