HDFS-11370. Optimize NamenodeFsck#getReplicaInfo. Contributed Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6f290d5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6f290d5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6f290d5 Branch: refs/heads/YARN-5734 Commit: b6f290d5b660ad157c7076767c619d02b3d0f894 Parents: 59c5f18 Author: Jing Zhao <[email protected]> Authored: Wed Feb 1 11:21:35 2017 -0800 Committer: Jing Zhao <[email protected]> Committed: Wed Feb 1 11:21:35 2017 -0800 ---------------------------------------------------------------------- .../BlockUnderConstructionFeature.java | 25 +++++++++++++++++ .../hdfs/server/namenode/NamenodeFsck.java | 28 +++++++++++--------- 2 files changed, 41 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6f290d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java index b935f43..7453184 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java @@ -24,7 +24,9 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.namenode.NameNode; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; +import java.util.NoSuchElementException; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState.COMPLETE; @@ -110,6 +112,29 @@ public class BlockUnderConstructionFeature { } /** + * Note that this iterator doesn't guarantee thread-safe. It depends on + * external mechanisms such as the FSNamesystem lock for protection. + */ + public Iterator<DatanodeStorageInfo> getExpectedStorageLocationsIterator() { + return new Iterator<DatanodeStorageInfo>() { + private int index = 0; + + @Override + public boolean hasNext() { + return index < replicas.length; + } + + @Override + public DatanodeStorageInfo next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return replicas[index++].getExpectedStorageLocation(); + } + }; + } + + /** * @return the index array indicating the block index in each storage. Used * only by striped blocks. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6f290d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index 859f541..7e91935 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -589,23 +589,27 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { return ""; } final boolean isComplete = storedBlock.isComplete(); - DatanodeStorageInfo[] storages = isComplete ? - blockManager.getStorages(storedBlock) : - storedBlock.getUnderConstructionFeature().getExpectedStorageLocations(); + Iterator<DatanodeStorageInfo> storagesItr; StringBuilder sb = new StringBuilder(" ["); final boolean isStriped = storedBlock.isStriped(); Map<DatanodeStorageInfo, Long> storage2Id = new HashMap<>(); - if (isStriped && isComplete) { - long blockId = storedBlock.getBlockId(); - Iterable<StorageAndBlockIndex> sis = - ((BlockInfoStriped)storedBlock).getStorageAndIndexInfos(); - for (StorageAndBlockIndex si: sis){ - storage2Id.put(si.getStorage(), blockId + si.getBlockIndex()); + if (isComplete) { + if (isStriped) { + long blockId = storedBlock.getBlockId(); + Iterable<StorageAndBlockIndex> sis = + ((BlockInfoStriped) storedBlock).getStorageAndIndexInfos(); + for (StorageAndBlockIndex si : sis) { + storage2Id.put(si.getStorage(), blockId + si.getBlockIndex()); + } } + storagesItr = storedBlock.getStorageInfos(); + } else { + storagesItr = storedBlock.getUnderConstructionFeature() + .getExpectedStorageLocationsIterator(); } - for (int i = 0; i < storages.length; i++) { - DatanodeStorageInfo storage = storages[i]; + while (storagesItr.hasNext()) { + DatanodeStorageInfo storage = storagesItr.next(); if (isStriped && isComplete) { long index = storage2Id.get(storage); sb.append("blk_" + index + ":"); @@ -649,7 +653,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { sb.append("LIVE)"); } } - if (i < storages.length - 1) { + if (storagesItr.hasNext()) { sb.append(", "); } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
