HDFS-11047. Remove deep copies of FinalizedReplica to alleviate heap consumption on DataNode. Contributed by Xiaobing Zhou
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e03ee52 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e03ee52 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e03ee52 Branch: refs/heads/HDFS-9806 Commit: 9e03ee527988ff85af7f2c224c5570b69d09279a Parents: f3ac1f4 Author: Mingliang Liu <[email protected]> Authored: Thu Oct 27 15:58:09 2016 -0700 Committer: Mingliang Liu <[email protected]> Committed: Thu Oct 27 16:00:27 2016 -0700 ---------------------------------------------------------------------- .../hdfs/server/datanode/DirectoryScanner.java | 14 +++++++------- .../server/datanode/fsdataset/FsDatasetSpi.java | 11 ++++++++++- .../datanode/fsdataset/impl/FsDatasetImpl.java | 16 +++++++++++----- 3 files changed, 28 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e03ee52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index 58071dc..e2baf32 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -398,14 +399,13 @@ public class DirectoryScanner implements Runnable { diffs.put(bpid, diffRecord); statsRecord.totalBlocks = blockpoolReport.length; - List<ReplicaInfo> bl = dataset.getFinalizedBlocks(bpid); - ReplicaInfo[] memReport = bl.toArray(new ReplicaInfo[bl.size()]); - Arrays.sort(memReport); // Sort based on blockId + final List<ReplicaInfo> bl = dataset.getFinalizedBlocks(bpid); + Collections.sort(bl); // Sort based on blockId int d = 0; // index for blockpoolReport int m = 0; // index for memReprot - while (m < memReport.length && d < blockpoolReport.length) { - ReplicaInfo memBlock = memReport[m]; + while (m < bl.size() && d < blockpoolReport.length) { + ReplicaInfo memBlock = bl.get(m); ScanInfo info = blockpoolReport[d]; if (info.getBlockId() < memBlock.getBlockId()) { if (!dataset.isDeletingBlock(bpid, info.getBlockId())) { @@ -452,8 +452,8 @@ public class DirectoryScanner implements Runnable { ++m; } } - while (m < memReport.length) { - ReplicaInfo current = memReport[m++]; + while (m < bl.size()) { + ReplicaInfo current = bl.get(m++); addDifference(diffRecord, statsRecord, current.getBlockId(), current.getVolume()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e03ee52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index f2ffa83..e113212 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -229,7 +229,16 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean { */ VolumeFailureSummary getVolumeFailureSummary(); - /** @return a list of finalized blocks for the given block pool. */ + /** + * Gets a list of references to the finalized blocks for the given block pool. + * <p> + * Callers of this function should call + * {@link FsDatasetSpi#acquireDatasetLock} to avoid blocks' status being + * changed during list iteration. + * </p> + * @return a list of references to the finalized blocks for the given block + * pool. + */ List<ReplicaInfo> getFinalizedBlocks(String bpid); /** @return a list of finalized blocks for the given block pool. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e03ee52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index abeda2c..0f40f2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -1714,17 +1714,23 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> { } /** - * Get the list of finalized blocks from in-memory blockmap for a block pool. + * Gets a list of references to the finalized blocks for the given block pool. + * <p> + * Callers of this function should call + * {@link FsDatasetSpi#acquireDatasetLock} to avoid blocks' status being + * changed during list iteration. + * </p> + * @return a list of references to the finalized blocks for the given block + * pool. */ @Override public List<ReplicaInfo> getFinalizedBlocks(String bpid) { try (AutoCloseableLock lock = datasetLock.acquire()) { - ArrayList<ReplicaInfo> finalized = - new ArrayList<ReplicaInfo>(volumeMap.size(bpid)); + final List<ReplicaInfo> finalized = new ArrayList<ReplicaInfo>( + volumeMap.size(bpid)); for (ReplicaInfo b : volumeMap.replicas(bpid)) { if (b.getState() == ReplicaState.FINALIZED) { - finalized.add(new ReplicaBuilder(ReplicaState.FINALIZED) - .from(b).build()); + finalized.add(b); } } return finalized; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
