MAPREDUCE-6059. Speed up history server startup time (Siqi Li via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6722247a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6722247a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6722247a Branch: refs/heads/HDFS-EC Commit: 6722247a6398ca57408fe325d8bc49525812cdf9 Parents: fa58390 Author: Allen Wittenauer <[email protected]> Authored: Wed Feb 4 17:36:28 2015 -0800 Committer: Zhe Zhang <[email protected]> Committed: Fri Feb 6 13:45:48 2015 -0800 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java | 8 ++++++++ 2 files changed, 10 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6722247a/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a633b4e..0fbf9a8 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -94,6 +94,8 @@ Trunk (Unreleased) MAPREDUCE-6013. [post-HADOOP-9902] mapred version is missing (Akira AJISAKA via aw) + MAPREDUCE-6059. Speed up history server startup time (Siqi Li via aw) + BUG FIXES MAPREDUCE-6191. Improve clearing stale state of Java serialization http://git-wip-us.apache.org/repos/asf/hadoop/blob/6722247a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 40844df..65f8a4f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -263,6 +263,10 @@ public class HistoryFileManager extends AbstractService { public HistoryFileInfo get(JobId jobId) { return cache.get(jobId); } + + public boolean isFull() { + return cache.size() >= maxSize; + } } /** @@ -668,6 +672,10 @@ public class HistoryFileManager extends AbstractService { for (FileStatus fs : timestampedDirList) { // TODO Could verify the correct format for these directories. addDirectoryToSerialNumberIndex(fs.getPath()); + } + for (int i= timestampedDirList.size() - 1; + i >= 0 && !jobListCache.isFull(); i--) { + FileStatus fs = timestampedDirList.get(i); addDirectoryToJobListCache(fs.getPath()); } }
