[29/48] hadoop git commit: YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. Contributed by Xuan Gong

2016-03-28 Thread aengineer 
YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. 
Contributed by Xuan Gong


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8fc81f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8fc81f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8fc81f9

Branch: refs/heads/HDFS-7240
Commit: e8fc81f9c812b0c167411de7f1789a9a433a0d57
Parents: 3f622a1
Author: Jason Lowe 
Authored: Fri Mar 25 20:15:49 2016 +
Committer: Jason Lowe 
Committed: Fri Mar 25 20:15:49 2016 +

--
 .../hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java  | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8fc81f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 35d9970..3fa8691 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -379,6 +379,8 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
   this.stream = createLogFileStream(fs, logPath);
   this.jsonGenerator = new JsonFactory().createJsonGenerator(stream);
   this.jsonGenerator.setPrettyPrinter(new MinimalPrettyPrinter("\n"));
+  this.jsonGenerator.configure(
+  JsonGenerator.Feature.FLUSH_PASSED_TO_STREAM, false);
   this.lastModifiedTime = Time.monotonicNow();
 }

[15/48] hadoop git commit: YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. Contributed by Xuan Gong.

2016-03-28 Thread aengineer 
YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. 
Contributed by Xuan Gong.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af1d125f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af1d125f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af1d125f

Branch: refs/heads/HDFS-7240
Commit: af1d125f9ce35ec69a610674a1c5c60cc17141a7
Parents: 368c773
Author: Junping Du 
Authored: Wed Mar 23 08:57:16 2016 -0700
Committer: Junping Du 
Committed: Wed Mar 23 08:57:16 2016 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   6 ++
 .../api/impl/FileSystemTimelineWriter.java  | 104 ++-
 2 files changed, 86 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/af1d125f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ff4b493..8acee57 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1747,6 +1747,12 @@ public class YarnConfiguration extends Configuration {
   public static final long
   TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT = 7 * 60;
 
+  public static final String
+  TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE =
+  TIMELINE_SERVICE_CLIENT_PREFIX + "internal-attempt-dir-cache-size";
+  public static final int
+  DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE = 1000;
+
   // This is temporary solution. The configuration will be deleted once we have
   // the FileSystem API to check whether append operation is supported or not.
   public static final String TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND

http://git-wip-us.apache.org/repos/asf/hadoop/blob/af1d125f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index b471b3b..35d9970 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -26,6 +26,7 @@ import java.net.URI;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -97,6 +98,7 @@ public class FileSystemTimelineWriter extends TimelineWriter{
   private long ttl;
   private LogFDsCache logFDsCache = null;
   private boolean isAppendSupported;
+  private final AttemptDirCache attemptDirCache;
 
   public FileSystemTimelineWriter(Configuration conf,
   UserGroupInformation authUgi, Client client, URI resURI)
@@ -158,6 +160,15 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 
 objMapper = createObjectMapper();
 
+int attemptDirCacheSize = conf.getInt(
+YarnConfiguration
+.TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE,
+YarnConfiguration
+.DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE);
+
+attemptDirCache =
+new AttemptDirCache(attemptDirCacheSize, fs, activePath);
+
 if (LOG.isDebugEnabled()) {
   StringBuilder debugMSG = new StringBuilder();
   debugMSG.append(
@@ -199,7 +210,7 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 = new ArrayList();
 List entitiesToEntityCache
 = new ArrayList();
-Path attemptDir = createAttemptDir(appAttemptId);
+Path attemptDir = attemptDirCache.getAppAttemptDir(appAttemptId);
 
 for (TimelineEntity entity : entities) {
   if (summaryEntityTypes.contains(entity.getEntityType())) {
@@ -279,32 +290,11 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 return mapper;
   }
 
-  private Path

hadoop git commit: YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. Contributed by Xuan Gong (cherry picked from commit e8fc81f9c812b0c167411de7f1789a9a433a0d57)

2016-03-25 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 416ea158c -> e06446be7


YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. 
Contributed by Xuan Gong
(cherry picked from commit e8fc81f9c812b0c167411de7f1789a9a433a0d57)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e06446be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e06446be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e06446be

Branch: refs/heads/branch-2.8
Commit: e06446be74c01f4e1713cf10de37883346c8ed43
Parents: 416ea15
Author: Jason Lowe 
Authored: Fri Mar 25 20:15:49 2016 +
Committer: Jason Lowe 
Committed: Fri Mar 25 20:20:01 2016 +

--
 .../hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java  | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06446be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 35d9970..3fa8691 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -379,6 +379,8 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
   this.stream = createLogFileStream(fs, logPath);
   this.jsonGenerator = new JsonFactory().createJsonGenerator(stream);
   this.jsonGenerator.setPrettyPrinter(new MinimalPrettyPrinter("\n"));
+  this.jsonGenerator.configure(
+  JsonGenerator.Feature.FLUSH_PASSED_TO_STREAM, false);
   this.lastModifiedTime = Time.monotonicNow();
 }

hadoop git commit: YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. Contributed by Xuan Gong (cherry picked from commit e8fc81f9c812b0c167411de7f1789a9a433a0d57)

2016-03-25 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 8360d98d6 -> 3cea00a29


YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. 
Contributed by Xuan Gong
(cherry picked from commit e8fc81f9c812b0c167411de7f1789a9a433a0d57)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3cea00a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3cea00a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3cea00a2

Branch: refs/heads/branch-2
Commit: 3cea00a29471b82ca020874ea51324c19b5cdd97
Parents: 8360d98
Author: Jason Lowe 
Authored: Fri Mar 25 20:15:49 2016 +
Committer: Jason Lowe 
Committed: Fri Mar 25 20:19:15 2016 +

--
 .../hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java  | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cea00a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 35d9970..3fa8691 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -379,6 +379,8 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
   this.stream = createLogFileStream(fs, logPath);
   this.jsonGenerator = new JsonFactory().createJsonGenerator(stream);
   this.jsonGenerator.setPrettyPrinter(new MinimalPrettyPrinter("\n"));
+  this.jsonGenerator.configure(
+  JsonGenerator.Feature.FLUSH_PASSED_TO_STREAM, false);
   this.lastModifiedTime = Time.monotonicNow();
 }

hadoop git commit: YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. Contributed by Xuan Gong

2016-03-25 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 3f622a143 -> e8fc81f9c


YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. 
Contributed by Xuan Gong


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8fc81f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8fc81f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8fc81f9

Branch: refs/heads/trunk
Commit: e8fc81f9c812b0c167411de7f1789a9a433a0d57
Parents: 3f622a1
Author: Jason Lowe 
Authored: Fri Mar 25 20:15:49 2016 +
Committer: Jason Lowe 
Committed: Fri Mar 25 20:15:49 2016 +

--
 .../hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java  | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8fc81f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 35d9970..3fa8691 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -379,6 +379,8 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
   this.stream = createLogFileStream(fs, logPath);
   this.jsonGenerator = new JsonFactory().createJsonGenerator(stream);
   this.jsonGenerator.setPrettyPrinter(new MinimalPrettyPrinter("\n"));
+  this.jsonGenerator.configure(
+  JsonGenerator.Feature.FLUSH_PASSED_TO_STREAM, false);
   this.lastModifiedTime = Time.monotonicNow();
 }

hadoop git commit: YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. Contributed by Xuan Gong. (cherry picked from commit af1d125f9ce35ec69a610674a1c5c60cc17141a7) (cherry pic

2016-03-23 Thread junping_du 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 f362b8a40 -> 762c7d436


YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. 
Contributed by Xuan Gong.
(cherry picked from commit af1d125f9ce35ec69a610674a1c5c60cc17141a7)
(cherry picked from commit 76602161c006ddc9cf2404ac762f127f107d14cd)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/762c7d43
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/762c7d43
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/762c7d43

Branch: refs/heads/branch-2.8
Commit: 762c7d4361d8acf34aff46b2a01f38efb989c777
Parents: f362b8a
Author: Junping Du 
Authored: Wed Mar 23 08:57:16 2016 -0700
Committer: Junping Du 
Committed: Wed Mar 23 09:02:56 2016 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   6 ++
 .../api/impl/FileSystemTimelineWriter.java  | 104 ++-
 2 files changed, 86 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/762c7d43/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 5e1c6fa..8018d1c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1679,6 +1679,12 @@ public class YarnConfiguration extends Configuration {
   public static final long
   TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT = 7 * 60;
 
+  public static final String
+  TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE =
+  TIMELINE_SERVICE_CLIENT_PREFIX + "internal-attempt-dir-cache-size";
+  public static final int
+  DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE = 1000;
+
   // This is temporary solution. The configuration will be deleted once we have
   // the FileSystem API to check whether append operation is supported or not.
   public static final String TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND

http://git-wip-us.apache.org/repos/asf/hadoop/blob/762c7d43/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index b471b3b..35d9970 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -26,6 +26,7 @@ import java.net.URI;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -97,6 +98,7 @@ public class FileSystemTimelineWriter extends TimelineWriter{
   private long ttl;
   private LogFDsCache logFDsCache = null;
   private boolean isAppendSupported;
+  private final AttemptDirCache attemptDirCache;
 
   public FileSystemTimelineWriter(Configuration conf,
   UserGroupInformation authUgi, Client client, URI resURI)
@@ -158,6 +160,15 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 
 objMapper = createObjectMapper();
 
+int attemptDirCacheSize = conf.getInt(
+YarnConfiguration
+.TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE,
+YarnConfiguration
+.DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE);
+
+attemptDirCache =
+new AttemptDirCache(attemptDirCacheSize, fs, activePath);
+
 if (LOG.isDebugEnabled()) {
   StringBuilder debugMSG = new StringBuilder();
   debugMSG.append(
@@ -199,7 +210,7 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 = new ArrayList();
 List entitiesToEntityCache
 = new ArrayList();
-Path attemptDir = createAttemptDir(appAttemptId);
+Path attemptDir = attemptDirCache.getAppAttemptDir(appAttemptId);
 
 for (TimelineEntity entity :

hadoop git commit: YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. Contributed by Xuan Gong. (cherry picked from commit af1d125f9ce35ec69a610674a1c5c60cc17141a7)

2016-03-23 Thread junping_du 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 7a3fd1bc1 -> 76602161c


YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. 
Contributed by Xuan Gong.
(cherry picked from commit af1d125f9ce35ec69a610674a1c5c60cc17141a7)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76602161
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76602161
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76602161

Branch: refs/heads/branch-2
Commit: 76602161c006ddc9cf2404ac762f127f107d14cd
Parents: 7a3fd1b
Author: Junping Du 
Authored: Wed Mar 23 08:57:16 2016 -0700
Committer: Junping Du 
Committed: Wed Mar 23 08:58:05 2016 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   6 ++
 .../api/impl/FileSystemTimelineWriter.java  | 104 ++-
 2 files changed, 86 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/76602161/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 174d0c4..e7d16d7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1747,6 +1747,12 @@ public class YarnConfiguration extends Configuration {
   public static final long
   TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT = 7 * 60;
 
+  public static final String
+  TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE =
+  TIMELINE_SERVICE_CLIENT_PREFIX + "internal-attempt-dir-cache-size";
+  public static final int
+  DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE = 1000;
+
   // This is temporary solution. The configuration will be deleted once we have
   // the FileSystem API to check whether append operation is supported or not.
   public static final String TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76602161/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index b471b3b..35d9970 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -26,6 +26,7 @@ import java.net.URI;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -97,6 +98,7 @@ public class FileSystemTimelineWriter extends TimelineWriter{
   private long ttl;
   private LogFDsCache logFDsCache = null;
   private boolean isAppendSupported;
+  private final AttemptDirCache attemptDirCache;
 
   public FileSystemTimelineWriter(Configuration conf,
   UserGroupInformation authUgi, Client client, URI resURI)
@@ -158,6 +160,15 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 
 objMapper = createObjectMapper();
 
+int attemptDirCacheSize = conf.getInt(
+YarnConfiguration
+.TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE,
+YarnConfiguration
+.DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE);
+
+attemptDirCache =
+new AttemptDirCache(attemptDirCacheSize, fs, activePath);
+
 if (LOG.isDebugEnabled()) {
   StringBuilder debugMSG = new StringBuilder();
   debugMSG.append(
@@ -199,7 +210,7 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 = new ArrayList();
 List entitiesToEntityCache
 = new ArrayList();
-Path attemptDir = createAttemptDir(appAttemptId);
+Path attemptDir = attemptDirCache.getAppAttemptDir(appAttemptId);
 
 for (TimelineEntity entity : entities) {
   if

hadoop git commit: YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. Contributed by Xuan Gong.

2016-03-23 Thread junping_du 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 368c77376 -> af1d125f9


YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. 
Contributed by Xuan Gong.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af1d125f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af1d125f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af1d125f

Branch: refs/heads/trunk
Commit: af1d125f9ce35ec69a610674a1c5c60cc17141a7
Parents: 368c773
Author: Junping Du 
Authored: Wed Mar 23 08:57:16 2016 -0700
Committer: Junping Du 
Committed: Wed Mar 23 08:57:16 2016 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   6 ++
 .../api/impl/FileSystemTimelineWriter.java  | 104 ++-
 2 files changed, 86 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/af1d125f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ff4b493..8acee57 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1747,6 +1747,12 @@ public class YarnConfiguration extends Configuration {
   public static final long
   TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT = 7 * 60;
 
+  public static final String
+  TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE =
+  TIMELINE_SERVICE_CLIENT_PREFIX + "internal-attempt-dir-cache-size";
+  public static final int
+  DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE = 1000;
+
   // This is temporary solution. The configuration will be deleted once we have
   // the FileSystem API to check whether append operation is supported or not.
   public static final String TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND

http://git-wip-us.apache.org/repos/asf/hadoop/blob/af1d125f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index b471b3b..35d9970 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -26,6 +26,7 @@ import java.net.URI;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -97,6 +98,7 @@ public class FileSystemTimelineWriter extends TimelineWriter{
   private long ttl;
   private LogFDsCache logFDsCache = null;
   private boolean isAppendSupported;
+  private final AttemptDirCache attemptDirCache;
 
   public FileSystemTimelineWriter(Configuration conf,
   UserGroupInformation authUgi, Client client, URI resURI)
@@ -158,6 +160,15 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 
 objMapper = createObjectMapper();
 
+int attemptDirCacheSize = conf.getInt(
+YarnConfiguration
+.TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE,
+YarnConfiguration
+.DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE);
+
+attemptDirCache =
+new AttemptDirCache(attemptDirCacheSize, fs, activePath);
+
 if (LOG.isDebugEnabled()) {
   StringBuilder debugMSG = new StringBuilder();
   debugMSG.append(
@@ -199,7 +210,7 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 = new ArrayList();
 List entitiesToEntityCache
 = new ArrayList();
-Path attemptDir = createAttemptDir(appAttemptId);
+Path attemptDir = attemptDirCache.getAppAttemptDir(appAttemptId);
 
 for (TimelineEntity entity : entities) {
   if (summaryEntityTypes.contains(entity.getEntityType())) {
@@ -279,32 +290,11 @@ public class FileSystemTimelineWriter extends