hadoop git commit: YARN-6999. Add log about how to solve Error: Could not find or load main class org.apache.hadoop.mapreduce.v2.app.MRAppMaster. (Linlin Zhou via gtcarrera9)

2017-08-25 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 c22b5108d -> b7f319c53


YARN-6999. Add log about how to solve Error: Could not find or load main class 
org.apache.hadoop.mapreduce.v2.app.MRAppMaster. (Linlin Zhou via gtcarrera9)

(cherry picked from commit 3bee7046c52bff681a5aecc956ba9be6c3543397)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7f319c5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7f319c5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7f319c5

Branch: refs/heads/branch-2
Commit: b7f319c53c51e72c8df8cc92df5ec7fe1ab303b1
Parents: c22b510
Author: Li Lu <gtcarre...@apache.org>
Authored: Fri Aug 25 22:43:25 2017 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Fri Aug 25 22:45:20 2017 -0700

--
 .../launcher/ContainerLaunch.java   | 27 +++-
 1 file changed, 26 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7f319c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 987bc30..c2d8222 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -540,9 +540,11 @@ public class ContainerLaunch implements Callable {
 errorFileIS = fileSystem.open(errorFile);
 errorFileIS.readFully(startPosition, tailBuffer);
 
+String tailBufferMsg = new String(tailBuffer, StandardCharsets.UTF_8);
 diagnosticInfo.append("Last ").append(tailSizeInBytes)
 .append(" bytes of ").append(errorFile.getName()).append(" :\n")
-.append(new String(tailBuffer, StandardCharsets.UTF_8));
+.append(tailBufferMsg).append("\n")
+.append(analysesErrorMsgOfContainerExitWithFailure(tailBufferMsg));
   }
 } catch (IOException e) {
   LOG.error("Failed to get tail of the container's error log file", e);
@@ -556,6 +558,29 @@ public class ContainerLaunch implements Callable {
 diagnosticInfo.toString()));
   }
 
+  private String analysesErrorMsgOfContainerExitWithFailure(String errorMsg) {
+StringBuilder analysis = new StringBuilder();
+if (errorMsg.indexOf("Error: Could not find or load main class"
++ " org.apache.hadoop.mapreduce") != -1) {
+  analysis.append("Please check whether your etc/hadoop/mapred-site.xml "
+  + "contains the below configuration:\n");
+  analysis.append("\n")
+  .append("  yarn.app.mapreduce.am.env\n")
+  .append("  HADOOP_MAPRED_HOME=${full path of your hadoop "
+  + "distribution directory}\n")
+  .append("\n\n")
+  .append("  mapreduce.map.env\n")
+  .append("  HADOOP_MAPRED_HOME=${full path of your hadoop "
+  + "distribution directory}\n")
+  .append("\n\n")
+  .append("  mapreduce.reduce.e nv\n")
+  .append("  HADOOP_MAPRED_HOME=${full path of your hadoop "
+  + "distribution directory}\n")
+  .append("\n");
+}
+return analysis.toString();
+  }
+
   protected String getPidFileSubpath(String appIdStr, String containerIdStr) {
 return getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
 + String.format(ContainerLaunch.PID_FILE_NAME_FMT, containerIdStr);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6999. Add log about how to solve Error: Could not find or load main class org.apache.hadoop.mapreduce.v2.app.MRAppMaster. (Linlin Zhou via gtcarrera9)

2017-08-25 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk b89ffcff3 -> 3bee7046c


YARN-6999. Add log about how to solve Error: Could not find or load main class 
org.apache.hadoop.mapreduce.v2.app.MRAppMaster. (Linlin Zhou via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3bee7046
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3bee7046
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3bee7046

Branch: refs/heads/trunk
Commit: 3bee7046c52bff681a5aecc956ba9be6c3543397
Parents: b89ffcf
Author: Li Lu <gtcarre...@apache.org>
Authored: Fri Aug 25 22:43:25 2017 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Fri Aug 25 22:43:25 2017 -0700

--
 .../launcher/ContainerLaunch.java   | 27 +++-
 1 file changed, 26 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bee7046/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index a0055c5..3f50c3f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -564,9 +564,11 @@ public class ContainerLaunch implements Callable {
 errorFileIS = fileSystem.open(errorFile);
 errorFileIS.readFully(startPosition, tailBuffer);
 
+String tailBufferMsg = new String(tailBuffer, StandardCharsets.UTF_8);
 diagnosticInfo.append("Last ").append(tailSizeInBytes)
 .append(" bytes of ").append(errorFile.getName()).append(" :\n")
-.append(new String(tailBuffer, StandardCharsets.UTF_8));
+.append(tailBufferMsg).append("\n")
+.append(analysesErrorMsgOfContainerExitWithFailure(tailBufferMsg));
   }
 } catch (IOException e) {
   LOG.error("Failed to get tail of the container's error log file", e);
@@ -580,6 +582,29 @@ public class ContainerLaunch implements Callable {
 diagnosticInfo.toString()));
   }
 
+  private String analysesErrorMsgOfContainerExitWithFailure(String errorMsg) {
+StringBuilder analysis = new StringBuilder();
+if (errorMsg.indexOf("Error: Could not find or load main class"
++ " org.apache.hadoop.mapreduce") != -1) {
+  analysis.append("Please check whether your etc/hadoop/mapred-site.xml "
+  + "contains the below configuration:\n");
+  analysis.append("\n")
+  .append("  yarn.app.mapreduce.am.env\n")
+  .append("  HADOOP_MAPRED_HOME=${full path of your hadoop "
+  + "distribution directory}\n")
+  .append("\n\n")
+  .append("  mapreduce.map.env\n")
+  .append("  HADOOP_MAPRED_HOME=${full path of your hadoop "
+  + "distribution directory}\n")
+  .append("\n\n")
+  .append("  mapreduce.reduce.e nv\n")
+  .append("  HADOOP_MAPRED_HOME=${full path of your hadoop "
+  + "distribution directory}\n")
+  .append("\n");
+}
+return analysis.toString();
+  }
+
   protected String getPidFileSubpath(String appIdStr, String containerIdStr) {
 return getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
 + String.format(ContainerLaunch.PID_FILE_NAME_FMT, containerIdStr);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6177. Yarn client should exit with an informative error message if an incompatible Jersey library is used at client. Contributed by Weiwei Yang.

2017-02-16 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 da944fe0f -> 83a589618


YARN-6177. Yarn client should exit with an informative error message if an 
incompatible Jersey library is used at client. Contributed by Weiwei Yang.

(cherry picked from commit 5d339c46f5b16b951afd82afd9e907b9aa2ded9a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83a58961
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83a58961
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83a58961

Branch: refs/heads/branch-2.8
Commit: 83a5896180019c6d147ace0ab992a783955aa5f8
Parents: da944fe
Author: Li Lu 
Authored: Thu Feb 16 13:40:26 2017 -0800
Committer: Li Lu 
Committed: Thu Feb 16 13:50:22 2017 -0800

--
 .../yarn/client/api/impl/YarnClientImpl.java|  10 ++
 .../yarn/client/api/impl/TestYarnClient.java| 165 ---
 2 files changed, 155 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a58961/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 93893f9..07250c8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -369,6 +369,16 @@ public class YarnClientImpl extends YarnClient {
 return null;
   }
   throw e;
+} catch (NoClassDefFoundError e) {
+  NoClassDefFoundError wrappedError = new NoClassDefFoundError(
+  e.getMessage() + ". It appears that the timeline client "
+  + "failed to initiate because an incompatible dependency "
+  + "in classpath. If timeline service is optional to this "
+  + "client, try to work around by setting "
+  + YarnConfiguration.TIMELINE_SERVICE_ENABLED
+  + " to false in client configuration.");
+  wrappedError.setStackTrace(e.getStackTrace());
+  throw wrappedError;
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a58961/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 8d199d2..c7d6c62 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -156,26 +156,6 @@ public class TestYarnClient {
   }
 
   @Test
-  public void testStartWithTimelineV15Failure() throws Exception{
-Configuration conf = new Configuration();
-conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
-conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
-conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
-true);
-YarnClient client = YarnClient.createYarnClient();
-if(client instanceof YarnClientImpl) {
-  YarnClientImpl impl = (YarnClientImpl) client;
-  YarnClientImpl spyClient = spy(impl);
-  when(spyClient.createTimelineClient()).thenThrow(
-  new IOException("ATS v1.5 client initialization failed. "));
-  spyClient.init(conf);
-  spyClient.start();
-  spyClient.getTimelineDelegationToken();
-  spyClient.stop();
-}
-  }
-
-  @Test
   public void testStartWithTimelineV15() throws Exception {
 Configuration conf = new Configuration();
 conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -186,6 +166,89 @@ public class TestYarnClient {
 client.stop();
   }
 
+  @Test
+  public void testStartTimelineClientWithErrors()
+  throws Exception {
+// If timeline client failed to init with a NoClassDefFoundError
+// it should be wrapped with an informative error message
+testCreateTimelineClientWithError(
+1.5f,
+true,
+false,
+new NoClassDefFoundError("Mock a

hadoop git commit: YARN-6177. Yarn client should exit with an informative error message if an incompatible Jersey library is used at client. Contributed by Weiwei Yang.

2017-02-16 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 f07b52dc5 -> 8fc67e597


YARN-6177. Yarn client should exit with an informative error message if an 
incompatible Jersey library is used at client. Contributed by Weiwei Yang.

(cherry picked from commit 5d339c46f5b16b951afd82afd9e907b9aa2ded9a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8fc67e59
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8fc67e59
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8fc67e59

Branch: refs/heads/branch-2
Commit: 8fc67e5973b0663b93271e4fc378441408aa6b1e
Parents: f07b52d
Author: Li Lu 
Authored: Thu Feb 16 13:40:26 2017 -0800
Committer: Li Lu 
Committed: Thu Feb 16 13:46:30 2017 -0800

--
 .../yarn/client/api/impl/YarnClientImpl.java|  10 ++
 .../yarn/client/api/impl/TestYarnClient.java| 165 ---
 2 files changed, 155 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fc67e59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index e406862..af46fbb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -376,6 +376,16 @@ public class YarnClientImpl extends YarnClient {
 return null;
   }
   throw e;
+} catch (NoClassDefFoundError e) {
+  NoClassDefFoundError wrappedError = new NoClassDefFoundError(
+  e.getMessage() + ". It appears that the timeline client "
+  + "failed to initiate because an incompatible dependency "
+  + "in classpath. If timeline service is optional to this "
+  + "client, try to work around by setting "
+  + YarnConfiguration.TIMELINE_SERVICE_ENABLED
+  + " to false in client configuration.");
+  wrappedError.setStackTrace(e.getStackTrace());
+  throw wrappedError;
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fc67e59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 6060a0b..d1c15e6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -156,26 +156,6 @@ public class TestYarnClient {
   }
 
   @Test
-  public void testStartWithTimelineV15Failure() throws Exception{
-Configuration conf = new Configuration();
-conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
-conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
-conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
-true);
-YarnClient client = YarnClient.createYarnClient();
-if(client instanceof YarnClientImpl) {
-  YarnClientImpl impl = (YarnClientImpl) client;
-  YarnClientImpl spyClient = spy(impl);
-  when(spyClient.createTimelineClient()).thenThrow(
-  new IOException("ATS v1.5 client initialization failed. "));
-  spyClient.init(conf);
-  spyClient.start();
-  spyClient.getTimelineDelegationToken();
-  spyClient.stop();
-}
-  }
-
-  @Test
   public void testStartWithTimelineV15() throws Exception {
 Configuration conf = new Configuration();
 conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -186,6 +166,89 @@ public class TestYarnClient {
 client.stop();
   }
 
+  @Test
+  public void testStartTimelineClientWithErrors()
+  throws Exception {
+// If timeline client failed to init with a NoClassDefFoundError
+// it should be wrapped with an informative error message
+testCreateTimelineClientWithError(
+1.5f,
+true,
+false,
+new NoClassDefFoundError("Mock a

hadoop git commit: YARN-6177. Yarn client should exit with an informative error message if an incompatible Jersey library is used at client. Contributed by Weiwei Yang.

2017-02-16 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 4fa1afdb8 -> 5d339c46f


YARN-6177. Yarn client should exit with an informative error message if an 
incompatible Jersey library is used at client. Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d339c46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d339c46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d339c46

Branch: refs/heads/trunk
Commit: 5d339c46f5b16b951afd82afd9e907b9aa2ded9a
Parents: 4fa1afd
Author: Li Lu 
Authored: Thu Feb 16 13:40:26 2017 -0800
Committer: Li Lu 
Committed: Thu Feb 16 13:41:42 2017 -0800

--
 .../yarn/client/api/impl/YarnClientImpl.java|  10 ++
 .../yarn/client/api/impl/TestYarnClient.java| 165 ---
 2 files changed, 155 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d339c46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 4a27fee..23b128c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -381,6 +381,16 @@ public class YarnClientImpl extends YarnClient {
 return null;
   }
   throw e;
+} catch (NoClassDefFoundError e) {
+  NoClassDefFoundError wrappedError = new NoClassDefFoundError(
+  e.getMessage() + ". It appears that the timeline client "
+  + "failed to initiate because an incompatible dependency "
+  + "in classpath. If timeline service is optional to this "
+  + "client, try to work around by setting "
+  + YarnConfiguration.TIMELINE_SERVICE_ENABLED
+  + " to false in client configuration.");
+  wrappedError.setStackTrace(e.getStackTrace());
+  throw wrappedError;
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d339c46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 240f31c..c2c9665 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -156,26 +156,6 @@ public class TestYarnClient {
   }
 
   @Test
-  public void testStartWithTimelineV15Failure() throws Exception{
-Configuration conf = new Configuration();
-conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
-conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
-conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
-true);
-YarnClient client = YarnClient.createYarnClient();
-if(client instanceof YarnClientImpl) {
-  YarnClientImpl impl = (YarnClientImpl) client;
-  YarnClientImpl spyClient = spy(impl);
-  when(spyClient.createTimelineClient()).thenThrow(
-  new IOException("ATS v1.5 client initialization failed. "));
-  spyClient.init(conf);
-  spyClient.start();
-  spyClient.getTimelineDelegationToken();
-  spyClient.stop();
-}
-  }
-
-  @Test
   public void testStartWithTimelineV15() throws Exception {
 Configuration conf = new Configuration();
 conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -186,6 +166,89 @@ public class TestYarnClient {
 client.stop();
   }
 
+  @Test
+  public void testStartTimelineClientWithErrors()
+  throws Exception {
+// If timeline client failed to init with a NoClassDefFoundError
+// it should be wrapped with an informative error message
+testCreateTimelineClientWithError(
+1.5f,
+true,
+false,
+new NoClassDefFoundError("Mock a NoClassDefFoundError"),
+new CreateTimelineClientErrorVerifier(1) {
+

hadoop git commit: YARN-5761. Separate QueueManager from Scheduler. (Xuan Gong via gtcarrera9)

2016-11-30 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 39b74b6f9 -> 8ffe86f78


YARN-5761. Separate QueueManager from Scheduler. (Xuan Gong via gtcarrera9)

(cherry picked from commit 69fb70c31aa277f7fb14b05c0185ddc5cd90793d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ffe86f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ffe86f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ffe86f7

Branch: refs/heads/branch-2
Commit: 8ffe86f7804af9446e2143ca0eee94fca1240c68
Parents: 39b74b6
Author: Li Lu <gtcarre...@apache.org>
Authored: Wed Nov 30 13:38:42 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Wed Nov 30 14:02:31 2016 -0800

--
 .../scheduler/SchedulerQueueManager.java|  75 
 .../scheduler/capacity/CapacityScheduler.java   | 294 +++
 .../capacity/CapacitySchedulerQueueManager.java | 361 +++
 .../capacity/TestApplicationLimits.java |  35 +-
 .../TestApplicationLimitsByPartition.java   |   7 +-
 .../scheduler/capacity/TestChildQueueOrder.java |   9 +-
 .../scheduler/capacity/TestLeafQueue.java   |   9 +-
 .../scheduler/capacity/TestParentQueue.java |  39 +-
 .../scheduler/capacity/TestReservations.java|   8 +-
 .../scheduler/capacity/TestUtils.java   |   2 +-
 10 files changed, 536 insertions(+), 303 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ffe86f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueueManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueueManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueueManager.java
new file mode 100644
index 000..92b989a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueueManager.java
@@ -0,0 +1,75 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
+
+/**
+ *
+ * Context of the Queues in Scheduler.
+ *
+ */
+@Private
+@Unstable
+public interface SchedulerQueueManager {
+
+  /**
+   * Get the root queue.
+   * @return root queue
+   */
+  T getRootQueue();
+
+  /**
+   * Get all the queues.
+   * @return a map contains all the queues as well as related queue names
+   */
+  Map<String, T> getQueues();
+
+  /**
+   * Remove the queue from the existing queue.
+   * @param queueName the queue name
+   */
+  void removeQueue(String queueName);
+
+  /**
+   * Add a new queue to the existing queues.
+   * @param queueName the queue name
+   * @param queue the queue object
+   */
+  void addQueue(String queueName, T queue);
+
+  /**
+   * Get a queue matching the specified queue name.
+   * @param queueName the queue name
+   * @return a queue object
+   */
+  T getQueue(String queueName);
+
+  /**
+   * Reinitialize the queues.
+   * @param newConf the configuration
+   * @throws IOException if fails to re-initialize queues
+   */
+  void reinitializeQueues(E newConf) throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ffe86f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/

hadoop git commit: YARN-5761. Separate QueueManager from Scheduler. (Xuan Gong via gtcarrera9)

2016-11-30 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 3fd844b99 -> 69fb70c31


YARN-5761. Separate QueueManager from Scheduler. (Xuan Gong via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69fb70c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69fb70c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69fb70c3

Branch: refs/heads/trunk
Commit: 69fb70c31aa277f7fb14b05c0185ddc5cd90793d
Parents: 3fd844b
Author: Li Lu <gtcarre...@apache.org>
Authored: Wed Nov 30 13:38:42 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Wed Nov 30 13:38:42 2016 -0800

--
 .../scheduler/SchedulerQueueManager.java|  75 
 .../scheduler/capacity/CapacityScheduler.java   | 294 +++
 .../capacity/CapacitySchedulerQueueManager.java | 361 +++
 .../capacity/TestApplicationLimits.java |  35 +-
 .../TestApplicationLimitsByPartition.java   |   7 +-
 .../scheduler/capacity/TestChildQueueOrder.java |   9 +-
 .../scheduler/capacity/TestLeafQueue.java   |   9 +-
 .../scheduler/capacity/TestParentQueue.java |  39 +-
 .../scheduler/capacity/TestReservations.java|   8 +-
 .../scheduler/capacity/TestUtils.java   |   2 +-
 10 files changed, 536 insertions(+), 303 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/69fb70c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueueManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueueManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueueManager.java
new file mode 100644
index 000..92b989a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueueManager.java
@@ -0,0 +1,75 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
+
+/**
+ *
+ * Context of the Queues in Scheduler.
+ *
+ */
+@Private
+@Unstable
+public interface SchedulerQueueManager {
+
+  /**
+   * Get the root queue.
+   * @return root queue
+   */
+  T getRootQueue();
+
+  /**
+   * Get all the queues.
+   * @return a map contains all the queues as well as related queue names
+   */
+  Map<String, T> getQueues();
+
+  /**
+   * Remove the queue from the existing queue.
+   * @param queueName the queue name
+   */
+  void removeQueue(String queueName);
+
+  /**
+   * Add a new queue to the existing queues.
+   * @param queueName the queue name
+   * @param queue the queue object
+   */
+  void addQueue(String queueName, T queue);
+
+  /**
+   * Get a queue matching the specified queue name.
+   * @param queueName the queue name
+   * @return a queue object
+   */
+  T getQueue(String queueName);
+
+  /**
+   * Reinitialize the queues.
+   * @param newConf the configuration
+   * @throws IOException if fails to re-initialize queues
+   */
+  void reinitializeQueues(E newConf) throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69fb70c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/

hadoop git commit: YARN-4668. Reuse objectMapper instance in Yarn. (Yiqun Lin via gtcarrera9)

2016-10-27 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 58ac40b55 -> 2b0fd1f4e


YARN-4668. Reuse objectMapper instance in Yarn. (Yiqun Lin via gtcarrera9)

(cherry picked from commit 221582c4ab0ff1d5936f754f23da140aac656654)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b0fd1f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b0fd1f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b0fd1f4

Branch: refs/heads/branch-2
Commit: 2b0fd1f4e28f667d670e9e9b6514e1ec1f70c832
Parents: 58ac40b
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Oct 27 15:19:59 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Oct 27 15:26:05 2016 -0700

--
 .../hadoop/yarn/client/api/impl/TimelineClientImpl.java  | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b0fd1f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index c433854..d4c68f7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -85,6 +85,7 @@ public class TimelineClientImpl extends TimelineClient {
 
   private static final Log LOG = LogFactory.getLog(TimelineClientImpl.class);
   private static final String RESOURCE_URI_STR = "/ws/v1/timeline/";
+  private static final ObjectMapper MAPPER = new ObjectMapper();
   private static final Joiner JOINER = Joiner.on("");
   public final static int DEFAULT_SOCKET_TIMEOUT = 1 * 60 * 1000; // 1 minute
 
@@ -563,15 +564,14 @@ public class TimelineClientImpl extends TimelineClient {
   LOG.error("File [" + jsonFile.getAbsolutePath() + "] doesn't exist");
   return;
 }
-ObjectMapper mapper = new ObjectMapper();
-YarnJacksonJaxbJsonProvider.configObjectMapper(mapper);
+YarnJacksonJaxbJsonProvider.configObjectMapper(MAPPER);
 TimelineEntities entities = null;
 TimelineDomains domains = null;
 try {
   if (type.equals(ENTITY_DATA_TYPE)) {
-entities = mapper.readValue(jsonFile, TimelineEntities.class);
+entities = MAPPER.readValue(jsonFile, TimelineEntities.class);
   } else if (type.equals(DOMAIN_DATA_TYPE)){
-domains = mapper.readValue(jsonFile, TimelineDomains.class);
+domains = MAPPER.readValue(jsonFile, TimelineDomains.class);
   }
 } catch (Exception e) {
   LOG.error("Error when reading  " + e.getMessage());


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-4668. Reuse objectMapper instance in Yarn. (Yiqun Lin via gtcarrera9)

2016-10-27 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 892a8348f -> 221582c4a


YARN-4668. Reuse objectMapper instance in Yarn. (Yiqun Lin via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/221582c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/221582c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/221582c4

Branch: refs/heads/trunk
Commit: 221582c4ab0ff1d5936f754f23da140aac656654
Parents: 892a834
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Oct 27 15:19:59 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Oct 27 15:20:17 2016 -0700

--
 .../hadoop/yarn/client/api/impl/TimelineClientImpl.java  | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/221582c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index dc4d3e6..d969c59 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -98,6 +98,7 @@ import com.sun.jersey.core.util.MultivaluedMapImpl;
 public class TimelineClientImpl extends TimelineClient {
 
   private static final Log LOG = LogFactory.getLog(TimelineClientImpl.class);
+  private static final ObjectMapper MAPPER = new ObjectMapper();
   private static final String RESOURCE_URI_STR_V1 = "/ws/v1/timeline/";
   private static final String RESOURCE_URI_STR_V2 = "/ws/v2/timeline/";
   private static final Joiner JOINER = Joiner.on("");
@@ -765,15 +766,14 @@ public class TimelineClientImpl extends TimelineClient {
   LOG.error("File [" + jsonFile.getAbsolutePath() + "] doesn't exist");
   return;
 }
-ObjectMapper mapper = new ObjectMapper();
-YarnJacksonJaxbJsonProvider.configObjectMapper(mapper);
+YarnJacksonJaxbJsonProvider.configObjectMapper(MAPPER);
 TimelineEntities entities = null;
 TimelineDomains domains = null;
 try {
   if (type.equals(ENTITY_DATA_TYPE)) {
-entities = mapper.readValue(jsonFile, TimelineEntities.class);
+entities = MAPPER.readValue(jsonFile, TimelineEntities.class);
   } else if (type.equals(DOMAIN_DATA_TYPE)){
-domains = mapper.readValue(jsonFile, TimelineDomains.class);
+domains = MAPPER.readValue(jsonFile, TimelineDomains.class);
   }
 } catch (Exception e) {
   LOG.error("Error when reading  " + e.getMessage());


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-4205. Addendum patch to fix branch-2.

2016-09-29 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 cf954d7a4 -> 919c0802f


YARN-4205. Addendum patch to fix branch-2.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/919c0802
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/919c0802
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/919c0802

Branch: refs/heads/branch-2
Commit: 919c0802fa397647466c8554e96baa0e23823b02
Parents: cf954d7
Author: Li Lu 
Authored: Thu Sep 29 15:41:31 2016 -0700
Committer: Li Lu 
Committed: Thu Sep 29 15:41:31 2016 -0700

--
 .../resourcemanager/rmapp/monitor/RMAppLifetimeMonitor.java | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/919c0802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/monitor/RMAppLifetimeMonitor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/monitor/RMAppLifetimeMonitor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/monitor/RMAppLifetimeMonitor.java
index e550c97..19a2c7b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/monitor/RMAppLifetimeMonitor.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/monitor/RMAppLifetimeMonitor.java
@@ -100,7 +100,10 @@ public class RMAppLifetimeMonitor
   ApplicationTimeoutType timeoutType, long monitorStartTime, long timeout) 
{
 RMAppToMonitor appToMonitor = new RMAppToMonitor(appId, timeoutType);
 register(appToMonitor, monitorStartTime);
-monitoredApps.putIfAbsent(appToMonitor, timeout);
+Long oldTimeout = monitoredApps.get(appToMonitor);
+if (oldTimeout == null) {
+  monitoredApps.put(appToMonitor, timeout);
+}
   }
 
   @Override


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5229. Refactor #isApplicationEntity and #getApplicationEvent from HBaseTimelineWriterImpl. (Vrushali C via gtcarrera9)

2016-07-29 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-5355-branch-2 2bf68b448 -> d6b374360


YARN-5229. Refactor #isApplicationEntity and #getApplicationEvent from 
HBaseTimelineWriterImpl. (Vrushali C via gtcarrera9)

(cherry picked from commit 5d7ad396dbf84ffb0bacb3372d149828dfb4b4ca)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6b37436
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6b37436
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6b37436

Branch: refs/heads/YARN-5355-branch-2
Commit: d6b37436079bdcce6cb3b24035f5a5e2f2792a66
Parents: 2bf68b4
Author: Li Lu <gtcarre...@apache.org>
Authored: Fri Jul 29 17:33:07 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Fri Jul 29 17:34:46 2016 -0700

--
 .../timelineservice/ApplicationEntity.java  | 28 
 .../timelineservice/TestApplicationEntity.java  | 71 
 .../storage/HBaseTimelineWriterImpl.java| 35 ++
 3 files changed, 103 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6b37436/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
index 6075ec4..20226aa 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
@@ -49,4 +49,32 @@ public class ApplicationEntity extends 
HierarchicalTimelineEntity {
   public void setQueue(String queue) {
 addInfo(QUEUE_INFO_KEY, queue);
   }
+
+  /**
+   * Checks if the input TimelineEntity object is an ApplicationEntity.
+   *
+   * @param te TimelineEntity object.
+   * @return true if input is an ApplicationEntity, false otherwise
+   */
+  public static boolean isApplicationEntity(TimelineEntity te) {
+return (te == null ? false
+: te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString()));
+  }
+
+  /**
+   * @param te TimelineEntity object.
+   * @param eventId event with this id needs to be fetched
+   * @return TimelineEvent if TimelineEntity contains the desired event.
+   */
+  public static TimelineEvent getApplicationEvent(TimelineEntity te,
+  String eventId) {
+if (isApplicationEntity(te)) {
+  for (TimelineEvent event : te.getEvents()) {
+if (event.getId().equals(eventId)) {
+  return event;
+}
+  }
+}
+return null;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6b37436/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestApplicationEntity.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestApplicationEntity.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestApplicationEntity.java
new file mode 100644
index 000..c3f2777
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestApplicationEntity.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.timelineservice;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+/**
+ * Various te

hadoop git commit: YARN-5229. Refactor #isApplicationEntity and #getApplicationEvent from HBaseTimelineWriterImpl. (Vrushali C via gtcarrera9)

2016-07-29 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-5355 d0a62d8ad -> 5d7ad396d


YARN-5229. Refactor #isApplicationEntity and #getApplicationEvent from 
HBaseTimelineWriterImpl. (Vrushali C via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d7ad396
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d7ad396
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d7ad396

Branch: refs/heads/YARN-5355
Commit: 5d7ad396dbf84ffb0bacb3372d149828dfb4b4ca
Parents: d0a62d8
Author: Li Lu <gtcarre...@apache.org>
Authored: Fri Jul 29 17:33:07 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Fri Jul 29 17:33:07 2016 -0700

--
 .../timelineservice/ApplicationEntity.java  | 28 
 .../timelineservice/TestApplicationEntity.java  | 71 
 .../storage/HBaseTimelineWriterImpl.java| 35 ++
 3 files changed, 103 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d7ad396/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
index 6075ec4..20226aa 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
@@ -49,4 +49,32 @@ public class ApplicationEntity extends 
HierarchicalTimelineEntity {
   public void setQueue(String queue) {
 addInfo(QUEUE_INFO_KEY, queue);
   }
+
+  /**
+   * Checks if the input TimelineEntity object is an ApplicationEntity.
+   *
+   * @param te TimelineEntity object.
+   * @return true if input is an ApplicationEntity, false otherwise
+   */
+  public static boolean isApplicationEntity(TimelineEntity te) {
+return (te == null ? false
+: te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString()));
+  }
+
+  /**
+   * @param te TimelineEntity object.
+   * @param eventId event with this id needs to be fetched
+   * @return TimelineEvent if TimelineEntity contains the desired event.
+   */
+  public static TimelineEvent getApplicationEvent(TimelineEntity te,
+  String eventId) {
+if (isApplicationEntity(te)) {
+  for (TimelineEvent event : te.getEvents()) {
+if (event.getId().equals(eventId)) {
+  return event;
+}
+  }
+}
+return null;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d7ad396/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestApplicationEntity.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestApplicationEntity.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestApplicationEntity.java
new file mode 100644
index 000..c3f2777
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestApplicationEntity.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.timelineservice;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+/**
+ * Various tests for the ApplicationEntity class.
+ *
+ */
+public class TestApplicatio

hadoop git commit: YARN-5436. Race in AsyncDispatcher can cause random test failures in Tez (probably YARN also). (Zhiyuan Yang via gtcarrera9)

2016-07-28 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 24d464a15 -> 0b5d96abb


YARN-5436. Race in AsyncDispatcher can cause random test failures in Tez 
(probably YARN also). (Zhiyuan Yang via gtcarrera9)

(cherry picked from commit 7086fc72eebc41fd174d91839ed703c014aac920)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b5d96ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b5d96ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b5d96ab

Branch: refs/heads/branch-2
Commit: 0b5d96abb5c3c1831489588ed81e15f9bf18bbe5
Parents: 24d464a
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Jul 28 16:50:57 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Jul 28 16:52:21 2016 -0700

--
 .../hadoop/yarn/event/AsyncDispatcher.java  |  8 ++-
 .../hadoop/yarn/event/DrainDispatcher.java  | 53 +++-
 2 files changed, 55 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b5d96ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index f5361c8..5dea1c8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -59,6 +59,9 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 
   // Indicates all the remaining dispatcher's events on stop have been drained
   // and processed.
+  // Race condition happens if dispatcher thread sets drained to true between
+  // handler setting drained to false and enqueueing event. YARN-3878 decided
+  // to ignore it because of its tiny impact. Also see YARN-5436.
   private volatile boolean drained = true;
   private final Object waitForDrained = new Object();
 
@@ -300,9 +303,4 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   protected boolean isEventThreadWaiting() {
 return eventHandlingThread.getState() == Thread.State.WAITING;
   }
-
-  @VisibleForTesting
-  protected boolean isDrained() {
-return this.drained;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b5d96ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
index e4a5a82..cf4b1b5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
@@ -22,6 +22,10 @@ import java.util.concurrent.LinkedBlockingQueue;
 
 @SuppressWarnings("rawtypes")
 public class DrainDispatcher extends AsyncDispatcher {
+  private volatile boolean drained = false;
+  private volatile boolean stopped = false;
+  private final BlockingQueue queue;
+  private final Object mutex;
 
   public DrainDispatcher() {
 this(new LinkedBlockingQueue());
@@ -29,6 +33,8 @@ public class DrainDispatcher extends AsyncDispatcher {
 
   public DrainDispatcher(BlockingQueue eventQueue) {
 super(eventQueue);
+this.queue = eventQueue;
+this.mutex = this;
   }
 
   /**
@@ -44,8 +50,53 @@ public class DrainDispatcher extends AsyncDispatcher {
* Busy loop waiting for all queued events to drain.
*/
   public void await() {
-while (!isDrained()) {
+while (!drained) {
   Thread.yield();
 }
   }
+
+  @Override
+  Runnable createThread() {
+return new Runnable() {
+  @Override
+  public void run() {
+while (!stopped && !Thread.currentThread().isInterrupted()) {
+  synchronized (mutex) {
+// !drained if dispatch queued new events on this dispatcher
+drained = queue.isEmpty();
+  }
+  Event event;
+  try {
+event = queue.take();
+  } catch (InterruptedException ie) {
+return;
+  }
+  if (event != null) {
+dispatch(event);
+  }
+}
+  }
+

hadoop git commit: YARN-5436. Race in AsyncDispatcher can cause random test failures in Tez (probably YARN also). (Zhiyuan Yang via gtcarrera9)

2016-07-28 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk d9aae22fd -> 7086fc72e


YARN-5436. Race in AsyncDispatcher can cause random test failures in Tez 
(probably YARN also). (Zhiyuan Yang via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7086fc72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7086fc72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7086fc72

Branch: refs/heads/trunk
Commit: 7086fc72eebc41fd174d91839ed703c014aac920
Parents: d9aae22
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Jul 28 16:50:57 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Jul 28 16:50:57 2016 -0700

--
 .../hadoop/yarn/event/AsyncDispatcher.java  |  8 ++-
 .../hadoop/yarn/event/DrainDispatcher.java  | 53 +++-
 2 files changed, 55 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7086fc72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index f5361c8..5dea1c8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -59,6 +59,9 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 
   // Indicates all the remaining dispatcher's events on stop have been drained
   // and processed.
+  // Race condition happens if dispatcher thread sets drained to true between
+  // handler setting drained to false and enqueueing event. YARN-3878 decided
+  // to ignore it because of its tiny impact. Also see YARN-5436.
   private volatile boolean drained = true;
   private final Object waitForDrained = new Object();
 
@@ -300,9 +303,4 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   protected boolean isEventThreadWaiting() {
 return eventHandlingThread.getState() == Thread.State.WAITING;
   }
-
-  @VisibleForTesting
-  protected boolean isDrained() {
-return this.drained;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7086fc72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
index e4a5a82..cf4b1b5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
@@ -22,6 +22,10 @@ import java.util.concurrent.LinkedBlockingQueue;
 
 @SuppressWarnings("rawtypes")
 public class DrainDispatcher extends AsyncDispatcher {
+  private volatile boolean drained = false;
+  private volatile boolean stopped = false;
+  private final BlockingQueue queue;
+  private final Object mutex;
 
   public DrainDispatcher() {
 this(new LinkedBlockingQueue());
@@ -29,6 +33,8 @@ public class DrainDispatcher extends AsyncDispatcher {
 
   public DrainDispatcher(BlockingQueue eventQueue) {
 super(eventQueue);
+this.queue = eventQueue;
+this.mutex = this;
   }
 
   /**
@@ -44,8 +50,53 @@ public class DrainDispatcher extends AsyncDispatcher {
* Busy loop waiting for all queued events to drain.
*/
   public void await() {
-while (!isDrained()) {
+while (!drained) {
   Thread.yield();
 }
   }
+
+  @Override
+  Runnable createThread() {
+return new Runnable() {
+  @Override
+  public void run() {
+while (!stopped && !Thread.currentThread().isInterrupted()) {
+  synchronized (mutex) {
+// !drained if dispatch queued new events on this dispatcher
+drained = queue.isEmpty();
+  }
+  Event event;
+  try {
+event = queue.take();
+  } catch (InterruptedException ie) {
+return;
+  }
+  if (event != null) {
+dispatch(event);
+  }
+}
+  }
+};
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public

hadoop git commit: YARN-5440. Use AHSClient in YarnClient when TimelineServer is running. (Xuan Gong via gtcarrera9)

2016-07-28 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 7f3c306e2 -> 26de4f0de


YARN-5440. Use AHSClient in YarnClient when TimelineServer is running. (Xuan 
Gong via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26de4f0d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26de4f0d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26de4f0d

Branch: refs/heads/trunk
Commit: 26de4f0de789f58736d1dc383125cffb54debdd0
Parents: 7f3c306
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Jul 28 10:23:18 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Jul 28 10:23:18 2016 -0700

--
 .../yarn/client/api/impl/YarnClientImpl.java | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/26de4f0d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index ae8716e..dee31c5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -165,13 +165,6 @@ public class YarnClientImpl extends YarnClient {
 
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
 }
 
-if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
-  YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
-  historyServiceEnabled = true;
-  historyClient = AHSClient.createAHSClient();
-  historyClient.init(conf);
-}
-
 if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
 YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
   timelineServiceEnabled = true;
@@ -181,6 +174,18 @@ public class YarnClientImpl extends YarnClient {
   timelineService = TimelineUtils.buildTimelineTokenService(conf);
 }
 
+// The AHSClientService is enabled by default when we start the
+// TimelineServer which means we are able to get history information
+// for applications/applicationAttempts/containers by using ahsClient
+// when the TimelineServer is running.
+if (timelineServiceEnabled || conf.getBoolean(
+YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
+  historyServiceEnabled = true;
+  historyClient = AHSClient.createAHSClient();
+  historyClient.init(conf);
+}
+
 timelineServiceBestEffort = conf.getBoolean(
 YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
 YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5440. Use AHSClient in YarnClient when TimelineServer is running. (Xuan Gong via gtcarrera9)

2016-07-28 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 932bf7924 -> 310245063


YARN-5440. Use AHSClient in YarnClient when TimelineServer is running. (Xuan 
Gong via gtcarrera9)

(cherry picked from commit 26de4f0de789f58736d1dc383125cffb54debdd0)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31024506
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31024506
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31024506

Branch: refs/heads/branch-2
Commit: 310245063b46281dfb335bd11f285cf813628b8f
Parents: 932bf79
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Jul 28 10:23:18 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Jul 28 10:25:55 2016 -0700

--
 .../yarn/client/api/impl/YarnClientImpl.java | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/31024506/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index ae8716e..dee31c5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -165,13 +165,6 @@ public class YarnClientImpl extends YarnClient {
 
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
 }
 
-if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
-  YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
-  historyServiceEnabled = true;
-  historyClient = AHSClient.createAHSClient();
-  historyClient.init(conf);
-}
-
 if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
 YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
   timelineServiceEnabled = true;
@@ -181,6 +174,18 @@ public class YarnClientImpl extends YarnClient {
   timelineService = TimelineUtils.buildTimelineTokenService(conf);
 }
 
+// The AHSClientService is enabled by default when we start the
+// TimelineServer which means we are able to get history information
+// for applications/applicationAttempts/containers by using ahsClient
+// when the TimelineServer is running.
+if (timelineServiceEnabled || conf.getBoolean(
+YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
+  historyServiceEnabled = true;
+  historyClient = AHSClient.createAHSClient();
+  historyClient.init(conf);
+}
+
 timelineServiceBestEffort = conf.getBoolean(
 YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
 YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5189. Make HBaseTimeline[Reader|Writer]Impl default and move FileSystemTimeline*Impl. (Joep Rottinghuis and Sangjin Lee via gtcarrera9)

2016-06-03 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-2928 646dd4da8 -> df49b170f


YARN-5189. Make HBaseTimeline[Reader|Writer]Impl default and move 
FileSystemTimeline*Impl. (Joep Rottinghuis and Sangjin Lee via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df49b170
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df49b170
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df49b170

Branch: refs/heads/YARN-2928
Commit: df49b170fc46084baa5b9e77513f7761b326c1ef
Parents: 646dd4d
Author: Li Lu <gtcarre...@apache.org>
Authored: Fri Jun 3 16:33:51 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Fri Jun 3 16:33:51 2016 -0700

--
 .../hadoop-mapreduce-client-jobclient/pom.xml   |  6 
 .../mapred/TestMRTimelineEventHandling.java |  3 ++
 .../hadoop/yarn/conf/YarnConfiguration.java |  8 --
 .../pom.xml |  1 +
 .../distributedshell/TestDistributedShell.java  | 18 ++--
 .../src/main/resources/yarn-default.xml | 29 +---
 .../hadoop-yarn-server-resourcemanager/pom.xml  |  6 
 .../TestSystemMetricsPublisherForV2.java|  3 ++
 .../TestTimelineServiceClientIntegration.java   |  8 +-
 .../collector/TimelineCollectorManager.java |  4 +--
 .../reader/TimelineReaderServer.java|  4 +--
 .../storage/FileSystemTimelineReaderImpl.java   |  5 +++-
 .../storage/FileSystemTimelineWriterImpl.java   |  4 ++-
 .../TestNMTimelineCollectorManager.java |  9 --
 ...TestPerNodeTimelineCollectorsAuxService.java |  4 +++
 .../reader/TestTimelineReaderServer.java|  5 +++-
 .../reader/TestTimelineReaderWebServices.java   | 13 +
 17 files changed, 99 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/df49b170/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
index bee9618..67a65fa 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
@@ -98,6 +98,12 @@
   test-jar
 
 
+  org.apache.hadoop
+  hadoop-yarn-server-timelineservice
+  test
+  test-jar
+
+
   org.hsqldb
   hsqldb
   test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df49b170/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index fde9e64..27ce48c 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.timeline.TimelineStore;
 import 
org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.junit.Assert;
 import org.junit.Test;
@@ -175,6 +176,8 @@ public class TestMRTimelineEventHandling {
 conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
 // enable new timeline service
 conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
+FileSystemTimelineWriterImpl.class, TimelineWriter.class);
 conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
 
 // enable aux-service based timeline collectors

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df49b170/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/h

hadoop git commit: YARN-5138. fix "no findbugs output file" error for hadoop-yarn-server-timelineservice-hbase-tests. (Vrushali C via gtcarrera9)

2016-05-26 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-2928 506ebff5f -> 10b26bb9f


YARN-5138. fix "no findbugs output file" error for 
hadoop-yarn-server-timelineservice-hbase-tests. (Vrushali C via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10b26bb9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10b26bb9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10b26bb9

Branch: refs/heads/YARN-2928
Commit: 10b26bb9fb20822d91b00aa7d394be102a292a2d
Parents: 506ebff
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu May 26 11:12:31 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu May 26 11:12:31 2016 -0700

--
 .../hadoop-yarn-server-timelineservice-hbase-tests/pom.xml| 7 +++
 1 file changed, 7 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/10b26bb9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
index b5dc3c0..a8e5195 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
@@ -362,6 +362,13 @@
   
 
   
+org.codehaus.mojo
+findbugs-maven-plugin
+ 
+  true
+
+  
+  
 maven-jar-plugin
 
   


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-13083. The number of javadocs warnings is limited to 100. (Gergely Novák via gtcarrera9)

2016-05-13 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 29699d8e7 -> 6b1af6ce6


HADOOP-13083. The number of javadocs warnings is limited to 100. (Gergely 
Novák via gtcarrera9)

(cherry picked from commit 3fa1380c221b9d659fb82c42284505ef19da38d0)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b1af6ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b1af6ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b1af6ce

Branch: refs/heads/branch-2.8
Commit: 6b1af6ce68cb1484fa1cb308e13694418094be6e
Parents: 29699d8
Author: Li Lu <gtcarre...@apache.org>
Authored: Fri May 13 13:43:29 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Fri May 13 13:46:28 2016 -0700

--
 hadoop-project/pom.xml | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b1af6ce/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 54ca9ee..fcadfce 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1043,6 +1043,9 @@
   org.apache.maven.plugins
   maven-javadoc-plugin
   2.8.1
+  
+-Xmaxwarns 1
+  
 
 
   org.apache.maven.plugins


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-13083. The number of javadocs warnings is limited to 100. (Gergely Novák via gtcarrera9)

2016-05-13 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 8a316676b -> 5c50c121a


HADOOP-13083. The number of javadocs warnings is limited to 100. (Gergely 
Novák via gtcarrera9)

(cherry picked from commit 3fa1380c221b9d659fb82c42284505ef19da38d0)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c50c121
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c50c121
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c50c121

Branch: refs/heads/branch-2
Commit: 5c50c121aa9dbd343673766beb1edeb7de536064
Parents: 8a31667
Author: Li Lu <gtcarre...@apache.org>
Authored: Fri May 13 13:43:29 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Fri May 13 13:45:05 2016 -0700

--
 hadoop-project/pom.xml | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c50c121/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 794b3c7..78fcc06 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1049,6 +1049,9 @@
   org.apache.maven.plugins
   maven-javadoc-plugin
   2.8.1
+  
+-Xmaxwarns 1
+  
 
 
   org.apache.maven.plugins


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-13083. The number of javadocs warnings is limited to 100. (Gergely Novák via gtcarrera9)

2016-05-13 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk f0ac18d00 -> 3fa1380c2


HADOOP-13083. The number of javadocs warnings is limited to 100. (Gergely 
Novák via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3fa1380c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3fa1380c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3fa1380c

Branch: refs/heads/trunk
Commit: 3fa1380c221b9d659fb82c42284505ef19da38d0
Parents: f0ac18d
Author: Li Lu <gtcarre...@apache.org>
Authored: Fri May 13 13:43:29 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Fri May 13 13:44:35 2016 -0700

--
 hadoop-project/pom.xml | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fa1380c/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 5e9693e..7994ded 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1060,6 +1060,9 @@
   org.apache.maven.plugins
   maven-javadoc-plugin
   2.8.1
+  
+-Xmaxwarns 1
+  
 
 
   org.apache.maven.plugins


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[37/50] [abbrv] hadoop git commit: Addendum to YARN-3863. Deleted files that were added incorrectly.

2016-05-04 Thread gtcarrera9 
Addendum to YARN-3863. Deleted files that were added incorrectly.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/974a9d7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/974a9d7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/974a9d7c

Branch: refs/heads/YARN-2928
Commit: 974a9d7cad7162c87576a92d28774592114d1317
Parents: 2b2df86
Author: Sangjin Lee 
Authored: Tue Apr 12 12:32:43 2016 -0700
Committer: Li Lu 
Committed: Wed May 4 16:35:33 2016 -0700

--
 .../reader/filter/TimelineExistsFilter.java | 62 -
 .../reader/filter/TimelineKeyValueFilter.java   | 48 -
 .../reader/filter/TimelineKeyValuesFilter.java  | 71 
 .../common/TimelineEntityFiltersType.java   | 71 
 4 files changed, 252 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/974a9d7c/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
--
diff --git 
a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
 
b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
deleted file mode 100644
index 36d0d7b..000
--- 
a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-/**
- * Filter class which represents filter to be applied based on existence of a
- * value.
- */
-@Private
-@Unstable
-public class TimelineExistsFilter extends TimelineFilter {
-
-  private final TimelineCompareOp compareOp;
-  private final String value;
-
-  public TimelineExistsFilter(TimelineCompareOp op, String value) {
-this.value = value;
-if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
-  throw new IllegalArgumentException("CompareOp for exists filter should " 
+
-  "be EQUAL or NOT_EQUAL");
-}
-this.compareOp = op;
-  }
-
-  @Override
-  public TimelineFilterType getFilterType() {
-return TimelineFilterType.EXISTS;
-  }
-
-  public String getValue() {
-return value;
-  }
-
-  public TimelineCompareOp getCompareOp() {
-return compareOp;
-  }
-
-  @Override
-  public String toString() {
-return String.format("%s (%s %s)",
-this.getClass().getSimpleName(), this.compareOp.name(), this.value);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/974a9d7c/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
--
diff --git 
a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
 
b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
deleted file mode 100644
index 58f0ee9..000
--- 
a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor

[43/50] [abbrv] hadoop git commit: YARN-3816. [Aggregation] App-level aggregation and accumulation for YARN system metrics (Li Lu via sjlee)

2016-05-04 Thread gtcarrera9 
YARN-3816. [Aggregation] App-level aggregation and accumulation for YARN system 
metrics (Li Lu via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64d15d9c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64d15d9c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64d15d9c

Branch: refs/heads/YARN-2928
Commit: 64d15d9cc408072594102adad474f9302234f80c
Parents: 974a9d7
Author: Sangjin Lee 
Authored: Fri Apr 22 10:24:40 2016 -0700
Committer: Li Lu 
Committed: Wed May 4 16:35:37 2016 -0700

--
 .../records/timelineservice/TimelineMetric.java | 140 ++--
 .../TimelineMetricCalculator.java   | 115 ++
 .../TimelineMetricOperation.java| 167 +++
 .../timelineservice/TestTimelineMetric.java | 100 +
 .../TestTimelineServiceRecords.java |   6 +-
 .../timelineservice/NMTimelinePublisher.java|   4 +
 .../collector/AppLevelTimelineCollector.java|  72 +++
 .../collector/TimelineCollector.java| 213 ++-
 .../storage/TimelineAggregationTrack.java   |   2 +-
 .../collector/TestTimelineCollector.java| 127 +++
 .../TestFileSystemTimelineWriterImpl.java   |  43 +++-
 .../storage/TestHBaseTimelineStorage.java   |  35 ++-
 12 files changed, 998 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
index 2f60515..f0c6849 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
@@ -19,12 +19,13 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
-import java.util.Comparator;
+import java.util.Collections;
 import java.util.Map;
 import java.util.TreeMap;
 
@@ -48,13 +49,13 @@ public class TimelineMetric {
 
   private Type type;
   private String id;
-  private Comparator reverseComparator = new Comparator() {
-@Override
-public int compare(Long l1, Long l2) {
-  return l2.compareTo(l1);
-}
-  };
-  private TreeMap values = new TreeMap<>(reverseComparator);
+  // By default, not to do any aggregation operations. This field will NOT be
+  // persisted (like a "transient" member).
+  private TimelineMetricOperation realtimeAggregationOp
+  = TimelineMetricOperation.NOP;
+
+  private TreeMap values
+  = new TreeMap<>(Collections.reverseOrder());
 
   public TimelineMetric() {
 this(Type.SINGLE_VALUE);
@@ -83,6 +84,26 @@ public class TimelineMetric {
 this.id = metricId;
   }
 
+  /**
+   * Get the real time aggregation operation of this metric.
+   *
+   * @return Real time aggregation operation
+   */
+  public TimelineMetricOperation getRealtimeAggregationOp() {
+return realtimeAggregationOp;
+  }
+
+  /**
+   * Set the real time aggregation operation of this metric.
+   *
+   * @param op A timeline metric operation that the metric should perform on
+   *   real time aggregations
+   */
+  public void setRealtimeAggregationOp(
+  final TimelineMetricOperation op) {
+this.realtimeAggregationOp = op;
+  }
+
   // required by JAXB
   @InterfaceAudience.Private
   @XmlElement(name = "values")
@@ -98,8 +119,8 @@ public class TimelineMetric {
 if (type == Type.SINGLE_VALUE) {
   overwrite(vals);
 } else {
-  if (values != null) {
-this.values = new TreeMap(reverseComparator);
+  if (vals != null) {
+this.values = new TreeMap<>(Collections.reverseOrder());
 this.values.putAll(vals);
   } else {
 this.values = null;
@@ -166,11 +187,100 @@ public class TimelineMetric {
 
   @Override
   public String toString() {
-String str = "{id:" + id + ",

[29/50] [abbrv] hadoop git commit: YARN-4700. ATS storage has one extra record each time the RM got restarted. (Naganarasimha G R via Varun Saxena)

2016-05-04 Thread gtcarrera9 
YARN-4700. ATS storage has one extra record each time the RM got restarted. 
(Naganarasimha G R via Varun Saxena)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34248bde
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34248bde
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34248bde

Branch: refs/heads/YARN-2928
Commit: 34248bdeab608c18860777fc77ba0706a4fe7ed9
Parents: f59c743
Author: Varun Saxena 
Authored: Fri Mar 4 19:42:22 2016 +0530
Committer: Li Lu 
Committed: Wed May 4 16:33:34 2016 -0700

--
 .../storage/HBaseTimelineWriterImpl.java| 47 +
 .../storage/common/TimelineStorageUtils.java| 35 +++--
 .../storage/flow/FlowActivityRowKey.java| 27 +++---
 ...stTimelineReaderWebServicesHBaseStorage.java | 25 +
 .../storage/flow/TestFlowDataGenerator.java | 22 
 .../flow/TestHBaseStorageFlowActivity.java  | 53 
 .../storage/flow/TestHBaseStorageFlowRun.java   |  4 +-
 7 files changed, 96 insertions(+), 117 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/34248bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 997b175..1afe878 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -36,6 +36,7 @@ import 
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
 import 
org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
 import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
@@ -53,11 +54,11 @@ import 
org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
 import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
-import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
@@ -140,19 +141,22 @@ public class HBaseTimelineWriterImpl extends 
AbstractService implements
   storeRelations(rowKey, te, isApplication);
 
   if (isApplication) {
-if (TimelineStorageUtils.isApplicationCreated(te)) {
+TimelineEvent event = TimelineStorageUtils.getApplicationEvent(te,
+ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+if (event != null) {
   onApplicationCreated(clusterId, userId, flowName, flowVersion,
-  flowRunId, appId, te);
+  flowRunId, appId, te, event.getTimestamp());
 }
 // if it's an application entity, store metrics
 storeFlowMetricsAppRunning(clusterId, userId, flowName, flowRunId,
 appId, te);
 // if application

[17/50] [abbrv] hadoop git commit: YARN-4224. Support fetching entities by UID and change the REST interface to conform to current REST APIs' in YARN. (Varun Saxena via gtcarrera9)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
new file mode 100644
index 000..d052d51
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.servlet.http.HttpServletRequest;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+
+/**
+ * Set of utility methods to be used by timeline reader web services.
+ */
+final class TimelineReaderWebServicesUtils {
+  private TimelineReaderWebServicesUtils() {
+  }
+
+  /**
+   * Parse a delimited string and convert it into a set of strings. For
+   * instance, if delimiter is ",", then the string should be represented as
+   * "value1,value2,value3".
+   * @param str delimited string.
+   * @param delimiter string is delimited by this delimiter.
+   * @return set of strings.
+   */
+  static Set parseValuesStr(String str, String delimiter) {
+if (str == null || str.isEmpty()) {
+  return null;
+}
+Set strSet = new HashSet();
+String[] strs = str.split(delimiter);
+for (String aStr : strs) {
+  strSet.add(aStr.trim());
+}
+return strSet;
+  }
+
+  @SuppressWarnings("unchecked")
+  private static  void parseKeyValues(Map map, String str,
+  String pairsDelim, String keyValuesDelim, boolean stringValue,
+  boolean multipleValues) {
+String[] pairs = str.split(pairsDelim);
+for (String pair : pairs) {
+  if (pair == null || pair.trim().isEmpty()) {
+continue;
+  }
+  String[] pairStrs = pair.split(keyValuesDelim);
+  if (pairStrs.length < 2) {
+continue;
+  }
+  if (!stringValue) {
+try {
+  Object value =
+  GenericObjectMapper.OBJECT_READER.readValue(pairStrs[1].trim());
+  map.put(pairStrs[0].trim(), (T) value);
+} catch (IOException e) {
+  map.put(pairStrs[0].trim(), (T) pairStrs[1].trim());
+}
+  } else {
+String key = pairStrs[0].trim();
+if (multipleValues) {
+  Set values = new HashSet();
+  for (int i = 1; i < pairStrs.length; i++) {
+values.add(pairStrs[i].trim());
+  }
+  map.put(key, (T) values);
+} else {
+  map.put(key, (T) pairStrs[1].trim());
+}
+  }
+}
+  }
+
+  /**
+   * Parse a delimited string and convert it into a map of key-values with each
+   * key having a set of values. Both the key and values are interpreted as
+   * strings.
+   * For instance, if pairsDelim is "," and keyValuesDelim is ":", then the
+   * string should be represented as
+   * "key1:value11:value12:value13,key2:value21,key3:value31:value32".
+   * @param str delimited string represented as multiple keys having multiple
+   * values.
+   * @param pairsDelim key-values pairs are delimited by this delimiter.
+   * @param keyValuesDelim values for a key are delimited by this delimiter.
+   * @return a map of key-values with each key having a set of values.
+   */
+  static Map

[31/50] [abbrv] hadoop git commit: MAPREDUCE-6546. reconcile the two versions of the timeline service performance tests. (Sangjin Lee via Naganarasimha G R)

2016-05-04 Thread gtcarrera9 
MAPREDUCE-6546. reconcile the two versions of the timeline service performance 
tests. (Sangjin Lee via Naganarasimha G R)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd444089
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd444089
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd444089

Branch: refs/heads/YARN-2928
Commit: cd4440899a2cebbbfb15e18228d3821961319986
Parents: 34248bd
Author: naganarasimha 
Authored: Wed Mar 9 11:20:32 2016 +0530
Committer: Li Lu 
Committed: Wed May 4 16:35:05 2016 -0700

--
 .../hadoop/mapred/JobHistoryFileParser.java |  53 
 .../mapred/JobHistoryFileReplayMapper.java  | 301 ---
 .../hadoop/mapred/SimpleEntityWriter.java   | 140 -
 .../hadoop/mapred/TimelineEntityConverter.java  | 211 -
 .../mapred/TimelineServicePerformanceV2.java| 229 --
 .../apache/hadoop/mapreduce/EntityWriterV2.java |  56 
 .../mapreduce/JobHistoryFileReplayMapperV1.java |  14 +-
 .../mapreduce/JobHistoryFileReplayMapperV2.java | 161 ++
 .../mapreduce/SimpleEntityWriterConstants.java  |  43 +++
 .../hadoop/mapreduce/SimpleEntityWriterV1.java  |  28 +-
 .../hadoop/mapreduce/SimpleEntityWriterV2.java  | 131 
 .../mapreduce/TimelineEntityConverterV1.java|   5 -
 .../mapreduce/TimelineEntityConverterV2.java| 211 +
 .../mapreduce/TimelineServicePerformance.java   | 129 +---
 .../apache/hadoop/test/MapredTestDriver.java|  35 +--
 15 files changed, 704 insertions(+), 1043 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
deleted file mode 100644
index 9d051df..000
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
-
-class JobHistoryFileParser {
-  private static final Log LOG = LogFactory.getLog(JobHistoryFileParser.class);
-
-  private final FileSystem fs;
-
-  public JobHistoryFileParser(FileSystem fs) {
-LOG.info("JobHistoryFileParser created with " + fs);
-this.fs = fs;
-  }
-
-  public JobInfo parseHistoryFile(Path path) throws IOException {
-LOG.info("parsing job history file " + path);
-JobHistoryParser parser = new JobHistoryParser(fs, path);
-return parser.parse();
-  }
-
-  public Configuration parseConfiguration(Path path) throws IOException {
-LOG.info("parsing job configuration file " + path);
-Configuration conf = new Configuration(false);
-conf.addResource(fs.open(path));
-return conf;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
--
diff --git

[11/50] [abbrv] hadoop git commit: YARN-4200. Refactor reader classes in storage to nest under hbase specific package name. Contributed by Li Lu.

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
new file mode 100644
index 000..181ec81
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
@@ -0,0 +1,383 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.FamilyFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnFamily;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for application entities that are stored in the
+ * application table.
+ */
+class ApplicationEntityReader extends GenericEntityReader {
+  private static final ApplicationTable APPLICATION_TABLE =
+  new ApplicationTable();
+
+  public ApplicationEntityReader(String userId, String clusterId,
+  String flowName, Long flowRunId, String appId, String entityType,
+  Long limit, Long createdTimeBegin, Long createdTimeEnd,
+  Long modifiedTimeBegin, Long modifiedTimeEnd,
+  Map relatesTo, Map isRelatedTo,
+  Map infoFilters, Map configFilters,
+  Set metricFilters, Set eventFilters,
+  TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
+  EnumSet fieldsToRetrieve) {
+super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
+createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
+

[44/50] [abbrv] hadoop git commit: YARN-3150. Documenting the timeline service v2. (Sangjin Lee and Vrushali C via gtcarrera9)

2016-05-04 Thread gtcarrera9 
YARN-3150. Documenting the timeline service v2. (Sangjin Lee and Vrushali C via 
gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/caf23c34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/caf23c34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/caf23c34

Branch: refs/heads/YARN-2928
Commit: caf23c3453b7476df24edaaa024108cabcd56b21
Parents: 084a334
Author: Li Lu <gtcarre...@apache.org>
Authored: Sat Apr 30 15:02:12 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Wed May 4 16:35:40 2016 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   8 +-
 .../src/main/resources/yarn-default.xml |   6 +-
 .../metrics/TimelineServiceV2Publisher.java |  16 +-
 .../TestSystemMetricsPublisherForV2.java|  16 +-
 .../src/site/markdown/TimelineServer.md |   2 +-
 .../src/site/markdown/TimelineServiceV2.md  | 576 +++
 .../src/site/resources/images/timeline_v2.jpg   | Bin 0 -> 45112 bytes
 7 files changed, 600 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/caf23c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b79b3a5..3837a2b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -477,13 +477,13 @@ public class YarnConfiguration extends Configuration {
   public static final boolean DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED = false;
 
   /**
-   * The setting that controls whether yarn container metrics is published to
-   * the timeline server or not by RM. This configuration setting is for ATS
+   * The setting that controls whether yarn container events are published to
+   * the timeline service or not by RM. This configuration setting is for ATS
* V2
*/
-  public static final String RM_PUBLISH_CONTAINER_METRICS_ENABLED = YARN_PREFIX
+  public static final String RM_PUBLISH_CONTAINER_EVENTS_ENABLED = YARN_PREFIX
   + "rm.system-metrics-publisher.emit-container-events";
-  public static final boolean DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED =
+  public static final boolean DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED =
   false;
 
   public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/caf23c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 13d4b4f..4899dcb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -799,9 +799,9 @@
   
 
   
-The setting that controls whether yarn container metrics is
-published to the timeline server or not by RM. This configuration setting 
is
-for ATS V2.
+The setting that controls whether yarn container events are
+published to the timeline service or not by RM. This configuration setting
+is for ATS V2.
 yarn.rm.system-metrics-publisher.emit-container-events
 false
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/caf23c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 14073d1..b7ece5f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publishe

[25/50] [abbrv] hadoop git commit: YARN-4409. Fix javadoc and checkstyle issues in timelineservice code (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
index 50136de..663a18a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
@@ -42,12 +42,13 @@ public interface TimelineWriter extends Service {
* @param userId context user ID
* @param flowName context flow name
* @param flowVersion context flow version
-   * @param flowRunId
-   * @param appId context app ID
+   * @param flowRunId run id for the flow.
+   * @param appId context app ID.
* @param data
*  a {@link TimelineEntities} object.
* @return a {@link TimelineWriteResponse} object.
-   * @throws IOException
+   * @throws IOException if there is any exception encountered while storing
+   * or writing entities to the backend storage.
*/
   TimelineWriteResponse write(String clusterId, String userId,
   String flowName, String flowVersion, long flowRunId, String appId,
@@ -65,8 +66,11 @@ public interface TimelineWriter extends Service {
*  a {@link TimelineEntity} object
*  a {@link TimelineAggregationTrack} enum
*  value.
+   * @param track Specifies the track or dimension along which aggregation 
would
+   * occur. Includes USER, FLOW, QUEUE, etc.
* @return a {@link TimelineWriteResponse} object.
-   * @throws IOException
+   * @throws IOException if there is any exception encountered while 
aggregating
+   * entities to the backend storage.
*/
   TimelineWriteResponse aggregate(TimelineEntity data,
   TimelineAggregationTrack track) throws IOException;
@@ -76,7 +80,8 @@ public interface TimelineWriter extends Service {
* written to the storage when the method returns. This may be a potentially
* time-consuming operation, and should be used judiciously.
*
-   * @throws IOException
+   * @throws IOException if there is any exception encountered while flushing
+   * entities to the backend storage.
*/
   void flush() throws IOException;
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
index c03c9b6..5734389 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
@@ -34,7 +34,7 @@ import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
 public enum ApplicationColumn implements Column {
 
   /**
-   * App id
+   * App id.
*/
   ID(ApplicationColumnFamily.INFO, "id"),
 
@@ -84,7 +84,7 @@ public enum ApplicationColumn implements 
Column {
   /**
* Retrieve an {@link ApplicationColumn} given a name, or null if there is no
* match. The following holds true: {@code columnFor(x) == columnFor(y)} if
-   * and only if {@code x.equals(y)} or {@code (x == y == null)}
+   * and only if {@code x.equals(y)} or {@code (x == y == null)}.
*
* @param columnQualifier Name of the column to retrieve
* @return the corresponding {@link ApplicationColumn} or null

[36/50] [abbrv] hadoop git commit: YARN-3461. Consolidate flow name/version/run defaults. (Sangjin Lee via Varun Saxena)

2016-05-04 Thread gtcarrera9 
YARN-3461. Consolidate flow name/version/run defaults. (Sangjin Lee via Varun 
Saxena)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/408f0014
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/408f0014
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/408f0014

Branch: refs/heads/YARN-2928
Commit: 408f00144b8e6808eed91c5549e60e63c50d052d
Parents: 899468d
Author: Varun Saxena 
Authored: Thu Apr 7 22:10:11 2016 +0530
Committer: Li Lu 
Committed: Wed May 4 16:35:28 2016 -0700

--
 .../mapred/TestMRTimelineEventHandling.java | 46 +++---
 .../distributedshell/TestDistributedShell.java  | 18 --
 .../yarn/util/timeline/TimelineUtils.java   |  8 ++-
 .../resourcemanager/amlauncher/AMLauncher.java  | 67 +++-
 .../RMTimelineCollectorManager.java | 36 +--
 .../TestSystemMetricsPublisherForV2.java| 20 +++---
 .../collector/AppLevelTimelineCollector.java| 11 +---
 .../collector/NodeTimelineCollectorManager.java | 12 
 .../collector/TimelineCollectorContext.java |  5 +-
 9 files changed, 148 insertions(+), 75 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f0014/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index f7283ae..300b4fb 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -20,15 +20,12 @@ package org.apache.hadoop.mapred;
 
 import java.io.File;
 import java.io.IOException;
-
 import java.util.EnumSet;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -38,9 +35,9 @@ import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
@@ -48,7 +45,6 @@ import org.apache.hadoop.yarn.server.timeline.TimelineStore;
 import 
org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -205,7 +201,7 @@ public class TestMRTimelineEventHandling {
   ApplicationReport appReport = apps.get(0);
   firstAppId = appReport.getApplicationId();
 
-  checkNewTimelineEvent(firstAppId);
+  checkNewTimelineEvent(firstAppId, appReport);
 
   LOG.info("Run 2nd job which should be failed.");
   job = UtilsForTests.runJobFail(new JobConf(conf), inDir, outDir);
@@ -214,11 +210,10 @@ public class TestMRTimelineEventHandling {
   
   apps = yarnClient.getApplications(appStates);
   Assert.assertEquals(apps.size(), 2);
-  
-  ApplicationId secAppId = null;
-  secAppId = apps.get(0).getApplicationId() == firstAppId ? 
-  apps.get(1).getApplicationId() : apps.get(0).getApplicationId();
-  checkNewTimelineEvent(firstAppId);
+
+  appReport = apps.get(0).getApplicationId().equals(firstAppId) ?
+  apps.get(0) : apps.get(1);
+  checkNewTimelineEvent(firstAppId, appReport);
 
 } finally {
   if (cluster != null) {
@@ -235,7 +230,8 @@ public class TestMRTimelineEventHandling {
 }
   }
   
-  private void checkNewTimelineEvent(ApplicationId

[48/50] [abbrv] hadoop git commit: YARN-4447. Provide a mechanism to represent complex filters and parse them at the REST layer (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
index 57d75db..2e667d6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
@@ -18,29 +18,19 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
-import java.io.IOException;
 import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.Set;
 
 import javax.servlet.http.HttpServletRequest;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
-import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
-import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
-import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
 
 /**
  * Set of utility methods to be used by timeline reader web services.
  */
 final class TimelineReaderWebServicesUtils {
-  private static final String COMMA_DELIMITER = ",";
-  private static final String COLON_DELIMITER = ":";
 
   private TimelineReaderWebServicesUtils() {
   }
@@ -56,11 +46,10 @@ final class TimelineReaderWebServicesUtils {
* @param entityType Entity Type.
* @param entityId Entity Id.
* @return a {@link TimelineReaderContext} object.
-   * @throws Exception if any problem occurs during parsing.
*/
   static TimelineReaderContext createTimelineReaderContext(String clusterId,
   String userId, String flowName, String flowRunId, String appId,
-  String entityType, String entityId) throws Exception {
+  String entityType, String entityId) {
 return new TimelineReaderContext(parseStr(clusterId), parseStr(userId),
 parseStr(flowName), parseLongStr(flowRunId), parseStr(appId),
 parseStr(entityType), parseStr(entityId));
@@ -79,20 +68,17 @@ final class TimelineReaderWebServicesUtils {
* @param metricfilters Entities to return must match these metric filters.
* @param eventfilters Entities to return must match these event filters.
* @return a {@link TimelineEntityFilters} object.
-   * @throws Exception if any problem occurs during parsing.
+   * @throws TimelineParseException if any problem occurs during parsing.
*/
   static TimelineEntityFilters createTimelineEntityFilters(String limit,
   String createdTimeStart, String createdTimeEnd, String relatesTo,
   String isRelatedTo, String infofilters, String conffilters,
-  String metricfilters, String eventfilters) throws Exception {
+  String metricfilters, String eventfilters) throws TimelineParseException 
{
 return new TimelineEntityFilters(parseLongStr(limit),
 parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd),
-parseKeyStrValuesStr(relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
-parseKeyStrValuesStr(isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
-parseKeyStrValueObj(infofilters, COMMA_DELIMITER, COLON_DELIMITER),
-parseKeyStrValueStr(conffilters, COMMA_DELIMITER, COLON_DELIMITER),
-parseMetricFilters(metricfilters, COMMA_DELIMITER),
-parseValuesStr(eventfilters, COMMA_DELIMITER));
+parseRelationFilters(relatesTo), parseRelationFilters(isRelatedTo),
+parseKVFilters(infofilters, false), parseKVFilters(conffilters, true),
+parseMetricFilters(metricfilters), parseEventFilters(eventfilters));
   }
 
   /**
@@ -102,12 +88,13 @@ final class TimelineReaderWebServicesUtils {
* @param metrics metrics to retrieve.
* @param fields fields to retrieve.
* @return a {@link TimelineDataToRetrieve} object.
-   * @throws Exception

[35/50] [abbrv] hadoop git commit: YARN-4711. NM is going down with NPE's due to single thread processing of events by Timeline client (Naganarasimha G R via sjlee)

2016-05-04 Thread gtcarrera9 
YARN-4711. NM is going down with NPE's due to single thread processing of 
events by Timeline client (Naganarasimha G R via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/899468de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/899468de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/899468de

Branch: refs/heads/YARN-2928
Commit: 899468de23e065c0d5df3f49470b9358a5ac944b
Parents: b1e7f16
Author: Sangjin Lee 
Authored: Mon Mar 28 15:50:03 2016 -0700
Committer: Li Lu 
Committed: Wed May 4 16:35:23 2016 -0700

--
 .../dev-support/findbugs-exclude.xml|  11 +-
 .../records/timelineservice/TimelineEntity.java |  25 ++-
 .../client/api/impl/TimelineClientImpl.java |  35 ++--
 .../api/impl/TestTimelineClientV2Impl.java  |  91 +++-
 .../metrics/ContainerMetricsConstants.java  |   8 +
 .../nodemanager/NodeStatusUpdaterImpl.java  |  10 +-
 .../collectormanager/NMCollectorService.java|  10 +-
 .../application/Application.java|   4 -
 .../application/ApplicationImpl.java|  24 +--
 .../timelineservice/NMTimelinePublisher.java| 210 +++
 .../TestNMTimelinePublisher.java|  24 +--
 .../yarn/server/nodemanager/webapp/MockApp.java |   5 -
 12 files changed, 278 insertions(+), 179 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml 
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index ba1af88..72fbb35 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -117,8 +117,15 @@
 
   
   
-
- 
+
+
+
+  
+
+  
+
+
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index acc132e..7ce8279 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -17,15 +17,6 @@
  */
 package org.apache.hadoop.yarn.api.records.timelineservice;
 
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.util.TimelineServiceHelper;
-import org.codehaus.jackson.annotate.JsonSetter;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -33,6 +24,16 @@ import java.util.NavigableSet;
 import java.util.Set;
 import java.util.TreeSet;
 
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.util.TimelineServiceHelper;
+import org.codehaus.jackson.annotate.JsonSetter;
+
 /**
  * The basic timeline entity data structure for timeline service v2. Timeline
  * entity objects are not thread safe and should not be accessed concurrently.
@@ -564,6 +565,10 @@ public class TimelineEntity implements 
Comparable {
   }
 
   public String toString() {
-return identifier.toString();
+if (real == null) {
+  return identifier.toString();
+} else {
+  return real.toString();
+}
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git

[38/50] [abbrv] hadoop git commit: YARN-3863. Support complex filters in TimelineReader (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
index 9793ce6..3b8036d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
@@ -154,6 +154,14 @@ class TestFlowDataGenerator {
 metrics.add(m2);
 
 entity.addMetrics(metrics);
+TimelineEvent event = new TimelineEvent();
+event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+long endTs = 1439379885000L;
+event.setTimestamp(endTs);
+String expKey = "foo_event_greater";
+String expVal = "test_app_greater";
+event.addInfo(expKey, expVal);
+entity.addEvent(event);
 return entity;
   }
 
@@ -178,6 +186,14 @@ class TestFlowDataGenerator {
 m1.setValues(metricValues);
 metrics.add(m1);
 entity.addMetrics(metrics);
+TimelineEvent event = new TimelineEvent();
+event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+long endTs = 1439379885000L;
+event.setTimestamp(endTs);
+String expKey = "foo_event_greater";
+String expVal = "test_app_greater";
+event.addInfo(expKey, expVal);
+entity.addEvent(event);
 return entity;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index f04dd48..a724db2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -47,8 +47,10 @@ import 
org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
@@ -307,7 +309,7 @@ public class TestHBaseStorageFlowRun {
   assertEquals(141L, Bytes.toLong(values.get(q)));
 
   // check metric2
-  assertEquals(2, values.size());
+  assertEquals(3, values.size());
   q = ColumnHelper.getColumnQualifier(
   FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), metric2);
   assertTrue(values.containsKey(q));
@@ -318,11 +320,10 @@ public class TestHBaseStorageFlowRun {
 
   @Test
   public void testWriteFlowRunMetricsPrefix() throws Exception {
-String cluster = "testWriteFlowRunMetricsOneFlow_cluster1";
-String user = "testWriteFlowRunMetricsOneFlow_user1";
-String flow = "testing_flowRun_metrics_flow_name";
+String cluster = "testWriteFlowRunMetricsPrefix_cluster1";
+String user =

[21/50] [abbrv] hadoop git commit: YARN-4446. Refactor reader API for better extensibility (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
index b7804e7..a8a2ff8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
@@ -37,6 +37,9 @@ import 
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.junit.AfterClass;
@@ -258,9 +261,10 @@ public class TestFileSystemTimelineReaderImpl {
   public void testGetEntityDefaultView() throws Exception {
 // If no fields are specified, entity is returned with default view i.e.
 // only the id, type and created time.
-TimelineEntity result =
-reader.getEntity("user1", "cluster1", "flow1", 1L, "app1",
-"app", "id_1", null, null, null);
+TimelineEntity result = reader.getEntity(
+new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+"app", "id_1"),
+new TimelineDataToRetrieve(null, null, null));
 Assert.assertEquals(
 (new TimelineEntity.Identifier("app", "id_1")).toString(),
 result.getIdentifier().toString());
@@ -272,9 +276,10 @@ public class TestFileSystemTimelineReaderImpl {
   @Test
   public void testGetEntityByClusterAndApp() throws Exception {
 // Cluster and AppId should be enough to get an entity.
-TimelineEntity result =
-reader.getEntity(null, "cluster1", null, null, "app1",
-"app", "id_1", null, null, null);
+TimelineEntity result = reader.getEntity(
+new TimelineReaderContext("cluster1", null, null, null, "app1", "app",
+"id_1"),
+new TimelineDataToRetrieve(null, null, null));
 Assert.assertEquals(
 (new TimelineEntity.Identifier("app", "id_1")).toString(),
 result.getIdentifier().toString());
@@ -288,9 +293,10 @@ public class TestFileSystemTimelineReaderImpl {
   public void testAppFlowMappingCsv() throws Exception {
 // Test getting an entity by cluster and app where flow entry
 // in app flow mapping csv has commas.
-TimelineEntity result =
-reader.getEntity(null, "cluster1", null, null, "app2",
-"app", "id_5", null, null, null);
+TimelineEntity result = reader.getEntity(
+new TimelineReaderContext("cluster1", null, null, null, "app2",
+"app", "id_5"),
+new TimelineDataToRetrieve(null, null, null));
 Assert.assertEquals(
 (new TimelineEntity.Identifier("app", "id_5")).toString(),
 result.getIdentifier().toString());
@@ -300,10 +306,11 @@ public class TestFileSystemTimelineReaderImpl {
   @Test
   public void testGetEntityCustomFields() throws Exception {
 // Specified fields in addition to default view will be returned.
-TimelineEntity result =
-reader.getEntity("user1", "cluster1", "flow1", 1L,
-"app1", "app", "id_1", null, null,
-EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS));
+TimelineEntity result = reader.getEntity(
+new TimelineReaderContext("cluster1","user1", "flow1", 1L, "app1",
+"app", "id_1"),
+new TimelineDataToRetrieve(null, null,
+EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS)));
 Assert.assertEquals(
 (new TimelineEntity.Identifier("app", "id_1")).toString(),
 result.getIdentifier().toString());
@@ -318,9 +325,10 @@ public class TestFileSystemTimelineReaderImpl {
   @Test
   public void testGetEntityAllFields() throws Exception {
 // All fields of

[23/50] [abbrv] hadoop git commit: YARN-4446. Refactor reader API for better extensibility (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
YARN-4446. Refactor reader API for better extensibility (Varun Saxena via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/675612d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/675612d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/675612d4

Branch: refs/heads/YARN-2928
Commit: 675612d4f4564195b272a86b6bf6826634ceea19
Parents: e02094f
Author: Sangjin Lee 
Authored: Wed Feb 3 16:03:55 2016 -0800
Committer: Li Lu 
Committed: Wed May 4 16:24:20 2016 -0700

--
 .../reader/TimelineDataToRetrieve.java  | 119 ++
 .../reader/TimelineEntityFilters.java   | 187 +
 .../reader/TimelineReaderContext.java   |  10 +
 .../reader/TimelineReaderManager.java   |  44 +-
 .../reader/TimelineReaderUtils.java |   4 +-
 .../reader/TimelineReaderWebServices.java   | 417 ---
 .../reader/TimelineReaderWebServicesUtils.java  |  68 +++
 .../storage/FileSystemTimelineReaderImpl.java   | 116 +++---
 .../storage/HBaseTimelineReaderImpl.java|  36 +-
 .../timelineservice/storage/TimelineReader.java | 234 +--
 .../storage/reader/ApplicationEntityReader.java | 204 -
 .../reader/FlowActivityEntityReader.java|  59 +--
 .../storage/reader/FlowRunEntityReader.java | 101 ++---
 .../storage/reader/GenericEntityReader.java | 192 -
 .../storage/reader/TimelineEntityReader.java| 101 ++---
 .../reader/TimelineEntityReaderFactory.java |  74 ++--
 .../TestFileSystemTimelineReaderImpl.java   | 156 ---
 .../storage/TestHBaseTimelineStorage.java   | 252 ++-
 .../flow/TestHBaseStorageFlowActivity.java  |  33 +-
 .../storage/flow/TestHBaseStorageFlowRun.java   |  48 ++-
 20 files changed, 1367 insertions(+), 1088 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java
new file mode 100644
index 000..0cc83d7
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.util.EnumSet;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+
+/**
+ * Encapsulates information regarding which data to retrieve for each entity
+ * while querying.
+ * Data to retrieve contains the following :
+ * 
+ * confsToRetrieve - Used for deciding which configs to return
+ * in response. This is represented as a {@link TimelineFilterList} object
+ * containing {@link TimelinePrefixFilter} objects. These can either be
+ * exact config keys' or prefixes which are then compared against config
+ * keys' to decide configs(inside entities) to return in response. If null
+ * or empty, all configurations will be fetched if fieldsToRetrieve
+ * contains {@link Field#CONFIGS} or {@link Field#ALL}. This should not be
+ * confused

[41/50] [abbrv] hadoop git commit: YARN-3863. Support complex filters in TimelineReader (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
index b5fc214..2d85bab 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
@@ -17,21 +17,26 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.storage.common;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.SortedSet;
-import java.io.IOException;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -39,6 +44,15 @@ import 
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
 import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter.TimelineFilterType;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
 import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
@@ -53,6 +67,8 @@ public final class TimelineStorageUtils {
   private TimelineStorageUtils() {
   }
 
+  private static final Log LOG = LogFactory.getLog(TimelineStorageUtils.class);
+
   /** empty bytes. */
   public static final byte[] EMPTY_BYTES = new byte[0];
 
@@ -312,6 +328,21 @@ public final class TimelineStorageUtils {
   }
 
   /**
+   * Check if we have a certain field amongst fields to retrieve. This method
+   * checks against {@link Field#ALL} as well because that would mean field
+   * passed needs to be matched.
+   *
+   * @param fieldsToRetrieve fields to be retrieved.
+   * @param requiredField fields to be checked in fieldsToRetrieve.
+   * @return true if has the required field, false otherwise.
+   */
+  public static boolean hasField(EnumSet fieldsToRetrieve,
+  Field requiredField) {
+return fieldsToRetrieve.contains(Field.ALL) ||
+fieldsToRetrieve.contains(requiredField);
+  }
+
+  /**
* Checks if the input TimelineEntity object is an ApplicationEntity.
*
* @param te TimelineEntity object.
@@ -385,87 +416,317 @@ public final class TimelineStorageUtils {
   }
 
   /**
+   * Matches key-values filter. Used for relatesTo/isRelatedTo filters.
*
-   * @param entityRelations the relations of an entity
-   * @param relationFilters the relations for filtering
-   * @return a boolean flag to indicate if both match
+   * @param entity entity which holds relatesTo/isRelatedTo relations

[22/50] [abbrv] hadoop git commit: YARN-4446. Refactor reader API for better extensibility (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
index 0eeb195..ccb33b7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
@@ -20,17 +20,14 @@ package 
org.apache.hadoop.yarn.server.timelineservice.storage;
 
 import java.io.IOException;
 
-import java.util.EnumSet;
-import java.util.Map;
 import java.util.Set;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
 
 /** ATSv2 reader interface. */
 @Private
@@ -38,11 +35,6 @@ import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefi
 public interface TimelineReader extends Service {
 
   /**
-   * Default limit for {@link #getEntities}.
-   */
-  long DEFAULT_LIMIT = 100;
-
-  /**
* Possible fields to retrieve for {@link #getEntities} and
* {@link #getEntity}.
*/
@@ -57,55 +49,61 @@ public interface TimelineReader extends Service {
   }
 
   /**
-   * The API to fetch the single entity given the entity identifier in the
-   * scope of the given context.
-   *
-   * @param userId
-   *Context user Id(optional).
-   * @param clusterId
-   *Context cluster Id(mandatory).
-   * @param flowName
-   *Context flow Id (optional).
-   * @param flowRunId
-   *Context flow run Id (optional).
-   * @param appId
-   *Context app Id (mandatory)
-   * @param entityType
-   *Entity type (mandatory)
-   * @param entityId
-   *Entity Id (mandatory)
-   * @param confsToRetrieve
-   *Used for deciding which configs to return in response. This is
-   *represented as a {@link TimelineFilterList} object containing
-   *{@link TimelinePrefixFilter} objects. These can either be exact config
-   *keys' or prefixes which are then compared against config keys' to 
decide
-   *configs to return in response.
-   * @param metricsToRetrieve
-   *Used for deciding which metrics to return in response. This is
-   *represented as a {@link TimelineFilterList} object containing
-   *{@link TimelinePrefixFilter} objects. These can either be exact metric
-   *ids' or prefixes which are then compared against metric ids' to decide
-   *metrics to return in response.
-   * @param fieldsToRetrieve
-   *Specifies which fields of the entity object to retrieve(optional), see
-   *{@link Field}. If null, retrieves 4 fields namely entity id,
-   *entity type and entity created time. All fields will be returned if
-   *{@link Field#ALL} is specified.
-   * @return a {@link TimelineEntity} instance or null. The entity will
-   *contain the metadata plus the given fields to retrieve.
+   * The API to fetch the single entity given the identifier(depending on
+   * the entity type) in the scope of the given context.
+   * @param context Context which defines the scope in which query has to be
+   *made. Use getters of {@link TimelineReaderContext} to fetch context
+   *fields. Context contains the following :
+   *
+   *entityType - Entity type(mandatory).
+   *clusterId - Identifies the cluster(mandatory).
+   *userId - Identifies the user.
+   *flowName - Context flow name.
+   *flowRunId - Context flow run id.
+   *appId - Context app id.
+   *entityId - Entity id.
+   *
+   *Fields in context which are mandatory depends on entity type.

[33/50] [abbrv] hadoop git commit: YARN-4062. Add the flush and compaction functionality via coprocessors and scanners for flow run table (Vrushali C via sjlee)

2016-05-04 Thread gtcarrera9 
YARN-4062. Add the flush and compaction functionality via coprocessors and 
scanners for flow run table (Vrushali C via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3157d32a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3157d32a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3157d32a

Branch: refs/heads/YARN-2928
Commit: 3157d32ae8ed526430beb85a4920243d38bd2b41
Parents: cd44408
Author: Sangjin Lee 
Authored: Thu Mar 17 18:22:04 2016 -0700
Committer: Li Lu 
Committed: Wed May 4 16:35:14 2016 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  16 +
 .../src/main/resources/yarn-default.xml |  10 +
 .../storage/HBaseTimelineWriterImpl.java|   5 +-
 .../storage/common/TimelineStorageUtils.java|  55 ++
 .../storage/common/TimestampGenerator.java  |  13 +-
 .../storage/flow/AggregationOperation.java  |  17 +-
 .../storage/flow/FlowRunColumn.java |   4 +-
 .../storage/flow/FlowRunColumnPrefix.java   |   2 +-
 .../storage/flow/FlowRunCoprocessor.java|  70 +-
 .../storage/flow/FlowRunRowKey.java |  16 +
 .../storage/flow/FlowScanner.java   | 269 ++--
 .../storage/flow/FlowScannerOperation.java  |  46 ++
 .../storage/flow/TestFlowDataGenerator.java | 178 +-
 .../storage/flow/TestHBaseStorageFlowRun.java   | 112 +++-
 .../flow/TestHBaseStorageFlowRunCompaction.java | 635 +++
 15 files changed, 1362 insertions(+), 86 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 5115758..b79b3a5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1900,6 +1900,22 @@ public class YarnConfiguration extends Configuration {
   public static final int
   DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS = 60;
 
+  /**
+   * The name for setting that controls how long the final value of
+   * a metric of a completed app is retained before merging
+   * into the flow sum.
+   */
+  public static final String APP_FINAL_VALUE_RETENTION_THRESHOLD =
+  TIMELINE_SERVICE_PREFIX
+  + "coprocessor.app-final-value-retention-milliseconds";
+
+  /**
+   * The setting that controls how long the final value of a metric
+   * of a completed app is retained before merging into the flow sum.
+   */
+  public static final long DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD = 3 * 24
+  * 60 * 60 * 1000L;
+
   public static final String ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS =
   TIMELINE_SERVICE_PREFIX + "app-collector.linger-period.ms";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index ca913b8..13d4b4f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2128,6 +2128,7 @@
 604800
   
 
+  
   
 
yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size
 
@@ -2196,6 +2197,15 @@
 
yarn.timeline-service.timeline-client.number-of-async-entities-to-merge
 10
   
+
+  
+ The setting that controls how long the final value
+of a metric of a completed app is retained before merging into
+the flow sum.
+
yarn.timeline-service.coprocessor.app-final-value-retention-milliseconds
+25920
+  
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
--
diff --git

[49/50] [abbrv] hadoop git commit: YARN-4447. Provide a mechanism to represent complex filters and parse them at the REST layer (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
YARN-4447. Provide a mechanism to represent complex filters and parse them at 
the REST layer (Varun Saxena via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/047cde55
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/047cde55
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/047cde55

Branch: refs/heads/YARN-2928
Commit: 047cde55aa957a1288b88fa503ef1204b56c649b
Parents: 28ec2be
Author: Sangjin Lee 
Authored: Mon May 2 14:06:19 2016 -0700
Committer: Li Lu 
Committed: Wed May 4 16:38:09 2016 -0700

--
 .../reader/TimelineParseConstants.java  |  34 +
 .../reader/TimelineParseException.java  |  36 +
 .../timelineservice/reader/TimelineParser.java  |  37 +
 .../reader/TimelineParserForCompareExpr.java| 300 ++
 .../reader/TimelineParserForDataToRetrieve.java |  95 ++
 .../reader/TimelineParserForEqualityExpr.java   | 343 +++
 .../reader/TimelineParserForExistFilters.java   |  51 +
 .../reader/TimelineParserForKVFilters.java  |  78 ++
 .../reader/TimelineParserForNumericFilters.java |  72 ++
 .../TimelineParserForRelationFilters.java   |  71 ++
 .../reader/TimelineReaderWebServices.java   | 220 -
 .../reader/TimelineReaderWebServicesUtils.java  | 196 ++--
 .../reader/filter/TimelineCompareFilter.java|  73 +-
 .../reader/filter/TimelineExistsFilter.java |  49 +-
 .../reader/filter/TimelineFilterList.java   |  36 +
 .../reader/filter/TimelineKeyValueFilter.java   |  13 +
 .../reader/filter/TimelineKeyValuesFilter.java  |  61 +-
 .../reader/filter/TimelinePrefixFilter.java |  37 +
 .../reader/TestTimelineReaderWebServices.java   |  14 +-
 ...stTimelineReaderWebServicesHBaseStorage.java | 900 +-
 .../TestTimelineReaderWebServicesUtils.java | 923 +++
 21 files changed, 3442 insertions(+), 197 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java
new file mode 100644
index 000..662a102
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+/**
+ * Set of constants used while parsing filter expressions.
+ */
+final class TimelineParseConstants {
+  private TimelineParseConstants() {
+  }
+  static final String COMMA_DELIMITER = ",";
+  static final String COLON_DELIMITER = ":";
+  static final char NOT_CHAR = '!';
+  static final char SPACE_CHAR = ' ';
+  static final char OPENING_BRACKET_CHAR = '(';
+  static final char CLOSING_BRACKET_CHAR = ')';
+  static final char COMMA_CHAR = ',';
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseException.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseException.java

[32/50] [abbrv] hadoop git commit: YARN-4062. Add the flush and compaction functionality via coprocessors and scanners for flow run table (Vrushali C via sjlee)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
new file mode 100644
index 000..ace218b
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
@@ -0,0 +1,635 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotEquals;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGenerator;
+import 
org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+
+/**
+ * Tests the FlowRun and FlowActivity Tables
+ */
+public class TestHBaseStorageFlowRunCompaction {
+
+  private static HBaseTestingUtility util;
+
+  private final String metric1 = "MAP_SLOT_MILLIS";
+  private final String metric2 = "HDFS_BYTES_READ";
+
+  private final byte[] aRowKey = Bytes.toBytes("a");
+  private final byte[] aFamily = Bytes.toBytes("family");
+  private final byte[] aQualifier = Bytes.toBytes("qualifier");
+
+  @BeforeClass
+  public static void setupBeforeClass() throws Exception {
+util = new HBaseTestingUtility();
+Configuration conf = util.getConfiguration();
+conf.setInt("hfile.format.version", 3);
+util.startMiniCluster();
+createSchema();
+  }
+
+  private static void createSchema() throws IOException {
+TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
+  }
+
+  @Test
+  public void testWriteFlowRunCompaction() throws Exception {
+String cluster = "kompaction_cluster1";
+String user = "kompaction_FlowRun__user1";
+String flow = "kompaction_flowRun_flow_name";
+String flowVersion =

[46/50] [abbrv] hadoop git commit: MAPREDUCE-6424. Store MR counters as timeline metrics instead of event. (Naganarasimha G R via varunsaxena)

2016-05-04 Thread gtcarrera9 
MAPREDUCE-6424. Store MR counters as timeline metrics instead of event. 
(Naganarasimha G R via varunsaxena)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28ec2bee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28ec2bee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28ec2bee

Branch: refs/heads/YARN-2928
Commit: 28ec2beeed35c5f30dfde0a62e36b7b3eebcf3b3
Parents: caf23c3
Author: Varun Saxena 
Authored: Sun May 1 17:17:24 2016 +0530
Committer: Li Lu 
Committed: Wed May 4 16:38:05 2016 -0700

--
 .../jobhistory/JobHistoryEventHandler.java  | 29 +++-
 .../hadoop/mapreduce/jobhistory/TestEvents.java | 12 +++-
 .../mapreduce/jobhistory/AMStartedEvent.java| 12 +++-
 .../mapreduce/jobhistory/HistoryEvent.java  |  6 ++
 .../mapreduce/jobhistory/JobFinishedEvent.java  | 20 --
 .../jobhistory/JobInfoChangeEvent.java  | 10 ++-
 .../mapreduce/jobhistory/JobInitedEvent.java| 11 ++-
 .../jobhistory/JobPriorityChangeEvent.java  | 11 ++-
 .../jobhistory/JobQueueChangeEvent.java |  8 +++
 .../jobhistory/JobStatusChangedEvent.java   | 11 ++-
 .../mapreduce/jobhistory/JobSubmittedEvent.java | 10 ++-
 .../JobUnsuccessfulCompletionEvent.java | 11 ++-
 .../jobhistory/MapAttemptFinishedEvent.java | 12 +++-
 .../jobhistory/NormalizedResourceEvent.java |  8 +++
 .../jobhistory/ReduceAttemptFinishedEvent.java  | 12 +++-
 .../jobhistory/TaskAttemptFinishedEvent.java| 11 ++-
 .../jobhistory/TaskAttemptStartedEvent.java | 13 +++-
 .../TaskAttemptUnsuccessfulCompletionEvent.java | 17 +++--
 .../mapreduce/jobhistory/TaskFailedEvent.java   | 14 ++--
 .../mapreduce/jobhistory/TaskFinishedEvent.java | 13 +++-
 .../mapreduce/jobhistory/TaskStartedEvent.java  |  8 +++
 .../mapreduce/jobhistory/TaskUpdatedEvent.java  | 11 ++-
 .../mapreduce/util/JobHistoryEventUtils.java| 21 +-
 .../mapred/TestMRTimelineEventHandling.java | 74 +++-
 .../org/apache/hadoop/mapred/UtilsForTests.java | 39 +--
 25 files changed, 345 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 639f56e..d32524e 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.BlockingQueue;
@@ -68,6 +69,8 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -1072,6 +1075,15 @@ public class JobHistoryEventHandler extends 
AbstractService
 return entity;
   }
   
+  // create ApplicationEntity with job finished Metrics from HistoryEvent
+  private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+  createAppEntityWithJobMetrics(HistoryEvent event, JobId jobId) {
+ApplicationEntity entity = new ApplicationEntity();
+entity.setId(jobId.getAppId().toString());
+entity.setMetrics(event.getTimelineMetrics());
+return entity;
+  }
+
   // create BaseEntity from HistoryEvent with adding other info, like: 
   // timestamp and entityType.
   private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity 
@@ -1088,6 +1100,10 @@ public class JobHistoryEventHandler extends 
AbstractService
 if (setCreatedTime) {
   entity.setCreatedTime(timestamp);

[40/50] [abbrv] hadoop git commit: YARN-3863. Support complex filters in TimelineReader (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
index d8f73d4..6696ac5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
@@ -19,13 +19,8 @@ package 
org.apache.hadoop.yarn.server.timelineservice.storage.reader;
 
 import java.io.IOException;
 import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Get;
@@ -33,28 +28,22 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
 import org.apache.hadoop.hbase.filter.FamilyFilter;
 import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.QualifierFilter;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import 
org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
@@ -71,7 +60,6 @@ import com.google.common.base.Preconditions;
  */
 class GenericEntityReader extends TimelineEntityReader {
   private static final EntityTable ENTITY_TABLE = new EntityTable();
-  private static final Log LOG = LogFactory.getLog(GenericEntityReader.class);
 
   /**
* Used to look up the flow context.
@@ -97,92 +85,322 @@ class GenericEntityReader extends TimelineEntityReader {
   }
 
   @Override
-  protected FilterList constructFilterListBasedOnFields() {
-FilterList list = new FilterList(Operator.MUST_PASS_ONE);
-TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
-// Fetch all the columns.
-if (dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
-(dataToRetrieve.getConfsToRetrieve() == null ||
-dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty()) &&
-(dataToRetrieve.getMetricsToRetrieve() == null ||
-dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty())) {
-  return list;
+  protected FilterList constructFilterListBasedOnFilters() throws IOException {
+// Filters here cannot be null for multiple entity reads

[42/50] [abbrv] hadoop git commit: YARN-3863. Support complex filters in TimelineReader (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
YARN-3863. Support complex filters in TimelineReader (Varun Saxena via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b2df86c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b2df86c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b2df86c

Branch: refs/heads/YARN-2928
Commit: 2b2df86c775f008c258fe128d3686f4a4d3b1a52
Parents: 408f001
Author: Sangjin Lee 
Authored: Mon Apr 11 21:07:32 2016 -0700
Committer: Li Lu 
Committed: Wed May 4 16:35:33 2016 -0700

--
 .../reader/TimelineEntityFilters.java   |  170 +-
 .../reader/TimelineReaderWebServicesUtils.java  |   88 +-
 .../reader/filter/TimelineCompareFilter.java|   35 +-
 .../reader/filter/TimelineExistsFilter.java |   62 +
 .../reader/filter/TimelineFilter.java   |   16 +-
 .../reader/filter/TimelineFilterList.java   |   14 +
 .../reader/filter/TimelineFilterUtils.java  |  206 +-
 .../reader/filter/TimelineKeyValueFilter.java   |   48 +
 .../reader/filter/TimelineKeyValuesFilter.java  |   71 +
 .../reader/filter/TimelinePrefixFilter.java |6 +
 .../storage/FileSystemTimelineReaderImpl.java   |   36 +-
 .../storage/HBaseTimelineWriterImpl.java|   31 +-
 .../storage/application/ApplicationColumn.java  |   28 +-
 .../application/ApplicationColumnPrefix.java|   37 +-
 .../storage/apptoflow/AppToFlowColumn.java  |   16 +
 .../timelineservice/storage/common/Column.java  |   17 +
 .../storage/common/ColumnHelper.java|   16 +
 .../storage/common/ColumnPrefix.java|   35 +
 .../common/TimelineEntityFiltersType.java   |   71 +
 .../storage/common/TimelineStorageUtils.java|  461 +++-
 .../storage/entity/EntityColumn.java|   28 +-
 .../storage/entity/EntityColumnPrefix.java  |   38 +-
 .../storage/flow/FlowActivityColumnPrefix.java  |   35 +
 .../storage/flow/FlowRunColumn.java |3 +
 .../storage/flow/FlowRunColumnPrefix.java   |   26 +
 .../storage/flow/FlowScanner.java   |1 +
 .../storage/reader/ApplicationEntityReader.java |  426 ++--
 .../reader/FlowActivityEntityReader.java|7 +
 .../storage/reader/FlowRunEntityReader.java |   97 +-
 .../storage/reader/GenericEntityReader.java |  623 ++---
 .../storage/reader/TimelineEntityReader.java|   71 +-
 .../reader/TestTimelineReaderWebServices.java   |   10 +-
 .../TestFileSystemTimelineReaderImpl.java   |  332 ++-
 .../storage/TestHBaseTimelineStorage.java   | 2172 +-
 .../storage/flow/TestFlowDataGenerator.java |   16 +
 .../storage/flow/TestHBaseStorageFlowRun.java   |  267 ++-
 .../flow/TestHBaseStorageFlowRunCompaction.java |2 +-
 .../reader/filter/TimelineExistsFilter.java |   62 +
 .../reader/filter/TimelineKeyValueFilter.java   |   48 +
 .../reader/filter/TimelineKeyValuesFilter.java  |   71 +
 .../common/TimelineEntityFiltersType.java   |   71 +
 41 files changed, 5054 insertions(+), 816 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
index 5b2c300..4821d31 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
@@ -18,11 +18,14 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
-import java.util.Map;
-import java.util.Set;
-
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
+import

[50/50] [abbrv] hadoop git commit: MAPREDUCE-6688. Store job configurations in Timeline Service v2 (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
MAPREDUCE-6688. Store job configurations in Timeline Service v2 (Varun Saxena 
via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2c4237f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2c4237f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2c4237f

Branch: refs/heads/YARN-2928
Commit: d2c4237fa7b15aaddfdd55f906f7953b29cf2fe8
Parents: 047cde5
Author: Sangjin Lee 
Authored: Tue May 3 09:19:36 2016 -0700
Committer: Li Lu 
Committed: Wed May 4 16:38:24 2016 -0700

--
 .../jobhistory/JobHistoryEventHandler.java  | 57 +++-
 .../mapreduce/v2/app/job/impl/JobImpl.java  |  2 +-
 .../mapreduce/jobhistory/JobSubmittedEvent.java | 38 +++-
 .../mapreduce/util/JobHistoryEventUtils.java|  3 +
 .../mapred/TestMRTimelineEventHandling.java | 92 +---
 .../org/apache/hadoop/mapred/UtilsForTests.java |  8 ++
 6 files changed, 181 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2c4237f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index d32524e..db58ecf 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -1074,7 +1074,16 @@ public class JobHistoryEventHandler extends 
AbstractService
 entity.setId(jobId.toString());
 return entity;
   }
-  
+
+  private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+  createJobEntity(JobId jobId) {
+org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+new 
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
+entity.setId(jobId.toString());
+entity.setType(MAPREDUCE_JOB_ENTITY_TYPE);
+return entity;
+  }
+
   // create ApplicationEntity with job finished Metrics from HistoryEvent
   private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
   createAppEntityWithJobMetrics(HistoryEvent event, JobId jobId) {
@@ -1133,6 +1142,46 @@ public class JobHistoryEventHandler extends 
AbstractService
 return entity;
   }
 
+  private void publishConfigsOnJobSubmittedEvent(JobSubmittedEvent event,
+  JobId jobId) {
+if (event.getJobConf() == null) {
+  return;
+}
+// Publish job configurations both as job and app entity.
+// Configs are split into multiple entities if they exceed 100kb in size.
+org.apache.hadoop.yarn.api.records.timelineservice.
+TimelineEntity jobEntityForConfigs = createJobEntity(jobId);
+ApplicationEntity appEntityForConfigs = new ApplicationEntity();
+String appId = jobId.getAppId().toString();
+appEntityForConfigs.setId(appId);
+try {
+  int configSize = 0;
+  for (Map.Entry entry : event.getJobConf()) {
+int size = entry.getKey().length() + entry.getValue().length();
+configSize += size;
+if (configSize > JobHistoryEventUtils.ATS_CONFIG_PUBLISH_SIZE_BYTES) {
+  if (jobEntityForConfigs.getConfigs().size() > 0) {
+timelineClient.putEntities(jobEntityForConfigs);
+timelineClient.putEntities(appEntityForConfigs);
+jobEntityForConfigs = createJobEntity(jobId);
+appEntityForConfigs = new ApplicationEntity();
+appEntityForConfigs.setId(appId);
+  }
+  configSize = size;
+}
+jobEntityForConfigs.addConfig(entry.getKey(), entry.getValue());
+appEntityForConfigs.addConfig(entry.getKey(), entry.getValue());
+  }
+  if (configSize > 0) {
+timelineClient.putEntities(jobEntityForConfigs);
+timelineClient.putEntities(appEntityForConfigs);
+  }
+} catch (IOException | YarnException e) {
+  LOG.error("Exception while publishing configs on JOB_SUBMITTED Event " +
+  " for the job : " + jobId, e);
+}
+  }
+
   private void processEventForNewTimelineService(HistoryEvent event,
   JobId jobId, long timestamp) {
 org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity

[39/50] [abbrv] hadoop git commit: YARN-3863. Support complex filters in TimelineReader (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
index 4e07ecf..6b57ec4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -52,10 +53,14 @@ import 
org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
 import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator;
+import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
@@ -75,9 +80,6 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
 /**
  * Various tests to test writing entities to HBase and reading them back from
  * it.
@@ -113,30 +115,29 @@ public class TestHBaseTimelineStorage {
 String id = "application_11_";
 entity.setId(id);
 entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
-Long cTime = 1425016501000L;
+Long cTime = 1425016502000L;
 entity.setCreatedTime(cTime);
 // add the info map in Timeline Entity
 Map infoMap = new HashMap();
-infoMap.put("infoMapKey1", "infoMapValue1");
-infoMap.put("infoMapKey2", 10);
+infoMap.put("infoMapKey1", "infoMapValue2");
+infoMap.put("infoMapKey2", 20);
+infoMap.put("infoMapKey3", 85.85);
 entity.addInfo(infoMap);
 // add the isRelatedToEntity info
-String key = "task";
-String value = "is_related_to_entity_id_here";
 Set isRelatedToSet = new HashSet();
-isRelatedToSet.add(value);
+isRelatedToSet.add("relatedto1");
 Map isRelatedTo = new HashMap();
-isRelatedTo.put(key, isRelatedToSet);
+isRelatedTo.put("task", isRelatedToSet);
 entity.setIsRelatedToEntities(isRelatedTo);
 // add the relatesTo info
-key = "container";
-value = "relates_to_entity_id_here";
 Set relatesToSet = new HashSet();
-relatesToSet.add(value);
-value = "relates_to_entity_id_here_Second";
-relatesToSet.add(value);
+relatesToSet.add("relatesto1");
+relatesToSet.add("relatesto3");
 Map relatesTo = new HashMap();
-relatesTo.put(key, relatesToSet);
+relatesTo.put("container", relatesToSet);
+Set relatesToSet11 = new HashSet();
+relatesToSet11.add("relatesto4");
+relatesTo.put("container1", relatesToSet11);
 entity.setRelatesToEntities(relatesTo);
 // add some config entries
 Map conf = new HashMap();
@@ -166,8 +167,8 @@ public class TestHBaseTimelineStorage {
 metrics.add(m12);
 entity.addMetrics(metrics);
 TimelineEvent event = new

[06/50] [abbrv] hadoop git commit: YARN-4445. Unify the term flowId and flowName in timeline v2 codebase. Contributed by Zhan Zhang.

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
index a14d2bc..2cd9625 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
@@ -27,14 +27,14 @@ import 
org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStor
 public class FlowRunRowKey {
   private final String clusterId;
   private final String userId;
-  private final String flowId;
+  private final String flowName;
   private final long flowRunId;
 
-  public FlowRunRowKey(String clusterId, String userId, String flowId,
+  public FlowRunRowKey(String clusterId, String userId, String flowName,
   long flowRunId) {
 this.clusterId = clusterId;
 this.userId = userId;
-this.flowId = flowId;
+this.flowName = flowName;
 this.flowRunId = flowRunId;
   }
 
@@ -46,8 +46,8 @@ public class FlowRunRowKey {
 return userId;
   }
 
-  public String getFlowId() {
-return flowId;
+  public String getFlowName() {
+return flowName;
   }
 
   public long getFlowRunId() {
@@ -56,33 +56,33 @@ public class FlowRunRowKey {
 
   /**
* Constructs a row key prefix for the flow run table as follows: {
-   * clusterId!userI!flowId!}
+   * clusterId!userI!flowName!}
*
* @param clusterId
* @param userId
-   * @param flowId
+   * @param flowName
* @return byte array with the row key prefix
*/
   public static byte[] getRowKeyPrefix(String clusterId, String userId,
-  String flowId) {
+  String flowName) {
 return Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId, userId,
-flowId, ""));
+flowName, ""));
   }
 
   /**
* Constructs a row key for the entity table as follows: {
-   * clusterId!userI!flowId!Inverted Flow Run Id}
+   * clusterId!userI!flowName!Inverted Flow Run Id}
*
* @param clusterId
* @param userId
-   * @param flowId
+   * @param flowName
* @param flowRunId
* @return byte array with the row key
*/
   public static byte[] getRowKey(String clusterId, String userId,
-  String flowId, Long flowRunId) {
+  String flowName, Long flowRunId) {
 byte[] first = Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId,
-userId, flowId));
+userId, flowName));
 // Note that flowRunId is a long, so we can't encode them all at the same
 // time.
 byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
@@ -104,10 +104,10 @@ public class FlowRunRowKey {
 Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[0]));
 String userId =
 Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[1]));
-String flowId =
+String flowName =
 Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[2]));
 long flowRunId =
 TimelineStorageUtils.invertLong(Bytes.toLong(rowKeyComponents[3]));
-return new FlowRunRowKey(clusterId, userId, flowId, flowRunId);
+return new FlowRunRowKey(clusterId, userId, flowName, flowRunId);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
index b1b93c1..2682fea 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
@@ -47,7

[26/50] [abbrv] hadoop git commit: YARN-4409. Fix javadoc and checkstyle issues in timelineservice code (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index fc05310..12daa95 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -388,15 +388,15 @@ public class RMActiveServiceContext {
   @Private
   @Unstable
   public void setRMTimelineCollectorManager(
-  RMTimelineCollectorManager timelineCollectorManager) {
-this.timelineCollectorManager = timelineCollectorManager;
+  RMTimelineCollectorManager collectorManager) {
+this.timelineCollectorManager = collectorManager;
   }
 
   @Private
   @Unstable
   public void setSystemMetricsPublisher(
-  SystemMetricsPublisher systemMetricsPublisher) {
-this.systemMetricsPublisher = systemMetricsPublisher;
+  SystemMetricsPublisher metricsPublisher) {
+this.systemMetricsPublisher = metricsPublisher;
   }
 
   @Private

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index e122ab4..4c72912 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -379,8 +379,8 @@ public class RMContextImpl implements RMContext {
   
   @Override
   public void setSystemMetricsPublisher(
-  SystemMetricsPublisher systemMetricsPublisher) {
-this.systemMetricsPublisher = systemMetricsPublisher;
+  SystemMetricsPublisher metricsPublisher) {
+this.systemMetricsPublisher = metricsPublisher;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
index a8c00a4..d4a4fc3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
@@ -30,6 +30,10 @@ import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
 
+/**
+ * Abstract implementation of SystemMetricsPublisher which is then extended by
+ * metrics publisher implementations depending on timeline service version.
+ */
 public abstract class AbstractSystemMetricsPublisher extends CompositeService
 implements SystemMetricsPublisher {
   private MultiThreadedDispatcher dispatcher;
@@ -46,13 +50,18 @@ public abstract class AbstractSystemMetricsPublisher 
extends CompositeService
   protected void serviceInit(Configuration conf) throws Exception {
 dispatcher =
 new

[45/50] [abbrv] hadoop git commit: YARN-4986. Add a check in the coprocessor for table to operated on (Vrushali C via sjlee)

2016-05-04 Thread gtcarrera9 
YARN-4986. Add a check in the coprocessor for table to operated on (Vrushali C 
via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/084a334a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/084a334a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/084a334a

Branch: refs/heads/YARN-2928
Commit: 084a334ae52179e2f14bdadab5b7c6d5b447be01
Parents: 64d15d9
Author: Sangjin Lee 
Authored: Fri Apr 29 17:13:32 2016 -0700
Committer: Li Lu 
Committed: Wed May 4 16:35:40 2016 -0700

--
 .../storage/common/TimelineStorageUtils.java| 20 +++
 .../storage/entity/EntityTable.java |  2 +-
 .../storage/flow/FlowRunCoprocessor.java| 39 +++--
 .../storage/flow/FlowScanner.java   | 13 +++--
 .../storage/flow/TestHBaseStorageFlowRun.java   | 61 
 .../flow/TestHBaseStorageFlowRunCompaction.java | 36 
 6 files changed, 160 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/084a334a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
index 2d85bab..18f975a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
@@ -32,8 +32,10 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.Tag;
 import org.apache.hadoop.hbase.client.Result;
@@ -56,6 +58,7 @@ import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Fiel
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
 import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 
 /**
@@ -887,4 +890,21 @@ public final class TimelineStorageUtils {
 Set eventsSet = new HashSet<>(eventsMap.values());
 entity.addEvents(eventsSet);
   }
+
+  public static boolean isFlowRunTable(HRegionInfo hRegionInfo,
+  Configuration conf) {
+String regionTableName = hRegionInfo.getTable().getNameAsString();
+String flowRunTableName = conf.get(FlowRunTable.TABLE_NAME_CONF_NAME,
+FlowRunTable.DEFAULT_TABLE_NAME);
+if (LOG.isDebugEnabled()) {
+  LOG.debug("regionTableName=" + regionTableName);
+}
+if (flowRunTableName.equalsIgnoreCase(regionTableName)) {
+  if (LOG.isDebugEnabled()) {
+LOG.debug(" table is the flow run table!! " + flowRunTableName);
+  }
+  return true;
+}
+return false;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/084a334a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index 3e3e3ab..b194f07 100644
---

[04/50] [abbrv] hadoop git commit: YARN-4392. ApplicationCreatedEvent event time resets after RM restart/failover. Contributed by Naganarasimha G R and Xuan Gong

2016-05-04 Thread gtcarrera9 
YARN-4392. ApplicationCreatedEvent event time resets after RM
restart/failover. Contributed by Naganarasimha G R and Xuan Gong

(cherry picked from commit 4546c7582b6762c18ba150d80a8976eb51a8290c)

Conflicts:

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d798de3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d798de3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d798de3

Branch: refs/heads/YARN-2928
Commit: 8d798de3f0e84f14dc9554b0b65f6906679b4128
Parents: 9ceeb2a
Author: Xuan 
Authored: Mon Dec 7 12:24:55 2015 -0800
Committer: Li Lu 
Committed: Wed May 4 16:17:07 2016 -0700

--
 .../apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java | 6 --
 .../hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java| 4 +++-
 .../server/resourcemanager/rmapp/TestRMAppTransitions.java | 2 --
 3 files changed, 7 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d798de3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 9979a59..b15278c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -335,7 +335,8 @@ public class TimelineClientImpl extends TimelineClient {
   @Override
   protected void serviceStart() throws Exception {
 timelineWriter = createTimelineWriter(
-configuration, authUgi, client, resURI);
+configuration, authUgi, client, constructResURI(getConfig(),
+getTimelineServiceAddress(), false));
   }
 
   protected TimelineWriter createTimelineWriter(Configuration conf,
@@ -613,7 +614,8 @@ public class TimelineClientImpl extends TimelineClient {
 
   @Override
   public String toString() {
-return super.toString() + " with timeline server " + resURI
+return super.toString() + " with timeline server "
++ constructResURI(getConfig(), getTimelineServiceAddress(), false)
 + " and writer " + timelineWriter;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d798de3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index ca2771a..a7df5a4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -897,6 +897,8 @@ public class RMAppImpl implements RMApp, Recoverable {
 //TODO recover collector address.
 //this.collectorAddr = appState.getCollectorAddr();
 
+// send the ATS create Event
+sendATSCreateEvent(this, this.startTime);
 RMAppAttemptImpl preAttempt = null;
 for (ApplicationAttemptId attemptId :
 new TreeSet<>(appState.attempts.keySet())) {
@@ -1863,7 +1865,7 @@ public class RMAppImpl implements RMApp, Recoverable {
 }
 return amNodeLabelExpression;
   }
-  
+
   @Override
   public CallerContext getCallerContext() {
 return callerContext;

[30/50] [abbrv] hadoop git commit: MAPREDUCE-6546. reconcile the two versions of the timeline service performance tests. (Sangjin Lee via Naganarasimha G R)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
index 92e0b14..4c42bd3 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
@@ -18,6 +18,19 @@
 
 package org.apache.hadoop.test;
 
+import org.apache.hadoop.fs.DFSCIOTest;
+import org.apache.hadoop.fs.DistributedFSCheck;
+import org.apache.hadoop.fs.JHLogAnalyzer;
+import org.apache.hadoop.fs.TestDFSIO;
+import org.apache.hadoop.fs.TestFileSystem;
+import org.apache.hadoop.fs.loadGenerator.DataGenerator;
+import org.apache.hadoop.fs.loadGenerator.LoadGenerator;
+import org.apache.hadoop.fs.loadGenerator.LoadGeneratorMR;
+import org.apache.hadoop.fs.loadGenerator.StructureGenerator;
+import org.apache.hadoop.fs.slive.SliveTest;
+import org.apache.hadoop.hdfs.NNBench;
+import org.apache.hadoop.hdfs.NNBenchWithoutMR;
+import org.apache.hadoop.io.FileBench;
 import org.apache.hadoop.io.TestSequenceFile;
 import org.apache.hadoop.mapred.BigMapOutput;
 import org.apache.hadoop.mapred.GenericMRLoadGenerator;
@@ -28,28 +41,13 @@ import org.apache.hadoop.mapred.TestMapRed;
 import org.apache.hadoop.mapred.TestSequenceFileInputFormat;
 import org.apache.hadoop.mapred.TestTextInputFormat;
 import org.apache.hadoop.mapred.ThreadedMapBenchmark;
-import org.apache.hadoop.mapreduce.TimelineServicePerformance;
-import org.apache.hadoop.mapred.TimelineServicePerformanceV2;
 import org.apache.hadoop.mapreduce.FailJob;
 import org.apache.hadoop.mapreduce.LargeSorter;
 import org.apache.hadoop.mapreduce.MiniHadoopClusterManager;
 import org.apache.hadoop.mapreduce.SleepJob;
+import org.apache.hadoop.mapreduce.TimelineServicePerformance;
 import org.apache.hadoop.util.ProgramDriver;
 
-import org.apache.hadoop.hdfs.NNBench;
-import org.apache.hadoop.hdfs.NNBenchWithoutMR;
-import org.apache.hadoop.fs.TestFileSystem;
-import org.apache.hadoop.fs.TestDFSIO;
-import org.apache.hadoop.fs.DFSCIOTest;
-import org.apache.hadoop.fs.DistributedFSCheck;
-import org.apache.hadoop.io.FileBench;
-import org.apache.hadoop.fs.JHLogAnalyzer;
-import org.apache.hadoop.fs.loadGenerator.DataGenerator;
-import org.apache.hadoop.fs.loadGenerator.LoadGenerator;
-import org.apache.hadoop.fs.loadGenerator.LoadGeneratorMR;
-import org.apache.hadoop.fs.loadGenerator.StructureGenerator;
-import org.apache.hadoop.fs.slive.SliveTest;
-
 /**
  * Driver for Map-reduce tests.
  *
@@ -93,9 +91,8 @@ public class MapredTestDriver {
   pgd.addClass("sleep", SleepJob.class,
"A job that sleeps at each map and reduce task.");
   pgd.addClass("timelineperformance", TimelineServicePerformance.class,
-   "A job that launches mappers to test timlineserver 
performance.");
-  pgd.addClass("timelineperformance", TimelineServicePerformanceV2.class,
-  "A job that launch mappers to test timline service v.2 
performance.");
+   "A job that launches mappers to test timline service " +
+   "performance.");
   pgd.addClass("nnbench", NNBench.class, 
   "A benchmark that stresses the namenode w/ MR.");
   pgd.addClass("nnbenchWithoutMR", NNBenchWithoutMR.class,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[28/50] [abbrv] hadoop git commit: YARN-3367. Replace starting a separate thread for post entity with event loop in TimelineClient (Naganarasimha G R via sjlee)

2016-05-04 Thread gtcarrera9 
YARN-3367. Replace starting a separate thread for post entity with event loop 
in TimelineClient (Naganarasimha G R via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f59c7433
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f59c7433
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f59c7433

Branch: refs/heads/YARN-2928
Commit: f59c74338313305752678ea140fa3195d9fee07e
Parents: ee803e1
Author: Sangjin Lee 
Authored: Tue Feb 9 09:07:37 2016 -0800
Committer: Li Lu 
Committed: Wed May 4 16:33:32 2016 -0700

--
 .../jobhistory/JobHistoryEventHandler.java  |  81 +
 .../mapred/JobHistoryFileReplayMapper.java  |   8 +-
 .../hadoop/mapred/TimelineEntityConverter.java  |  12 +-
 .../timelineservice/TimelineEntities.java   |  17 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |   6 +
 .../distributedshell/ApplicationMaster.java |  78 +
 .../api/async/impl/AMRMClientAsyncImpl.java |  26 +-
 .../hadoop/yarn/client/api/TimelineClient.java  |   8 +-
 .../client/api/impl/TimelineClientImpl.java | 285 ++---
 .../src/main/resources/yarn-default.xml |   7 +
 .../api/impl/TestTimelineClientV2Impl.java  | 304 +++
 .../nodemanager/NodeStatusUpdaterImpl.java  |   4 +-
 12 files changed, 619 insertions(+), 217 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 853c506..639f56e 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -28,10 +28,7 @@ import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -85,7 +82,6 @@ import org.codehaus.jackson.node.ObjectNode;
 import com.google.common.annotations.VisibleForTesting;
 import com.sun.jersey.api.client.ClientHandlerException;
 
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
 /**
  * The job history events get routed to this class. This class writes the Job
  * history events to the DFS directly into a staging dir and then moved to a
@@ -141,10 +137,6 @@ public class JobHistoryEventHandler extends AbstractService
   
   private boolean timelineServiceV2Enabled = false;
 
-  // For posting entities in new timeline service in a non-blocking way
-  // TODO YARN-3367 replace with event loop in TimelineClient.
-  private ExecutorService threadPool;
-
   private static String MAPREDUCE_JOB_ENTITY_TYPE = "MAPREDUCE_JOB";
   private static String MAPREDUCE_TASK_ENTITY_TYPE = "MAPREDUCE_TASK";
   private static final String MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE =
@@ -284,10 +276,6 @@ public class JobHistoryEventHandler extends AbstractService
 YarnConfiguration.timelineServiceV2Enabled(conf);
 LOG.info("Timeline service is enabled; version: " +
 YarnConfiguration.getTimelineServiceVersion(conf));
-if (timelineServiceV2Enabled) {
-  // initialize the thread pool for v.2 timeline service
-  threadPool = createThreadPool();
-}
   } else {
 LOG.info("Timeline service is not enabled");
   }
@@ -461,35 +449,9 @@ public class JobHistoryEventHandler extends AbstractService
 if (timelineClient != null) {
   timelineClient.stop();
 }
-if (threadPool != null) {
-  shutdownAndAwaitTermination();
-}
 LOG.info("Stopped JobHistoryEventHandler. super.stop()");
 super.serviceStop();
   }
-  
-  // TODO remove threadPool after adding non-blocking call in TimelineClient
-  private ExecutorService createThreadPool() {
-return Executors.newCachedThreadPool(
-  new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
-  .build());
-

[18/50] [abbrv] hadoop git commit: YARN-4224. Support fetching entities by UID and change the REST interface to conform to current REST APIs' in YARN. (Varun Saxena via gtcarrera9)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index a054ee5..7a70de8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -18,17 +18,12 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
-import java.io.IOException;
 import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Collections;
 import java.util.Date;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Set;
 import java.util.TimeZone;
 
@@ -51,10 +46,11 @@ import 
org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
@@ -63,7 +59,7 @@ import org.apache.hadoop.yarn.webapp.NotFoundException;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.inject.Singleton;
 
-/** REST end point for Timeline Reader */
+/** REST end point for Timeline Reader. */
 @Private
 @Unstable
 @Singleton
@@ -167,117 +163,6 @@ public class TimelineReaderWebServices {
 }
   }
 
-  private static Set parseValuesStr(String str, String delimiter) {
-if (str == null || str.isEmpty()) {
-  return null;
-}
-Set strSet = new HashSet();
-String[] strs = str.split(delimiter);
-for (String aStr : strs) {
-  strSet.add(aStr.trim());
-}
-return strSet;
-  }
-
-  @SuppressWarnings("unchecked")
-  private static  void parseKeyValues(Map map, String str,
-  String pairsDelim, String keyValuesDelim, boolean stringValue,
-  boolean multipleValues) {
-String[] pairs = str.split(pairsDelim);
-for (String pair : pairs) {
-  if (pair == null || pair.trim().isEmpty()) {
-continue;
-  }
-  String[] pairStrs = pair.split(keyValuesDelim);
-  if (pairStrs.length < 2) {
-continue;
-  }
-  if (!stringValue) {
-try {
-  Object value =
-  GenericObjectMapper.OBJECT_READER.readValue(pairStrs[1].trim());
-  map.put(pairStrs[0].trim(), (T) value);
-} catch (IOException e) {
-  map.put(pairStrs[0].trim(), (T) pairStrs[1].trim());
-}
-  } else {
-String key = pairStrs[0].trim();
-if (multipleValues) {
-  Set values = new HashSet();
-  for (int i = 1; i < pairStrs.length; i++) {
-values.add(pairStrs[i].trim());
-  }
-  map.put(key, (T) values);
-} else {
-  map.put(key, (T) pairStrs[1].trim());
-}
-  }
-}
-  }
-
-  private static Map parseKeyStrValuesStr(String str,
-  String pairsDelim, String keyValuesDelim) {
-if (str == null) {
-  return null;
-}
-Map map = new HashMap();
-parseKeyValues(map, str,pairsDelim, keyValuesDelim, true, true);
-return map;
-  }
-
-  private static Map parseKeyStrValueStr(String str,
-  String pairsDelim, String keyValDelim) {
-if (str == null) {
-  return null;
-}
-Map map = new HashMap();
-parseKeyValues(map, str, pairsDelim, keyValDelim, true, false);
-return map;
-  }
-
-  private static Map

[10/50] [abbrv] hadoop git commit: YARN-3995. Some of the NM events are not getting published due race condition when AM container finishes in NM (Naganarasimha G R via sjlee)

2016-05-04 Thread gtcarrera9 
YARN-3995. Some of the NM events are not getting published due race condition 
when AM container finishes in NM (Naganarasimha G R via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed5f7db9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed5f7db9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed5f7db9

Branch: refs/heads/YARN-2928
Commit: ed5f7db9cfe66f9eb5a70d1b1551c239436c0f23
Parents: 4a6388f
Author: Sangjin Lee 
Authored: Mon Jan 11 10:09:34 2016 -0800
Committer: Li Lu 
Committed: Wed May 4 16:22:08 2016 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  5 
 .../src/main/resources/yarn-default.xml |  7 ++
 .../PerNodeTimelineCollectorsAuxService.java| 25 +---
 ...TestPerNodeTimelineCollectorsAuxService.java | 11 +
 4 files changed, 35 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed5f7db9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ed81eaa..8ee417b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1899,6 +1899,11 @@ public class YarnConfiguration extends Configuration {
   public static final int
   DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS = 60;
 
+  public static final String ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS =
+  TIMELINE_SERVICE_PREFIX + "app-collector.linger-period.ms";
+
+  public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 1000;
+
   // mark app-history related configs @Private as application history is going
   // to be integrated into the timeline service
   @Private

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed5f7db9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e4b562e..1867e98 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2182,6 +2182,13 @@
 60
   
 
+  
+Time period till which the application collector will be alive
+ in NM, after the  application master container finishes.
+yarn.timeline-service.app-collector.linger-period.ms
+1000
+  
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed5f7db9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index 0319e34..b738530 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -19,6 +19,9 @@
 package org.apache.hadoop.yarn.server.timelineservice.collector;
 
 import java.nio.ByteBuffer;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -54,6 +57,8 @@ public class PerNodeTimelineCollectorsAuxService extends 
AuxiliaryService {
   private static final int SHUTDOWN_HOOK_PRIORITY = 30;
 
   private final

[24/50] [abbrv] hadoop git commit: YARN-4409. Fix javadoc and checkstyle issues in timelineservice code (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
index d991b42..f9eb5b4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
@@ -24,7 +24,8 @@ import 
org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
 /**
  * Represents the flow run table column families.
  */
-public enum FlowActivityColumnFamily implements 
ColumnFamily {
+public enum FlowActivityColumnFamily
+implements ColumnFamily {
 
   /**
* Info column family houses known columns, specifically ones included in

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
index 21ddcc2..a5933da 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
@@ -31,12 +31,13 @@ import 
org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStor
 import 
org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
 
 /**
- * Identifies partially qualified columns for the {@link FlowActivityTable}
+ * Identifies partially qualified columns for the {@link FlowActivityTable}.
  */
-public enum FlowActivityColumnPrefix implements 
ColumnPrefix {
+public enum FlowActivityColumnPrefix
+implements ColumnPrefix {
 
   /**
-   * To store run ids of the flows
+   * To store run ids of the flows.
*/
   RUN_ID(FlowActivityColumnFamily.INFO, "r", null);
 
@@ -162,8 +163,8 @@ public enum FlowActivityColumnPrefix implements 
ColumnPrefix
* org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
* #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result)
*/
-  public  NavigableMap> 
readResultsWithTimestamps(
-  Result result) throws IOException {
+  public  NavigableMap>
+  readResultsWithTimestamps(Result result) throws IOException {
 return column.readResultsWithTimestamps(result, columnPrefixBytes);
   }
 
@@ -179,8 +180,8 @@ public enum FlowActivityColumnPrefix implements 
ColumnPrefix
   public static final FlowActivityColumnPrefix columnFor(String columnPrefix) {
 
 // Match column based on value, assume column family matches.
-for (FlowActivityColumnPrefix flowActivityColPrefix : 
FlowActivityColumnPrefix
-.values()) {
+for (FlowActivityColumnPrefix flowActivityColPrefix :
+FlowActivityColumnPrefix.values()) {
   // Find a match based only on name.
   if (flowActivityColPrefix.getColumnPrefix().equals(columnPrefix)) {
 return flowActivityColPrefix;
@@ -209,8 +210,8 @@ public enum FlowActivityColumnPrefix implements 
ColumnPrefix
 // TODO: needs unit test to confirm and need to update javadoc to explain
 // null prefix case.
 
-for (FlowActivityColumnPrefix flowActivityColumnPrefix : 
FlowActivityColumnPrefix
-.values()) {
+for (FlowActivityColumnPrefix flowActivityColumnPrefix :
+FlowActivityColumnPrefix.values()) {
   // Find a match based column family and on name.
   if

[13/50] [abbrv] hadoop git commit: Rebase to latest trunk

2016-05-04 Thread gtcarrera9 
Rebase to latest trunk


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d1fc87e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d1fc87e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d1fc87e

Branch: refs/heads/YARN-2928
Commit: 2d1fc87e1e6894eefb48f8456fdc58ce32f60a45
Parents: 6173732
Author: Li Lu 
Authored: Wed Jan 20 01:05:52 2016 -0800
Committer: Li Lu 
Committed: Wed May 4 16:24:08 2016 -0700

--
 hadoop-tools/hadoop-aws/pom.xml   |  8 
 .../yarn/client/api/impl/TimelineClientImpl.java  |  4 ++--
 .../metrics/TimelineServiceV1Publisher.java   | 17 +++--
 .../metrics/TimelineServiceV2Publisher.java   | 18 +++---
 .../TestResourceTrackerService.java   | 12 
 .../hadoop-yarn-server-timelineservice/pom.xml|  6 --
 6 files changed, 48 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d1fc87e/hadoop-tools/hadoop-aws/pom.xml
--
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index f422846..db98f94 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -155,6 +155,14 @@
   hadoop-yarn-server-tests
   test
   test-jar
+  
+
+
+  joda-time
+  joda-time
+
+  
 
 
   org.apache.hadoop

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d1fc87e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index b15278c..f36167d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -335,8 +335,8 @@ public class TimelineClientImpl extends TimelineClient {
   @Override
   protected void serviceStart() throws Exception {
 timelineWriter = createTimelineWriter(
-configuration, authUgi, client, constructResURI(getConfig(),
-getTimelineServiceAddress(), false));
+configuration, authUgi, client,
+constructResURI(getConfig(), timelineServiceAddress, false));
   }
 
   protected TimelineWriter createTimelineWriter(Configuration conf,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d1fc87e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index d858a6b..ddc8a16 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -87,6 +87,17 @@ public class TimelineServiceV1Publisher extends 
AbstractSystemMetricsPublisher {
 app.getAmNodeLabelExpression());
 entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
 app.getAppNodeLabelExpression());
+if (app.getCallerContext() != null) {
+  if (app.getCallerContext().getContext() != null) {
+entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
+app.getCallerContext().getContext());
+  }
+  if (app.getCallerContext().getSignature() != null) {
+entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_SIGNATURE,
+app.getCallerContext().getSignature());
+  }
+}
+
 entity.setOtherInfo(entityInfo);
 TimelineEvent tEvent = new TimelineEvent();

[27/50] [abbrv] hadoop git commit: YARN-4409. Fix javadoc and checkstyle issues in timelineservice code (Varun Saxena via sjlee)

2016-05-04 Thread gtcarrera9 
YARN-4409. Fix javadoc and checkstyle issues in timelineservice code (Varun 
Saxena via sjlee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee803e16
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee803e16
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee803e16

Branch: refs/heads/YARN-2928
Commit: ee803e16e13bd01520e111dd448123c76d778225
Parents: 675612d
Author: Sangjin Lee 
Authored: Mon Feb 8 12:17:43 2016 -0800
Committer: Li Lu 
Committed: Wed May 4 16:28:05 2016 -0700

--
 .../jobhistory/JobHistoryEventHandler.java  | 170 ++-
 .../hadoop/mapreduce/jobhistory/TestEvents.java |   2 +-
 .../mapreduce/util/JobHistoryEventUtils.java|   7 +-
 .../hadoop/mapred/TimelineEntityConverter.java  |   6 +-
 .../hadoop/mapreduce/JobHistoryFileParser.java  |   3 +
 .../ApplicationAttemptEntity.java   |   9 +-
 .../timelineservice/ApplicationEntity.java  |   9 +-
 .../records/timelineservice/ClusterEntity.java  |   6 +-
 .../timelineservice/ContainerEntity.java|   9 +-
 .../records/timelineservice/FlowRunEntity.java  |   9 +-
 .../HierarchicalTimelineEntity.java |   8 +-
 .../records/timelineservice/QueueEntity.java|   6 +-
 .../timelineservice/TimelineEntities.java   |  11 +-
 .../records/timelineservice/TimelineEntity.java | 106 ++--
 .../timelineservice/TimelineEntityType.java |  71 +---
 .../records/timelineservice/TimelineEvent.java  |  30 ++--
 .../records/timelineservice/TimelineMetric.java |  39 +++--
 .../timelineservice/TimelineWriteResponse.java  |  59 +++
 .../api/records/timelineservice/UserEntity.java |   6 +-
 .../records/timelineservice/package-info.java   |   8 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |   9 +-
 .../hadoop/yarn/client/api/TimelineClient.java  |  13 +-
 .../client/api/impl/TimelineClientImpl.java |  16 +-
 .../yarn/util/timeline/TimelineUtils.java   |  21 ++-
 .../yarn/server/nodemanager/NodeManager.java|   2 +-
 .../collectormanager/NMCollectorService.java|   9 +-
 .../collectormanager/package-info.java  |  28 +++
 .../timelineservice/NMTimelineEvent.java|   4 +
 .../timelineservice/NMTimelineEventType.java|   3 +
 .../timelineservice/NMTimelinePublisher.java|  14 +-
 .../timelineservice/package-info.java   |  29 
 .../resourcemanager/RMActiveServiceContext.java |   8 +-
 .../server/resourcemanager/RMContextImpl.java   |   4 +-
 .../metrics/AbstractSystemMetricsPublisher.java |  20 ++-
 .../metrics/NoOpSystemMetricPublisher.java  |   2 +-
 .../metrics/SystemMetricsPublisher.java |   3 +
 .../metrics/TimelineServiceV1Publisher.java |   8 +-
 .../metrics/TimelineServiceV2Publisher.java |   7 +-
 .../resourcemanager/metrics/package-info.java   |  28 +++
 .../rmapp/RMAppCollectorUpdateEvent.java|   3 +
 .../server/resourcemanager/rmapp/RMAppImpl.java |   5 +-
 .../RMTimelineCollectorManager.java |  33 ++--
 .../timelineservice/package-info.java   |  28 +++
 .../collector/AppLevelTimelineCollector.java|   3 +-
 .../collector/NodeTimelineCollectorManager.java |   8 +-
 .../PerNodeTimelineCollectorsAuxService.java|   2 +
 .../collector/TimelineCollector.java|   2 +
 .../collector/TimelineCollectorManager.java |  12 +-
 .../collector/TimelineCollectorWebService.java  |  70 +---
 .../timelineservice/collector/package-info.java |  29 
 .../reader/TimelineReaderManager.java   |  32 +++-
 .../reader/TimelineReaderServer.java|   2 +-
 .../reader/TimelineReaderWebServices.java   |  28 +--
 .../reader/TimelineReaderWebServicesUtils.java  |  50 +++---
 .../reader/TimelineUIDConverter.java|  10 +-
 .../reader/filter/TimelineFilterUtils.java  |   8 +-
 .../timelineservice/reader/package-info.java|   6 +
 .../storage/FileSystemTimelineReaderImpl.java   |  70 
 .../storage/FileSystemTimelineWriterImpl.java   |   7 +-
 .../storage/HBaseTimelineReaderImpl.java|   3 +
 .../storage/HBaseTimelineWriterImpl.java|  29 ++--
 .../storage/OfflineAggregationWriter.java   |  13 +-
 .../PhoenixOfflineAggregationWriterImpl.java|  27 +--
 .../storage/TimelineAggregationTrack.java   |   2 +-
 .../timelineservice/storage/TimelineReader.java |   6 +-
 .../storage/TimelineSchemaCreator.java  |   4 +-
 .../timelineservice/storage/TimelineWriter.java |  15 +-
 .../storage/application/ApplicationColumn.java  |   4 +-
 .../application/ApplicationColumnPrefix.java|   8 +-
 .../storage/application/ApplicationRowKey.java  |  33 ++--
 .../storage/application/ApplicationTable.java   |  16 +-
 .../storage/application/package-info.java   |   4 +
 .../storage/apptoflow/AppToFlowColumn.java  |   6 +-

[12/50] [abbrv] hadoop git commit: YARN-4200. Refactor reader classes in storage to nest under hbase specific package name. Contributed by Li Lu.

2016-05-04 Thread gtcarrera9 
YARN-4200. Refactor reader classes in storage to nest under hbase
specific package name. Contributed by Li Lu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61737325
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61737325
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61737325

Branch: refs/heads/YARN-2928
Commit: 61737325ccaaea48e20d2f288c0c9d1b43a8bccb
Parents: ed5f7db
Author: Li Lu 
Authored: Mon Jan 11 18:05:36 2016 -0800
Committer: Li Lu 
Committed: Wed May 4 16:22:12 2016 -0700

--
 .../storage/ApplicationEntityReader.java| 382 --
 .../storage/FlowActivityEntityReader.java   | 163 --
 .../storage/FlowRunEntityReader.java| 225 -
 .../storage/GenericEntityReader.java| 496 --
 .../storage/HBaseTimelineReaderImpl.java|   2 +
 .../storage/TimelineEntityReader.java   | 274 --
 .../storage/TimelineEntityReaderFactory.java| 100 
 .../storage/reader/ApplicationEntityReader.java | 383 ++
 .../reader/FlowActivityEntityReader.java| 164 ++
 .../storage/reader/FlowRunEntityReader.java | 226 +
 .../storage/reader/GenericEntityReader.java | 497 +++
 .../storage/reader/TimelineEntityReader.java| 274 ++
 .../reader/TimelineEntityReaderFactory.java | 100 
 .../storage/reader/package-info.java|  23 +
 14 files changed, 1669 insertions(+), 1640 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
deleted file mode 100644
index d812a6c..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
+++ /dev/null
@@ -1,382 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.FamilyFilter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.filter.PageFilter;
-import org.apache.hadoop.hbase.filter.QualifierFilter;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import 
org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
-import 
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import 
org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
-import

[14/50] [abbrv] hadoop git commit: YARN-4622. TestDistributedShell fails for v2 test cases after modifications for 1.5. (Naganarasimha G R via Varun Saxena)

2016-05-04 Thread gtcarrera9 
YARN-4622. TestDistributedShell fails for v2 test cases after modifications for 
1.5. (Naganarasimha G R via Varun Saxena)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2269e78a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2269e78a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2269e78a

Branch: refs/heads/YARN-2928
Commit: 2269e78adbd3969e84b83a541d2cd84498c46f47
Parents: 2d1fc87
Author: Varun Saxena 
Authored: Fri Jan 22 01:41:25 2016 +0530
Committer: Li Lu 
Committed: Wed May 4 16:24:10 2016 -0700

--
 .../hadoop/yarn/client/api/impl/TimelineClientImpl.java  | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2269e78a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index f36167d..4db3b71 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import 
org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
 import org.codehaus.jackson.map.ObjectMapper;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
@@ -334,9 +335,10 @@ public class TimelineClientImpl extends TimelineClient {
 
   @Override
   protected void serviceStart() throws Exception {
-timelineWriter = createTimelineWriter(
-configuration, authUgi, client,
-constructResURI(getConfig(), timelineServiceAddress, false));
+if (!timelineServiceV2) {
+  timelineWriter = createTimelineWriter(configuration, authUgi, client,
+  constructResURI(getConfig(), timelineServiceAddress, false));
+}
   }
 
   protected TimelineWriter createTimelineWriter(Configuration conf,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[34/50] [abbrv] hadoop git commit: YARN-4712. CPU Usage Metric is not captured properly in YARN-2928. (Naganarasimha G R via Varun Saxena)

2016-05-04 Thread gtcarrera9 
YARN-4712. CPU Usage Metric is not captured properly in YARN-2928. 
(Naganarasimha G R via Varun Saxena)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1e7f16c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1e7f16c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1e7f16c

Branch: refs/heads/YARN-2928
Commit: b1e7f16cdcd3f6b3f650f44dd7187ee03cfe440d
Parents: 3157d32
Author: Varun Saxena 
Authored: Fri Mar 18 23:19:18 2016 +0530
Committer: Li Lu 
Committed: Wed May 4 16:35:18 2016 -0700

--
 .../monitor/ContainersMonitorImpl.java  |   9 +-
 .../timelineservice/NMTimelinePublisher.java|  21 ++-
 .../TestNMTimelinePublisher.java| 157 +++
 3 files changed, 171 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1e7f16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index da4aac2..e0bd35e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -22,7 +22,6 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -32,11 +31,10 @@ import 
org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
 import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -47,6 +45,7 @@ import 
org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
 import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
 public class ContainersMonitorImpl extends AbstractService implements
@@ -567,8 +566,8 @@ public class ContainersMonitorImpl extends AbstractService 
implements
 NMTimelinePublisher nmMetricsPublisher =
 container.getNMTimelinePublisher();
 if (nmMetricsPublisher != null) {
-  nmMetricsPublisher.reportContainerResourceUsage(container, pId,
-  currentPmemUsage, cpuUsageTotalCoresPercentage);
+  nmMetricsPublisher.reportContainerResourceUsage(container,
+  currentPmemUsage, cpuUsagePercentPerCore);
 }
   } catch (Exception e) {
 // Log the exception and proceed to the next container.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1e7f16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java

[03/50] [abbrv] hadoop git commit: YARN-4450. TestTimelineAuthenticationFilter and TestYarnConfigurationFields fail. Contributed by Sangjin Lee.

2016-05-04 Thread gtcarrera9 
YARN-4450. TestTimelineAuthenticationFilter and
TestYarnConfigurationFields fail. Contributed by Sangjin Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ceeb2a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ceeb2a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ceeb2a7

Branch: refs/heads/YARN-2928
Commit: 9ceeb2a7af267fb6b5c767c746faf50485d006fd
Parents: ef71c1f
Author: Li Lu 
Authored: Mon Dec 14 10:48:39 2015 -0800
Committer: Li Lu 
Committed: Wed May 4 16:01:41 2016 -0700

--
 .../yarn/conf/TestYarnConfigurationFields.java|  3 +++
 .../yarn/client/api/impl/TimelineClientImpl.java  | 18 ++
 2 files changed, 13 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ceeb2a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index c92a276..c8e1868 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -94,6 +94,9 @@ public class TestYarnConfigurationFields extends 
TestConfigurationFieldsBase {
 
 // Ignore all YARN Application Timeline Service (version 1) properties
 configurationPrefixToSkipCompare.add("yarn.timeline-service.");
+// skip deprecated RM_SYSTEM_METRICS_PUBLISHER_ENABLED
+configurationPropsToSkipCompare
+.add(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED);
 
 // Used as Java command line properties, not XML
 configurationPrefixToSkipCompare.add("yarn.app.container");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ceeb2a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index f92e6a7..9979a59 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -111,7 +111,6 @@ public class TimelineClientImpl extends TimelineClient {
   private ConnectionConfigurator connConfigurator;
   private DelegationTokenAuthenticator authenticator;
   private DelegationTokenAuthenticatedURL.Token token;
-  private URI resURI;
   private UserGroupInformation authUgi;
   private String doAsUser;
   private Configuration configuration;
@@ -552,8 +551,8 @@ public class TimelineClientImpl extends TimelineClient {
   @Override
   public Long run() throws Exception {
 // If the timeline DT to renew is different than cached, replace 
it.
-// Token to set every time for retry, because when exception 
happens,
-// DelegationTokenAuthenticatedURL will reset it to null;
+// Token to set every time for retry, because when exception
+// happens, DelegationTokenAuthenticatedURL will reset it to null;
 if (!timelineDT.equals(token.getDelegationToken())) {
   token.setDelegationToken((Token) timelineDT);
 }
@@ -562,7 +561,8 @@ public class TimelineClientImpl extends TimelineClient {
 connConfigurator);
 // If the token service address is not available, fall back to use
 // the configured service address.
-final URI serviceURI = isTokenServiceAddrEmpty ? resURI
+final URI serviceURI = isTokenServiceAddrEmpty ?
+constructResURI(getConfig(), getTimelineServiceAddress(), 
false)
 : new URI(scheme, null, address.getHostName(),
 address.getPort(), RESOURCE_URI_STR_V1, null, null);
 return authUrl
@@ -588,9 +588,10 @@ public class TimelineClientImpl extends TimelineClient {
 
   @Override
   public Void run() throws Exception {
-// If the timeline DT to

[16/50] [abbrv] hadoop git commit: YARN-4238. createdTime and modifiedTime is not reported while publishing entities to ATSv2. (Varun Saxena via Naganarasimha G R)

2016-05-04 Thread gtcarrera9 
YARN-4238. createdTime and modifiedTime is not reported while publishing 
entities to ATSv2. (Varun Saxena via Naganarasimha G R)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd369a54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd369a54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd369a54

Branch: refs/heads/YARN-2928
Commit: fd369a54e3cc08b812746f7c339826b0c4c7977f
Parents: 2269e78
Author: Naganarasimha 
Authored: Wed Jan 27 11:59:40 2016 +0530
Committer: Li Lu 
Committed: Wed May 4 16:24:12 2016 -0700

--
 .../jobhistory/JobHistoryEventHandler.java  | 47 ++
 .../records/timelineservice/TimelineEntity.java | 19 
 .../TestTimelineServiceRecords.java |  1 -
 .../monitor/ContainersMonitorImpl.java  |  6 +-
 .../timelineservice/NMTimelinePublisher.java|  6 +-
 .../TestSystemMetricsPublisherForV2.java| 34 +--
 .../reader/TimelineReaderManager.java   |  7 +-
 .../reader/TimelineReaderWebServices.java   | 51 ---
 .../storage/FileSystemTimelineReaderImpl.java   | 21 +
 .../storage/HBaseTimelineReaderImpl.java|  8 +-
 .../storage/HBaseTimelineWriterImpl.java|  4 -
 .../PhoenixOfflineAggregationWriterImpl.java|  9 +-
 .../timelineservice/storage/TimelineReader.java | 32 +++
 .../storage/application/ApplicationColumn.java  |  5 --
 .../storage/application/ApplicationTable.java   | 11 +--
 .../storage/entity/EntityColumn.java|  5 --
 .../storage/entity/EntityTable.java | 11 +--
 .../storage/reader/ApplicationEntityReader.java | 23 +
 .../reader/FlowActivityEntityReader.java|  7 +-
 .../storage/reader/FlowRunEntityReader.java |  7 +-
 .../storage/reader/GenericEntityReader.java | 22 +
 .../storage/reader/TimelineEntityReader.java|  5 --
 .../reader/TimelineEntityReaderFactory.java | 27 +++---
 .../reader/TestTimelineReaderWebServices.java   | 48 +-
 .../TestFileSystemTimelineReaderImpl.java   | 92 +--
 .../TestFileSystemTimelineWriterImpl.java   |  3 +-
 .../storage/TestHBaseTimelineStorage.java   | 95 +++-
 ...TestPhoenixOfflineAggregationWriterImpl.java |  1 -
 .../storage/flow/TestFlowDataGenerator.java |  1 -
 .../flow/TestHBaseStorageFlowActivity.java  |  6 +-
 .../storage/flow/TestHBaseStorageFlowRun.java   |  9 +-
 31 files changed, 197 insertions(+), 426 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index d88588c..a10872a 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -1135,10 +1135,10 @@ public class JobHistoryEventHandler extends 
AbstractService
   // jobId, timestamp and entityType.
   private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity 
   createJobEntity(HistoryEvent event, long timestamp, JobId jobId, 
-  String entityType) {
-
-org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = 
-createBaseEntity(event, timestamp, entityType);
+  String entityType, boolean setCreatedTime) {
+
+org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+createBaseEntity(event, timestamp, entityType, setCreatedTime);
 entity.setId(jobId.toString());
 return entity;
   }
@@ -1146,8 +1146,9 @@ public class JobHistoryEventHandler extends 
AbstractService
   // create BaseEntity from HistoryEvent with adding other info, like: 
   // timestamp and entityType.
   private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity 
-  createBaseEntity(HistoryEvent event, long timestamp, String entityType) {
-org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent = 
+  createBaseEntity(HistoryEvent event, long timestamp, String entityType,
+  boolean setCreatedTime) {
+

[09/50] [abbrv] hadoop git commit: YARN-3586. RM to only get back addresses of Collectors that NM needs to know. (Junping Du via Varun Saxena).

2016-05-04 Thread gtcarrera9 
YARN-3586. RM to only get back addresses of Collectors that NM needs to know.
(Junping Du via Varun Saxena).


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a6388ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a6388ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a6388ff

Branch: refs/heads/YARN-2928
Commit: 4a6388ff7637b3c149574068e8956a871a8cd3e0
Parents: 94b7fb4
Author: Varun Saxena 
Authored: Tue Dec 22 20:58:54 2015 +0530
Committer: Li Lu 
Committed: Wed May 4 16:22:06 2016 -0700

--
 .../resourcemanager/ResourceTrackerService.java | 30 +++
 .../TestResourceTrackerService.java | 82 
 2 files changed, 97 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6388ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 1dbbeb5..fea8183 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -26,7 +26,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
 import org.apache.commons.logging.Log;
@@ -525,16 +524,15 @@ public class ResourceTrackerService extends 
AbstractService implements
   nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials);
 }
 
-List keepAliveApps =
-remoteNodeStatus.getKeepAliveApplications();
-if (timelineV2Enabled && keepAliveApps != null) {
+if (timelineV2Enabled) {
   // Return collectors' map that NM needs to know
-  // TODO we should optimize this to only include collector info that NM
-  // doesn't know yet.
-  setAppCollectorsMapToResponse(keepAliveApps, nodeHeartBeatResponse);
+  setAppCollectorsMapToResponse(rmNode.getRunningApps(),
+  nodeHeartBeatResponse);
 }
 
 // 4. Send status to RMNode, saving the latest response.
+List keepAliveApps =
+remoteNodeStatus.getKeepAliveApplications();
 RMNodeStatusEvent nodeStatusEvent =
 new RMNodeStatusEvent(nodeId, remoteNodeStatus, nodeHeartBeatResponse);
 if (request.getLogAggregationReportsForApps() != null
@@ -562,18 +560,20 @@ public class ResourceTrackerService extends 
AbstractService implements
   }
 
   private void setAppCollectorsMapToResponse(
-  List liveApps, NodeHeartbeatResponse response) {
+  List runningApps, NodeHeartbeatResponse response) {
 Map liveAppCollectorsMap = new
-ConcurrentHashMap();
+HashMap();
 Map rmApps = rmContext.getRMApps();
-// Set collectors for all apps now.
-// TODO set collectors for only active apps running on NM (liveApps cannot 
be
-// used for this case)
-for (Map.Entry rmApp : rmApps.entrySet()) {
-  ApplicationId appId = rmApp.getKey();
-  String appCollectorAddr = rmApp.getValue().getCollectorAddr();
+// Set collectors for all running apps on this node.
+for (ApplicationId appId : runningApps) {
+  String appCollectorAddr = rmApps.get(appId).getCollectorAddr();
   if (appCollectorAddr != null) {
 liveAppCollectorsMap.put(appId, appCollectorAddr);
+  } else {
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Collector for applicaton: " + appId +
+  " hasn't registered yet!");
+}
   }
 }
 response.setAppCollectorsMap(liveAppCollectorsMap);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6388ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
--
diff --git

[08/50] [abbrv] hadoop git commit: YARN-4350. TestDistributedShell fails for V2 scenarios. (Naganarasimha G R via Varun Saxena)

2016-05-04 Thread gtcarrera9 
YARN-4350. TestDistributedShell fails for V2 scenarios. (Naganarasimha G R via 
Varun Saxena)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94b7fb47
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94b7fb47
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94b7fb47

Branch: refs/heads/YARN-2928
Commit: 94b7fb4707ed4e37ca6d9902aa358b5c8c83c66f
Parents: 89b4101
Author: Varun Saxena 
Authored: Sun Dec 20 02:14:54 2015 +0530
Committer: Li Lu 
Committed: Wed May 4 16:22:04 2016 -0700

--
 .../applications/distributedshell/TestDistributedShell.java | 2 ++
 .../java/org/apache/hadoop/yarn/server/MiniYARNCluster.java | 5 +++--
 .../java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java | 2 --
 3 files changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b7fb47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index e35a32e..203a7fc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -176,6 +176,8 @@ public class TestDistributedShell {
   conf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES,
   DistributedShellTimelinePlugin.class.getName());
 } else if (timelineVersion == 2.0f) {
+  // set version to 2
+  conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
   // disable v1 timeline server since we no longer have a server here
   // enable aux-service based timeline aggregators
   conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b7fb47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 641cef3..b8003c4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.ha.HAServiceProtocol;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.util.Shell;
@@ -770,8 +771,8 @@ public class MiniYARNCluster extends CompositeService {
   if (!useFixedPorts) {
 String hostname = MiniYARNCluster.getHostname();
 conf.set(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, hostname + ":0");
-conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, hostname
-+ ":0");
+conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+hostname + ":" + ServerSocketUtil.getPort(9188, 10));
   }
   appHistoryServer.init(conf);
   super.serviceInit(conf);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b7fb47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
--
diff --git

[15/50] [abbrv] hadoop git commit: YARN-4238. createdTime and modifiedTime is not reported while publishing entities to ATSv2. (Varun Saxena via Naganarasimha G R)

2016-05-04 Thread gtcarrera9 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
index e864d61..b7804e7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
@@ -107,7 +107,6 @@ public class TestFileSystemTimelineReaderImpl {
 entity11.setId("id_1");
 entity11.setType("app");
 entity11.setCreatedTime(1425016502000L);
-entity11.setModifiedTime(1425016502050L);
 Map info1 = new HashMap();
 info1.put("info1", "val1");
 entity11.addInfo(info1);
@@ -136,7 +135,6 @@ public class TestFileSystemTimelineReaderImpl {
 TimelineEntity entity12 = new TimelineEntity();
 entity12.setId("id_1");
 entity12.setType("app");
-entity12.setModifiedTime(1425016503000L);
 configs.clear();
 configs.put("config_2", "23");
 configs.put("config_3", "abc");
@@ -166,7 +164,6 @@ public class TestFileSystemTimelineReaderImpl {
 entity2.setId("id_2");
 entity2.setType("app");
 entity2.setCreatedTime(1425016501050L);
-entity2.setModifiedTime(1425016502010L);
 Map info2 = new HashMap();
 info1.put("info2", 4);
 entity2.addInfo(info2);
@@ -203,7 +200,6 @@ public class TestFileSystemTimelineReaderImpl {
 entity3.setId("id_3");
 entity3.setType("app");
 entity3.setCreatedTime(1425016501050L);
-entity3.setModifiedTime(1425016502010L);
 Map info3 = new HashMap();
 info3.put("info2", 3.5);
 entity3.addInfo(info3);
@@ -239,7 +235,6 @@ public class TestFileSystemTimelineReaderImpl {
 entity4.setId("id_4");
 entity4.setType("app");
 entity4.setCreatedTime(1425016502050L);
-entity4.setModifiedTime(1425016503010L);
 TimelineEvent event44 = new TimelineEvent();
 event44.setId("event_4");
 event44.setTimestamp(1425016502003L);
@@ -252,7 +247,6 @@ public class TestFileSystemTimelineReaderImpl {
 entity5.setId("id_5");
 entity5.setType("app");
 entity5.setCreatedTime(1425016502050L);
-entity5.setModifiedTime(1425016503010L);
 writeEntityFile(entity5, appDir2);
   }
 
@@ -263,7 +257,7 @@ public class TestFileSystemTimelineReaderImpl {
   @Test
   public void testGetEntityDefaultView() throws Exception {
 // If no fields are specified, entity is returned with default view i.e.
-// only the id, created and modified time
+// only the id, type and created time.
 TimelineEntity result =
 reader.getEntity("user1", "cluster1", "flow1", 1L, "app1",
 "app", "id_1", null, null, null);
@@ -271,7 +265,6 @@ public class TestFileSystemTimelineReaderImpl {
 (new TimelineEntity.Identifier("app", "id_1")).toString(),
 result.getIdentifier().toString());
 Assert.assertEquals(1425016502000L, result.getCreatedTime());
-Assert.assertEquals(1425016503000L, result.getModifiedTime());
 Assert.assertEquals(0, result.getConfigs().size());
 Assert.assertEquals(0, result.getMetrics().size());
   }
@@ -286,7 +279,6 @@ public class TestFileSystemTimelineReaderImpl {
 (new TimelineEntity.Identifier("app", "id_1")).toString(),
 result.getIdentifier().toString());
 Assert.assertEquals(1425016502000L, result.getCreatedTime());
-Assert.assertEquals(1425016503000L, result.getModifiedTime());
 Assert.assertEquals(0, result.getConfigs().size());
 Assert.assertEquals(0, result.getMetrics().size());
   }
@@ -303,7 +295,6 @@ public class TestFileSystemTimelineReaderImpl {
 (new TimelineEntity.Identifier("app", "id_5")).toString(),
 result.getIdentifier().toString());
 Assert.assertEquals(1425016502050L, result.getCreatedTime());
-Assert.assertEquals(1425016503010L, result.getModifiedTime());
   }
 
   @Test
@@ -317,7 +308,6 @@ public class TestFileSystemTimelineReaderImpl {
 (new TimelineEntity.Identifier("app", "id_1")).toString(),
 result.getIdentifier().toString());
 Assert.assertEquals(1425016502000L,

[01/50] [abbrv] hadoop git commit: YARN-4356. Ensure the timeline service v.2 is disabled cleanly and has no impact when it's turned off. Contributed by Sangjin Lee. [Forced Update!]

2016-05-04 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-2928 bdd5d7dfa -> d2c4237fa (forced update)


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
index 38b3172f..f31a98c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -535,7 +534,7 @@ public class TestApplication {
   this.appId = BuilderUtils.newApplicationId(timestamp, id);
 
   app = new ApplicationImpl(
-  dispatcher, this.user, null, null, 0, appId, null, context);
+  dispatcher, this.user, appId, null, context);
   containers = new ArrayList();
   for (int i = 0; i < numContainers; i++) {
 Container container = createMockedContainer(this.appId, i);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index e5ae1f8..c2dadd6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -31,17 +31,14 @@ import javax.ws.rs.core.MediaType;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 
-import org.junit.Assert;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.NodeHealthScriptRunner;
 import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -63,6 +60,7 @@ import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.w3c.dom.Document;
@@ -326,7 +324,7 @@ public class TestNMWebServices extends JerseyTestBase {
 final String filename = "logfile1";
 final String logMessage = "log message\n";
 nmContext.getApplications().put(appId, new ApplicationImpl(null, "user",
-null, null, 0, appId, null, nmContext));
+appId, null, nmContext));
 
 MockContainer container = new MockContainer(appAttemptId,
 new AsyncDispatcher(), new Configuration(), "user", appId, 1);

[07/50] [abbrv] hadoop git commit: YARN-4445. Unify the term flowId and flowName in timeline v2 codebase. Contributed by Zhan Zhang.

2016-05-04 Thread gtcarrera9 
YARN-4445. Unify the term flowId and flowName in timeline v2 codebase.
Contributed by Zhan Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89b4101f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89b4101f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89b4101f

Branch: refs/heads/YARN-2928
Commit: 89b4101f2c8b1e9473a8352b0412c6ea83066caf
Parents: 4764f4a
Author: Li Lu 
Authored: Wed Dec 16 16:42:57 2015 -0800
Committer: Li Lu 
Committed: Wed May 4 16:17:12 2016 -0700

--
 .../mapred/TestMRTimelineEventHandling.java |  2 +-
 .../distributedshell/TestDistributedShell.java  |  2 +-
 .../yarn/util/timeline/TimelineUtils.java   |  2 +-
 .../TestSystemMetricsPublisherForV2.java|  2 +-
 .../collector/AppLevelTimelineCollector.java|  2 +-
 .../reader/TimelineReaderManager.java   |  8 +--
 .../reader/TimelineReaderWebServices.java   | 74 ++--
 .../storage/ApplicationEntityReader.java| 20 +++---
 .../storage/FileSystemTimelineReaderImpl.java   | 14 ++--
 .../storage/FlowActivityEntityReader.java   | 10 +--
 .../storage/FlowRunEntityReader.java| 16 ++---
 .../storage/GenericEntityReader.java| 22 +++---
 .../storage/HBaseTimelineReaderImpl.java|  8 +--
 .../storage/TimelineEntityReader.java   | 10 +--
 .../storage/TimelineEntityReaderFactory.java| 20 +++---
 .../timelineservice/storage/TimelineReader.java |  8 +--
 .../storage/application/ApplicationRowKey.java  | 38 +-
 .../storage/application/ApplicationTable.java   |  2 +-
 .../apptoflow/AppToFlowColumnFamily.java|  2 +-
 .../storage/apptoflow/AppToFlowTable.java   |  4 +-
 .../storage/entity/EntityRowKey.java| 38 +-
 .../storage/entity/EntityTable.java |  2 +-
 .../storage/flow/FlowActivityRowKey.java| 30 
 .../storage/flow/FlowActivityTable.java |  2 +-
 .../storage/flow/FlowRunRowKey.java | 30 
 .../storage/flow/FlowRunTable.java  |  2 +-
 .../reader/TestTimelineReaderWebServices.java   |  2 +-
 ...stTimelineReaderWebServicesHBaseStorage.java |  4 +-
 .../storage/TestHBaseTimelineStorage.java   |  4 +-
 .../flow/TestHBaseStorageFlowActivity.java  |  6 +-
 30 files changed, 193 insertions(+), 193 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index 1896c7b..f7283ae 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -245,7 +245,7 @@ public class TestMRTimelineEventHandling {
 Assert.assertTrue(tmpRootFolder.isDirectory());
 String basePath = tmpRoot + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
 UserGroupInformation.getCurrentUser().getShortUserName() +
-"/" + TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId) +
+"/" + TimelineUtils.generateDefaultFlowNameBasedOnAppId(appId) +
 "/1/1/" + appId.toString();
 // for this test, we expect MAPREDUCE_JOB and MAPREDUCE_TASK dirs
 String outputDirJob = basePath + "/MAPREDUCE_JOB/";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index bdf6e2b..e35a32e 100644
---

[19/50] [abbrv] hadoop git commit: YARN-4224. Support fetching entities by UID and change the REST interface to conform to current REST APIs' in YARN. (Varun Saxena via gtcarrera9)

2016-05-04 Thread gtcarrera9 
YARN-4224. Support fetching entities by UID and change the REST
interface to conform to current REST APIs' in YARN. (Varun Saxena via
gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/97d5cf32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/97d5cf32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/97d5cf32

Branch: refs/heads/YARN-2928
Commit: 97d5cf323128e9d6e3f832de891b90e1434fee7d
Parents: fd369a5
Author: Li Lu <gtcarre...@apache.org>
Authored: Wed Jan 27 14:04:09 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Wed May 4 16:24:15 2016 -0700

--
 .../records/timelineservice/TimelineEntity.java |   13 +
 .../server/timelineservice/TimelineContext.java |  146 ++
 .../collector/TimelineCollectorContext.java |   86 +-
 .../server/timelineservice/package-info.java|   28 +
 .../reader/TimelineReaderContext.java   |   88 ++
 .../reader/TimelineReaderManager.java   |   82 +-
 .../reader/TimelineReaderUtils.java |  171 ++
 .../reader/TimelineReaderWebServices.java   | 1469 +++---
 .../reader/TimelineReaderWebServicesUtils.java  |  222 +++
 .../reader/TimelineUIDConverter.java|  245 +++
 .../timelineservice/storage/TimelineReader.java |   16 +-
 .../storage/reader/GenericEntityReader.java |7 +-
 .../reader/TestTimelineReaderUtils.java |   55 +
 .../reader/TestTimelineReaderWebServices.java   |   83 +-
 ...stTimelineReaderWebServicesHBaseStorage.java |  348 -
 .../reader/TestTimelineUIDConverter.java|   97 ++
 ...TestPhoenixOfflineAggregationWriterImpl.java |2 +-
 17 files changed, 2781 insertions(+), 377 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index dcf2473..a661f7a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -504,6 +504,19 @@ public class TimelineEntity implements 
Comparable {
 }
   }
 
+  /**
+   * Set UID in info which will be then used for query by UI.
+   * @param uidKey key for UID in info.
+   * @param uId UID to be set for the key.
+   */
+  public void setUID(String uidKey, String uId) {
+if (real == null) {
+  info.put(uidKey, uId);
+} else {
+  real.addInfo(uidKey, uId);
+}
+  }
+
   public boolean isValid() {
 return (getId() != null && getType() != null);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/TimelineContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/TimelineContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/TimelineContext.java
new file mode 100644
index 000..694b709
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/TimelineContext.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limita

[02/50] [abbrv] hadoop git commit: YARN-4356. Ensure the timeline service v.2 is disabled cleanly and has no impact when it's turned off. Contributed by Sangjin Lee.

2016-05-04 Thread gtcarrera9 
YARN-4356. Ensure the timeline service v.2 is disabled cleanly and has no
impact when it's turned off. Contributed by Sangjin Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef71c1fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef71c1fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef71c1fc

Branch: refs/heads/YARN-2928
Commit: ef71c1fc4c37b355e895cea23a868ba57555d9b5
Parents: e987c7e
Author: Li Lu 
Authored: Fri Dec 11 11:17:34 2015 -0800
Committer: Li Lu 
Committed: Wed May 4 16:01:41 2016 -0700

--
 .../jobhistory/JobHistoryEventHandler.java  |  63 
 .../hadoop/mapreduce/v2/app/MRAppMaster.java|  11 +-
 .../apache/hadoop/mapreduce/MRJobConfig.java|   5 -
 .../src/main/resources/mapred-default.xml   |   7 -
 .../mapred/TestMRTimelineEventHandling.java |   5 +-
 .../hadoop/mapreduce/v2/MiniMRYarnCluster.java  |   2 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |  58 ++-
 .../distributedshell/ApplicationMaster.java | 153 ---
 .../applications/distributedshell/Client.java   |  16 --
 .../distributedshell/TestDistributedShell.java  |  10 +-
 .../hadoop/yarn/client/api/TimelineClient.java  |  18 ++-
 .../client/api/impl/TimelineClientImpl.java |   3 +
 .../src/main/resources/yarn-default.xml |   5 +-
 .../impl/pb/NodeHeartbeatRequestPBImpl.java |  10 +-
 .../impl/pb/NodeHeartbeatResponsePBImpl.java|  10 +-
 .../hadoop/yarn/server/nodemanager/Context.java |   3 +-
 .../yarn/server/nodemanager/NodeManager.java|  23 ++-
 .../nodemanager/NodeStatusUpdaterImpl.java  |  48 +++---
 .../collectormanager/NMCollectorService.java|  10 +-
 .../containermanager/ContainerManagerImpl.java  |  62 +---
 .../application/ApplicationImpl.java|  70 +++--
 .../monitor/ContainersMonitorImpl.java  |  11 +-
 .../timelineservice/NMTimelinePublisher.java|  49 +++---
 .../TestContainerManagerRecovery.java   |   9 +-
 .../application/TestApplication.java|   3 +-
 .../nodemanager/webapp/TestNMWebServices.java   |   8 +-
 .../ApplicationMasterService.java   |  11 +-
 .../server/resourcemanager/ClientRMService.java |  35 +++--
 .../server/resourcemanager/RMAppManager.java|   7 +-
 .../server/resourcemanager/ResourceManager.java |  33 ++--
 .../resourcemanager/ResourceTrackerService.java |  21 ++-
 .../resourcemanager/amlauncher/AMLauncher.java  |  15 +-
 .../metrics/TimelineServiceV2Publisher.java |   2 +-
 .../server/resourcemanager/rmapp/RMAppImpl.java |  16 +-
 .../resourcemanager/TestClientRMService.java|   3 +
 .../metrics/TestSystemMetricsPublisher.java |   2 +-
 .../TestSystemMetricsPublisherForV2.java|   1 +
 .../TestTimelineServiceClientIntegration.java   |  30 +++-
 .../PerNodeTimelineCollectorsAuxService.java|  15 +-
 .../reader/TimelineReaderServer.java|  14 +-
 ...TestPerNodeTimelineCollectorsAuxService.java |   9 +-
 .../reader/TestTimelineReaderServer.java|   3 +
 .../reader/TestTimelineReaderWebServices.java   |   2 +
 ...stTimelineReaderWebServicesHBaseStorage.java |   2 +
 44 files changed, 520 insertions(+), 373 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index b1c1a52..d88588c 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -19,9 +19,6 @@
 package org.apache.hadoop.mapreduce.jobhistory;
 
 import java.io.IOException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -31,7 +28,11 @@ import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import

[05/50] [abbrv] hadoop git commit: YARN-4460. [Bug fix] RM fails to start when SMP is enabled. (Li Lu via Varun Saxena)

2016-05-04 Thread gtcarrera9 
YARN-4460. [Bug fix] RM fails to start when SMP is enabled. (Li Lu via Varun 
Saxena)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4764f4ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4764f4ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4764f4ac

Branch: refs/heads/YARN-2928
Commit: 4764f4accc5c7fe02f892a78e979819fbf46f90c
Parents: 8d798de
Author: Varun Saxena 
Authored: Wed Dec 16 15:24:57 2015 +0530
Committer: Li Lu 
Committed: Wed May 4 16:17:09 2016 -0700

--
 .../resourcemanager/metrics/TimelineServiceV2Publisher.java | 9 +
 .../metrics/TestSystemMetricsPublisherForV2.java| 5 -
 2 files changed, 9 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4764f4ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index b96114e..1954783 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -75,13 +76,13 @@ public class TimelineServiceV2Publisher extends 
AbstractSystemMetricsPublisher {
   }
 
   @Override
-  protected void serviceStart() throws Exception {
-super.serviceStart();
+  protected void serviceInit(Configuration conf) throws Exception {
+super.serviceInit(conf);
+getDispatcher().register(SystemMetricsEventType.class,
+new TimelineV2EventHandler());
 publishContainerMetrics = getConfig().getBoolean(
 YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
 YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED);
-getDispatcher().register(SystemMetricsEventType.class,
-new TimelineV2EventHandler());
   }
 
   @VisibleForTesting

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4764f4ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index baaa566..57258d5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -156,7 +156,7 @@ public class TestSystemMetricsPublisherForV2 {
 try {
   Configuration conf = getTimelineV2Conf();
   conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
-  false);
+  YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED);
   metricsPublisher.init(conf);
   assertFalse(
   "Default configuration should not publish container Metrics from RM",
@@ -167,6 +167,9 @@ public class TestSystemMetricsPublisherForV2 {
   metricsPublisher = new

hadoop git commit: YARN-3150. Documenting the timeline service v2. (Sangjin Lee and Vrushali C via gtcarrera9)

2016-04-30 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-2928 c6e70a6cf -> 735048c61


YARN-3150. Documenting the timeline service v2. (Sangjin Lee and Vrushali C via 
gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/735048c6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/735048c6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/735048c6

Branch: refs/heads/YARN-2928
Commit: 735048c6124b9f6ce0078ace73d6f3070b04b079
Parents: c6e70a6
Author: Li Lu <gtcarre...@apache.org>
Authored: Sat Apr 30 15:02:12 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Sat Apr 30 15:02:12 2016 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   8 +-
 .../src/main/resources/yarn-default.xml |   6 +-
 .../metrics/TimelineServiceV2Publisher.java |  16 +-
 .../TestSystemMetricsPublisherForV2.java|  16 +-
 .../src/site/markdown/TimelineServer.md |   2 +-
 .../src/site/markdown/TimelineServiceV2.md  | 576 +++
 .../src/site/resources/images/timeline_v2.jpg   | Bin 0 -> 45112 bytes
 7 files changed, 600 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/735048c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 863b5a1..9fae6d0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -413,13 +413,13 @@ public class YarnConfiguration extends Configuration {
   public static final boolean DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED = false;
 
   /**
-   * The setting that controls whether yarn container metrics is published to
-   * the timeline server or not by RM. This configuration setting is for ATS
+   * The setting that controls whether yarn container events are published to
+   * the timeline service or not by RM. This configuration setting is for ATS
* V2
*/
-  public static final String RM_PUBLISH_CONTAINER_METRICS_ENABLED = YARN_PREFIX
+  public static final String RM_PUBLISH_CONTAINER_EVENTS_ENABLED = YARN_PREFIX
   + "rm.system-metrics-publisher.emit-container-events";
-  public static final boolean DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED =
+  public static final boolean DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED =
   false;
 
   public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/735048c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 31b897b..88ffaf8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -785,9 +785,9 @@
   
 
   
-The setting that controls whether yarn container metrics is
-published to the timeline server or not by RM. This configuration setting 
is
-for ATS V2.
+The setting that controls whether yarn container events are
+published to the timeline service or not by RM. This configuration setting
+is for ATS V2.
 yarn.rm.system-metrics-publisher.emit-container-events
 false
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/735048c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 14073d1..b7ece5f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main

hadoop git commit: YARN-4956. findbug issue on LevelDBCacheTimelineStore. (Zhiyuan Yang via gtcarrera9)

2016-04-27 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 7cd017731 -> 76b4ec32e


YARN-4956. findbug issue on LevelDBCacheTimelineStore. (Zhiyuan Yang via 
gtcarrera9)

(cherry picked from commit f16722d2ef31338a57a13e2c8d18c1c62d58bbaf)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76b4ec32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76b4ec32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76b4ec32

Branch: refs/heads/branch-2.8
Commit: 76b4ec32e663a924d73193d809428d776bef84e9
Parents: 7cd0177
Author: Li Lu <gtcarre...@apache.org>
Authored: Wed Apr 27 10:57:10 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Wed Apr 27 10:58:50 2016 -0700

--
 .../hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/76b4ec32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
index 3ff5dd7..ccf2d94 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
@@ -78,7 +78,7 @@ public class LevelDBCacheTimelineStore extends 
KeyValueBasedTimelineStore {
   }
 
   @Override
-  protected void serviceInit(Configuration conf) throws Exception {
+  protected synchronized void serviceInit(Configuration conf) throws Exception 
{
 configuration = conf;
 Options options = new Options();
 options.createIfMissing(true);

hadoop git commit: YARN-4956. findbug issue on LevelDBCacheTimelineStore. (Zhiyuan Yang via gtcarrera9)

2016-04-27 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 14b8701a6 -> 50b7a35d5


YARN-4956. findbug issue on LevelDBCacheTimelineStore. (Zhiyuan Yang via 
gtcarrera9)

(cherry picked from commit f16722d2ef31338a57a13e2c8d18c1c62d58bbaf)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50b7a35d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50b7a35d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50b7a35d

Branch: refs/heads/branch-2
Commit: 50b7a35d560d6493884a423d7b928d48fab1ba65
Parents: 14b8701
Author: Li Lu <gtcarre...@apache.org>
Authored: Wed Apr 27 10:57:10 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Wed Apr 27 10:58:12 2016 -0700

--
 .../hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/50b7a35d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
index 3ff5dd7..ccf2d94 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
@@ -78,7 +78,7 @@ public class LevelDBCacheTimelineStore extends 
KeyValueBasedTimelineStore {
   }
 
   @Override
-  protected void serviceInit(Configuration conf) throws Exception {
+  protected synchronized void serviceInit(Configuration conf) throws Exception 
{
 configuration = conf;
 Options options = new Options();
 options.createIfMissing(true);

hadoop git commit: YARN-4956. findbug issue on LevelDBCacheTimelineStore. (Zhiyuan Yang via gtcarrera9)

2016-04-27 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 3337ef2bf -> f16722d2e


YARN-4956. findbug issue on LevelDBCacheTimelineStore. (Zhiyuan Yang via 
gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f16722d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f16722d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f16722d2

Branch: refs/heads/trunk
Commit: f16722d2ef31338a57a13e2c8d18c1c62d58bbaf
Parents: 3337ef2
Author: Li Lu <gtcarre...@apache.org>
Authored: Wed Apr 27 10:57:10 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Wed Apr 27 10:57:10 2016 -0700

--
 .../hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f16722d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
index 3ff5dd7..ccf2d94 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
@@ -78,7 +78,7 @@ public class LevelDBCacheTimelineStore extends 
KeyValueBasedTimelineStore {
   }
 
   @Override
-  protected void serviceInit(Configuration conf) throws Exception {
+  protected synchronized void serviceInit(Configuration conf) throws Exception 
{
 configuration = conf;
 Options options = new Options();
 options.createIfMissing(true);

hadoop git commit: YARN-4968. A couple of AM retry unit tests need to wait SchedulerApplicationAttempt stopped. (Wangda Tan via gtcarrera9)

2016-04-21 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk a749ba0ce -> 7c6339f66


YARN-4968. A couple of AM retry unit tests need to wait 
SchedulerApplicationAttempt stopped. (Wangda Tan via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c6339f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c6339f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c6339f6

Branch: refs/heads/trunk
Commit: 7c6339f66ac301406504be28841bc3f3bfebc8ae
Parents: a749ba0
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Apr 21 13:25:33 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Apr 21 13:25:33 2016 -0700

--
 .../server/resourcemanager/TestRMRestart.java   |  9 +++-
 .../applicationsmanager/TestAMRestart.java  | 16 ++
 .../scheduler/TestSchedulerUtils.java   | 23 
 3 files changed, 47 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c6339f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 3b9952b..3e4a334 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -104,6 +104,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
@@ -2328,6 +2329,8 @@ public class TestRMRestart extends 
ParameterizedSchedulerTestBase {
 // start RM
 MockRM rm1 = new MockRM(conf, memStore);
 rm1.start();
+CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+
 MockNM nm1 =
 new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
 nm1.registerNode();
@@ -2338,10 +2341,12 @@ public class TestRMRestart extends 
ParameterizedSchedulerTestBase {
 nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1,
 ContainerState.COMPLETE);
 am0.waitForState(RMAppAttemptState.FAILED);
+TestSchedulerUtils.waitSchedulerApplicationAttemptStopped(cs,
+am0.getApplicationAttemptId());
+
 for (int i = 0; i < 4; i++) {
   am0 = MockRM.launchAM(app0, rm1, nm1);
   am0.registerAppAttempt();
-  CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
   // get scheduler app
   FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications()
   .get(app0.getApplicationId()).getCurrentAppAttempt();
@@ -2349,6 +2354,8 @@ public class TestRMRestart extends 
ParameterizedSchedulerTestBase {
   cs.markContainerForKillable(schedulerAppAttempt.getRMContainer(
   app0.getCurrentAppAttempt().getMasterContainer().getId()));
   am0.waitForState(RMAppAttemptState.FAILED);
+  TestSchedulerUtils.waitSchedulerApplicationAttemptStopped(cs,
+  am0.getApplicationAttemptId());
 }
 am0 = MockRM.launchAM(app0, rm1, nm1);
 am0.registerAppAttempt();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c6339f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yar

hadoop git commit: YARN-4968. A couple of AM retry unit tests need to wait SchedulerApplicationAttempt stopped. (Wangda Tan via gtcarrera9)

2016-04-21 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 f98f4151a -> 054fa104c


YARN-4968. A couple of AM retry unit tests need to wait 
SchedulerApplicationAttempt stopped. (Wangda Tan via gtcarrera9)

(cherry picked from commit 7c6339f66ac301406504be28841bc3f3bfebc8ae)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/054fa104
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/054fa104
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/054fa104

Branch: refs/heads/branch-2
Commit: 054fa104c5ffc39743bd6c2dfa95e66042226ed5
Parents: f98f415
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Apr 21 13:25:33 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Apr 21 13:27:47 2016 -0700

--
 .../server/resourcemanager/TestRMRestart.java   |  9 +++-
 .../applicationsmanager/TestAMRestart.java  | 16 ++
 .../scheduler/TestSchedulerUtils.java   | 23 
 3 files changed, 47 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/054fa104/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 3b9952b..3e4a334 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -104,6 +104,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
@@ -2328,6 +2329,8 @@ public class TestRMRestart extends 
ParameterizedSchedulerTestBase {
 // start RM
 MockRM rm1 = new MockRM(conf, memStore);
 rm1.start();
+CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+
 MockNM nm1 =
 new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
 nm1.registerNode();
@@ -2338,10 +2341,12 @@ public class TestRMRestart extends 
ParameterizedSchedulerTestBase {
 nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1,
 ContainerState.COMPLETE);
 am0.waitForState(RMAppAttemptState.FAILED);
+TestSchedulerUtils.waitSchedulerApplicationAttemptStopped(cs,
+am0.getApplicationAttemptId());
+
 for (int i = 0; i < 4; i++) {
   am0 = MockRM.launchAM(app0, rm1, nm1);
   am0.registerAppAttempt();
-  CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
   // get scheduler app
   FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications()
   .get(app0.getApplicationId()).getCurrentAppAttempt();
@@ -2349,6 +2354,8 @@ public class TestRMRestart extends 
ParameterizedSchedulerTestBase {
   cs.markContainerForKillable(schedulerAppAttempt.getRMContainer(
   app0.getCurrentAppAttempt().getMasterContainer().getId()));
   am0.waitForState(RMAppAttemptState.FAILED);
+  TestSchedulerUtils.waitSchedulerApplicationAttemptStopped(cs,
+  am0.getApplicationAttemptId());
 }
 am0 = MockRM.launchAM(app0, rm1, nm1);
 am0.registerAppAttempt();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/054fa104/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAM

hadoop git commit: YARN-4706. UI Hosting Configuration in TimelineServer doc is broken. (Akira AJISAKA via gtcarrera9)

2016-04-04 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 353f37916 -> f242b78fd


YARN-4706. UI Hosting Configuration in TimelineServer doc is broken. (Akira 
AJISAKA via gtcarrera9)

(cherry picked from commit f61de4173684aa1767cef20b3cb4d54df20273cd)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f242b78f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f242b78f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f242b78f

Branch: refs/heads/branch-2.7
Commit: f242b78fd9f9d31f1344d3a0fadf7d54851ae14f
Parents: 353f379
Author: Li Lu <gtcarre...@apache.org>
Authored: Mon Apr 4 14:39:47 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Mon Apr 4 14:46:28 2016 -0700

--
 hadoop-yarn-project/CHANGES.txt| 2 ++
 .../hadoop-yarn-site/src/site/markdown/TimelineServer.md   | 2 ++
 2 files changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f242b78f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 74bcdda..d26d1d3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -25,6 +25,8 @@ Release 2.7.3 - UNRELEASED
 
   BUG FIXES
 
+YARN-4706. UI Hosting Configuration in TimelineServer doc is broken. 
(Akira AJISAKA via gtcarrera9)
+
 YARN-4326. Fix TestDistributedShell timeout as AHS in MiniYarnCluster no 
longer 
 binds to default port 8188. (Meng Ding via wangda)
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f242b78f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index f3473f0..0009f34 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -52,6 +52,7 @@ With the introduction of the timeline server, the Application 
History Server bec
 the Timeline Server.
 
 Generic information includes application level data such as 
+
 * queue-name, 
 * user information and the like set in the `ApplicationSubmissionContext`,
 * a list of application-attempts that ran for an application
@@ -188,6 +189,7 @@ selected if this policy is either of `HTTPS_ONLY` or 
`HTTP_AND_HTTPS`.
  UI Hosting Configuration
 
 The timeline service can host multiple UIs if enabled. The service can support 
both static web sites hosted in a directory or war files bundled. The web UI is 
then hosted on the timeline service HTTP port under the path configured.
+
 | Configuration Property | Description |
 |: |: |
 | `yarn.timeline-service.ui-names` | Comma separated list of UIs that will be 
hosted. Defaults to `none`. |

hadoop git commit: YARN-4706. UI Hosting Configuration in TimelineServer doc is broken. (Akira AJISAKA via gtcarrera9)

2016-04-04 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 7d671cad3 -> 3574b4559


YARN-4706. UI Hosting Configuration in TimelineServer doc is broken. (Akira 
AJISAKA via gtcarrera9)

(cherry picked from commit f61de4173684aa1767cef20b3cb4d54df20273cd)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3574b455
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3574b455
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3574b455

Branch: refs/heads/branch-2.8
Commit: 3574b4559c8558a94f94c35132276f4d4aa99bcb
Parents: 7d671ca
Author: Li Lu <gtcarre...@apache.org>
Authored: Mon Apr 4 14:39:47 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Mon Apr 4 14:43:37 2016 -0700

--
 .../hadoop-yarn-site/src/site/markdown/TimelineServer.md   | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3574b455/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index d8e534e..efea80b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -52,6 +52,7 @@ With the introduction of the timeline server, the Application 
History Server bec
 the Timeline Server.
 
 Generic information includes application level data such as 
+
 * queue-name, 
 * user information and the like set in the `ApplicationSubmissionContext`,
 * a list of application-attempts that ran for an application
@@ -188,6 +189,7 @@ selected if this policy is `HTTPS_ONLY`.
  UI Hosting Configuration
 
 The timeline service can host multiple UIs if enabled. The service can support 
both static web sites hosted in a directory or war files bundled. The web UI is 
then hosted on the timeline service HTTP port under the path configured.
+
 | Configuration Property | Description |
 |: |: |
 | `yarn.timeline-service.ui-names` | Comma separated list of UIs that will be 
hosted. Defaults to `none`. |

hadoop git commit: YARN-4706. UI Hosting Configuration in TimelineServer doc is broken. (Akira AJISAKA via gtcarrera9)

2016-04-04 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 9e9143329 -> 236d1a509


YARN-4706. UI Hosting Configuration in TimelineServer doc is broken. (Akira 
AJISAKA via gtcarrera9)

(cherry picked from commit f61de4173684aa1767cef20b3cb4d54df20273cd)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/236d1a50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/236d1a50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/236d1a50

Branch: refs/heads/branch-2
Commit: 236d1a5093fd1f4cddae6bc6d1efe9e31cea6eea
Parents: 9e91433
Author: Li Lu <gtcarre...@apache.org>
Authored: Mon Apr 4 14:39:47 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Mon Apr 4 14:41:40 2016 -0700

--
 .../hadoop-yarn-site/src/site/markdown/TimelineServer.md   | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/236d1a50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index 9283e58..f20bd2c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -56,6 +56,7 @@ With the introduction of the timeline server, the Application 
History Server bec
 the Timeline Server.
 
 Generic information includes application level data such as 
+
 * queue-name, 
 * user information and the like set in the `ApplicationSubmissionContext`,
 * a list of application-attempts that ran for an application
@@ -192,6 +193,7 @@ selected if this policy is `HTTPS_ONLY`.
  UI Hosting Configuration
 
 The timeline service can host multiple UIs if enabled. The service can support 
both static web sites hosted in a directory or war files bundled. The web UI is 
then hosted on the timeline service HTTP port under the path configured.
+
 | Configuration Property | Description |
 |: |: |
 | `yarn.timeline-service.ui-names` | Comma separated list of UIs that will be 
hosted. Defaults to `none`. |

hadoop git commit: YARN-4706. UI Hosting Configuration in TimelineServer doc is broken. (Akira AJISAKA via gtcarrera9)

2016-04-04 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk a7d1fb0cd -> f61de4173


YARN-4706. UI Hosting Configuration in TimelineServer doc is broken. (Akira 
AJISAKA via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f61de417
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f61de417
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f61de417

Branch: refs/heads/trunk
Commit: f61de4173684aa1767cef20b3cb4d54df20273cd
Parents: a7d1fb0
Author: Li Lu <gtcarre...@apache.org>
Authored: Mon Apr 4 14:39:47 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Mon Apr 4 14:40:27 2016 -0700

--
 .../hadoop-yarn-site/src/site/markdown/TimelineServer.md   | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f61de417/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index 9283e58..f20bd2c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -56,6 +56,7 @@ With the introduction of the timeline server, the Application 
History Server bec
 the Timeline Server.
 
 Generic information includes application level data such as 
+
 * queue-name, 
 * user information and the like set in the `ApplicationSubmissionContext`,
 * a list of application-attempts that ran for an application
@@ -192,6 +193,7 @@ selected if this policy is `HTTPS_ONLY`.
  UI Hosting Configuration
 
 The timeline service can host multiple UIs if enabled. The service can support 
both static web sites hosted in a directory or war files bundled. The web UI is 
then hosted on the timeline service HTTP port under the path configured.
+
 | Configuration Property | Description |
 |: |: |
 | `yarn.timeline-service.ui-names` | Comma separated list of UIs that will be 
hosted. Defaults to `none`. |

hadoop git commit: MAPREDUCE-6110. JobHistoryServer CLI throws NullPointerException with job ids that do not exist. (Kai Sasaki via gtcarrera9)

2016-03-22 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 4dd02f948 -> 8bbb5efbf


MAPREDUCE-6110. JobHistoryServer CLI throws NullPointerException with job ids 
that do not exist. (Kai Sasaki via gtcarrera9)

(cherry picked from commit 57ead18a85e15aef1993f49157cf05aed38f1c87)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bbb5efb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bbb5efb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bbb5efb

Branch: refs/heads/branch-2
Commit: 8bbb5efbfb4f6fc207bb325407a9f37a55f2594b
Parents: 4dd02f9
Author: Li Lu <gtcarre...@apache.org>
Authored: Tue Mar 22 14:27:32 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Tue Mar 22 14:29:12 2016 -0700

--
 .../org/apache/hadoop/mapreduce/tools/CLI.java  | 48 +---
 .../apache/hadoop/mapreduce/tools/TestCLI.java  | 38 +++-
 2 files changed, 68 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbb5efb/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
index a8208c0..41fe133 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
@@ -402,8 +402,13 @@ public class CLI extends Configured implements Tool {
   }
 }
   } else if (listEvents) {
-listEvents(getJob(JobID.forName(jobid)), fromEvent, nEvents);
-exitCode = 0;
+Job job = getJob(JobID.forName(jobid));
+if (job == null) {
+  System.out.println("Could not find job " + jobid);
+} else {
+  listEvents(job, fromEvent, nEvents);
+  exitCode = 0;
+}
   } else if (listJobs) {
 listJobs(cluster);
 exitCode = 0;
@@ -417,8 +422,13 @@ public class CLI extends Configured implements Tool {
 listBlacklistedTrackers(cluster);
 exitCode = 0;
   } else if (displayTasks) {
-displayTasks(getJob(JobID.forName(jobid)), taskType, taskState);
-exitCode = 0;
+Job job = getJob(JobID.forName(jobid));
+if (job == null) {
+  System.out.println("Could not find job " + jobid);
+} else {
+  displayTasks(getJob(JobID.forName(jobid)), taskType, taskState);
+  exitCode = 0;
+}
   } else if(killTask) {
 TaskAttemptID taskID = TaskAttemptID.forName(taskid);
 Job job = getJob(taskID.getJobID());
@@ -444,20 +454,24 @@ public class CLI extends Configured implements Tool {
   exitCode = -1;
 }
   } else if (logs) {
-try {
 JobID jobID = JobID.forName(jobid);
-TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskid);
-LogParams logParams = cluster.getLogParams(jobID, taskAttemptID);
-LogCLIHelpers logDumper = new LogCLIHelpers();
-logDumper.setConf(getConf());
-exitCode = logDumper.dumpAContainersLogs(logParams.getApplicationId(),
-logParams.getContainerId(), logParams.getNodeId(),
-logParams.getOwner());
-} catch (IOException e) {
-  if (e instanceof RemoteException) {
-throw e;
-  } 
-  System.out.println(e.getMessage());
+if (getJob(jobID) == null) {
+  System.out.println("Could not find job " + jobid);
+} else {
+  try {
+TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskid);
+LogParams logParams = cluster.getLogParams(jobID, taskAttemptID);
+LogCLIHelpers logDumper = new LogCLIHelpers();
+logDumper.setConf(getConf());
+exitCode = logDumper.dumpAContainersLogs(
+logParams.getApplicationId(), logParams.getContainerId(),
+logParams.getNodeId(), logParams.getOwner());
+  } catch (IOException e) {
+if (e instanceof RemoteException) {
+  throw e;
+}
+System.out.println(e.getMessage());
+  }
 }
   }
 } catch (RemoteException re) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbb5efb/hadoop-mapreduce-project/hadoop-

hadoop git commit: MAPREDUCE-6110. JobHistoryServer CLI throws NullPointerException with job ids that do not exist. (Kai Sasaki via gtcarrera9)

2016-03-22 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk e7ed05e4f -> 57ead18a8


MAPREDUCE-6110. JobHistoryServer CLI throws NullPointerException with job ids 
that do not exist. (Kai Sasaki via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57ead18a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57ead18a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57ead18a

Branch: refs/heads/trunk
Commit: 57ead18a85e15aef1993f49157cf05aed38f1c87
Parents: e7ed05e
Author: Li Lu <gtcarre...@apache.org>
Authored: Tue Mar 22 14:27:32 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Tue Mar 22 14:27:32 2016 -0700

--
 .../org/apache/hadoop/mapreduce/tools/CLI.java  | 48 +---
 .../apache/hadoop/mapreduce/tools/TestCLI.java  | 38 +++-
 2 files changed, 68 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ead18a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
index bc3c0c8..4f55792 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
@@ -402,8 +402,13 @@ public class CLI extends Configured implements Tool {
   }
 }
   } else if (listEvents) {
-listEvents(getJob(JobID.forName(jobid)), fromEvent, nEvents);
-exitCode = 0;
+Job job = getJob(JobID.forName(jobid));
+if (job == null) {
+  System.out.println("Could not find job " + jobid);
+} else {
+  listEvents(job, fromEvent, nEvents);
+  exitCode = 0;
+}
   } else if (listJobs) {
 listJobs(cluster);
 exitCode = 0;
@@ -417,8 +422,13 @@ public class CLI extends Configured implements Tool {
 listBlacklistedTrackers(cluster);
 exitCode = 0;
   } else if (displayTasks) {
-displayTasks(getJob(JobID.forName(jobid)), taskType, taskState);
-exitCode = 0;
+Job job = getJob(JobID.forName(jobid));
+if (job == null) {
+  System.out.println("Could not find job " + jobid);
+} else {
+  displayTasks(getJob(JobID.forName(jobid)), taskType, taskState);
+  exitCode = 0;
+}
   } else if(killTask) {
 TaskAttemptID taskID = TaskAttemptID.forName(taskid);
 Job job = getJob(taskID.getJobID());
@@ -444,20 +454,24 @@ public class CLI extends Configured implements Tool {
   exitCode = -1;
 }
   } else if (logs) {
-try {
 JobID jobID = JobID.forName(jobid);
-TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskid);
-LogParams logParams = cluster.getLogParams(jobID, taskAttemptID);
-LogCLIHelpers logDumper = new LogCLIHelpers();
-logDumper.setConf(getConf());
-exitCode = logDumper.dumpAContainersLogs(logParams.getApplicationId(),
-logParams.getContainerId(), logParams.getNodeId(),
-logParams.getOwner());
-} catch (IOException e) {
-  if (e instanceof RemoteException) {
-throw e;
-  } 
-  System.out.println(e.getMessage());
+if (getJob(jobID) == null) {
+  System.out.println("Could not find job " + jobid);
+} else {
+  try {
+TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskid);
+LogParams logParams = cluster.getLogParams(jobID, taskAttemptID);
+LogCLIHelpers logDumper = new LogCLIHelpers();
+logDumper.setConf(getConf());
+exitCode = logDumper.dumpAContainersLogs(
+logParams.getApplicationId(), logParams.getContainerId(),
+logParams.getNodeId(), logParams.getOwner());
+  } catch (IOException e) {
+if (e instanceof RemoteException) {
+  throw e;
+}
+System.out.println(e.getMessage());
+  }
 }
   }
 } catch (RemoteException re) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/57ead18a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/o

hadoop git commit: YARN-4817. Change Log Level to DEBUG for putDomain call in ATS 1.5. (Xuan Gong via gtcarrera9)

2016-03-15 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 0ebc94714 -> 8640a9c45


YARN-4817. Change Log Level to DEBUG for putDomain call in ATS 1.5. (Xuan Gong 
via gtcarrera9)

(cherry-picked from commit 5176a69868484f864c977a6cd13923167082407c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8640a9c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8640a9c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8640a9c4

Branch: refs/heads/branch-2.8
Commit: 8640a9c45f01576a11e4c9d6c870bf009c4516a2
Parents: 0ebc947
Author: Li Lu <gtcarre...@apache.org>
Authored: Tue Mar 15 11:47:39 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Tue Mar 15 11:49:09 2016 -0700

--
 .../hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java  | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8640a9c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 44cad60..c494f65 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -306,8 +306,10 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 Path domainLogPath =
 new Path(createAttemptDir(appAttemptId), DOMAIN_LOG_PREFIX
 + appAttemptId.toString());
-LOG.info("Writing domains for " + appAttemptId.toString() + " to "
-+ domainLogPath);
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Writing domains for " + appAttemptId.toString() + " to "
+  + domainLogPath);
+}
 this.logFDsCache.writeDomainLog(
 fs, domainLogPath, objMapper, domain, isAppendSupported);
   }

hadoop git commit: YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. (Xuan Gong via gtcarrera9)

2016-03-15 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 da24fde33 -> 14a4110b7


YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. 
(Xuan Gong via gtcarrera9)

(cherry picked from commit 3a0b69544183f71c4683bc477656473494fc09d0)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14a4110b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14a4110b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14a4110b

Branch: refs/heads/branch-2
Commit: 14a4110b75cb6cca41dca9c121344d876a45
Parents: da24fde
Author: Li Lu <gtcarre...@apache.org>
Authored: Tue Mar 15 11:40:54 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Tue Mar 15 11:42:30 2016 -0700

--
 .../hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a4110b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 9e719b7..44cad60 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -276,6 +276,7 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 mapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector());
 mapper.setSerializationInclusion(Inclusion.NON_NULL);
 mapper.configure(Feature.CLOSE_CLOSEABLE, false);
+mapper.configure(Feature.FLUSH_AFTER_WRITE_VALUE, false);
 return mapper;
   }

hadoop git commit: YARN-4817. Change Log Level to DEBUG for putDomain call in ATS 1.5. (Xuan Gong via gtcarrera9)

2016-03-15 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 14a4110b7 -> 94649b9cc


YARN-4817. Change Log Level to DEBUG for putDomain call in ATS 1.5. (Xuan Gong 
via gtcarrera9)

(cherry-picked from commit 5176a69868484f864c977a6cd13923167082407c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94649b9c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94649b9c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94649b9c

Branch: refs/heads/branch-2
Commit: 94649b9cc088b52eabc0a7f5106e340e98a98e00
Parents: 14a4110
Author: Li Lu <gtcarre...@apache.org>
Authored: Tue Mar 15 11:47:39 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Tue Mar 15 11:48:37 2016 -0700

--
 .../hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java  | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/94649b9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 44cad60..c494f65 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -306,8 +306,10 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 Path domainLogPath =
 new Path(createAttemptDir(appAttemptId), DOMAIN_LOG_PREFIX
 + appAttemptId.toString());
-LOG.info("Writing domains for " + appAttemptId.toString() + " to "
-+ domainLogPath);
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Writing domains for " + appAttemptId.toString() + " to "
+  + domainLogPath);
+}
 this.logFDsCache.writeDomainLog(
 fs, domainLogPath, objMapper, domain, isAppendSupported);
   }

hadoop git commit: YARN-4817. Change Log Level to DEBUG for putDomain call in ATS 1.5. (Xuan Gong via gtcarrera9)

2016-03-15 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 3a0b69544 -> 5176a6986


YARN-4817. Change Log Level to DEBUG for putDomain call in ATS 1.5. (Xuan Gong 
via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5176a698
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5176a698
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5176a698

Branch: refs/heads/trunk
Commit: 5176a69868484f864c977a6cd13923167082407c
Parents: 3a0b695
Author: Li Lu <gtcarre...@apache.org>
Authored: Tue Mar 15 11:47:39 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Tue Mar 15 11:47:39 2016 -0700

--
 .../hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java  | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5176a698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 44cad60..c494f65 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -306,8 +306,10 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 Path domainLogPath =
 new Path(createAttemptDir(appAttemptId), DOMAIN_LOG_PREFIX
 + appAttemptId.toString());
-LOG.info("Writing domains for " + appAttemptId.toString() + " to "
-+ domainLogPath);
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Writing domains for " + appAttemptId.toString() + " to "
+  + domainLogPath);
+}
 this.logFDsCache.writeDomainLog(
 fs, domainLogPath, objMapper, domain, isAppendSupported);
   }

hadoop git commit: YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. (Xuan Gong via gtcarrera9)

2016-03-15 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 fce39557d -> 0ebc94714


YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. 
(Xuan Gong via gtcarrera9)

(cherry picked from commit 3a0b69544183f71c4683bc477656473494fc09d0)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ebc9471
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ebc9471
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ebc9471

Branch: refs/heads/branch-2.8
Commit: 0ebc9471404c9e8b1ded7521e052727f519b0008
Parents: fce3955
Author: Li Lu <gtcarre...@apache.org>
Authored: Tue Mar 15 11:40:54 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Tue Mar 15 11:43:39 2016 -0700

--
 .../hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ebc9471/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 9e719b7..44cad60 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -276,6 +276,7 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 mapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector());
 mapper.setSerializationInclusion(Inclusion.NON_NULL);
 mapper.configure(Feature.CLOSE_CLOSEABLE, false);
+mapper.configure(Feature.FLUSH_AFTER_WRITE_VALUE, false);
 return mapper;
   }

hadoop git commit: YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. (Xuan Gong via gtcarrera9)

2016-03-15 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 22ca176df -> 3a0b69544


YARN-4814. ATS 1.5 timelineclient impl call flush after every event write. 
(Xuan Gong via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a0b6954
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a0b6954
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a0b6954

Branch: refs/heads/trunk
Commit: 3a0b69544183f71c4683bc477656473494fc09d0
Parents: 22ca176
Author: Li Lu <gtcarre...@apache.org>
Authored: Tue Mar 15 11:40:54 2016 -0700
Committer: Li Lu <gtcarre...@apache.org>
Committed: Tue Mar 15 11:40:54 2016 -0700

--
 .../hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a0b6954/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 9e719b7..44cad60 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -276,6 +276,7 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 mapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector());
 mapper.setSerializationInclusion(Inclusion.NON_NULL);
 mapper.configure(Feature.CLOSE_CLOSEABLE, false);
+mapper.configure(Feature.FLUSH_AFTER_WRITE_VALUE, false);
 return mapper;
   }

hadoop git commit: YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test setup, and concurrency.

2016-03-10 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 8e89bb9d3 -> 92e86ff0c


YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test 
setup, and concurrency.

This commit amends commit d49cfb350454c2dfa2f3eb70f79b6d5030ce7bec with a 
missed test file.

(cherry picked from commit 017d2c127b9cbd75d3e31467172ed832f27ef826)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92e86ff0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92e86ff0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92e86ff0

Branch: refs/heads/branch-2.8
Commit: 92e86ff0cb2c8438553a318b6534fee5b9e634aa
Parents: 8e89bb9
Author: Li Lu 
Authored: Thu Mar 10 13:02:28 2016 -0800
Committer: Li Lu 
Committed: Thu Mar 10 13:06:02 2016 -0800

--
 .../TestOverrideTimelineStoreYarnClient.java| 56 
 1 file changed, 56 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/92e86ff0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
new file mode 100644
index 000..c190266
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class TestOverrideTimelineStoreYarnClient {
+
+  @Test
+  public void testLifecycleAndOverride() throws Throwable {
+YarnConfiguration conf = new YarnConfiguration();
+try(NoRMStore store = new NoRMStore()) {
+  store.init(conf);
+  store.start();
+  Assert.assertEquals(EntityGroupFSTimelineStore.AppState.ACTIVE,
+  store.getAppState(ApplicationId.newInstance(1, 1)));
+  store.stop();
+}
+  }
+
+  private static class NoRMStore extends EntityGroupFSTimelineStore {
+@Override
+protected YarnClient createAndInitYarnClient(Configuration conf) {
+  return null;
+}
+
+@Override
+protected AppState getAppState(ApplicationId appId)
+throws IOException {
+  return AppState.ACTIVE;
+}
+  }
+}

hadoop git commit: YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test setup, and concurrency.

2016-03-10 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 92125e605 -> 2b16a54fb


YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test 
setup, and concurrency.

This commit amends commit d49cfb350454c2dfa2f3eb70f79b6d5030ce7bec with a 
missed test file.

(cherry picked from commit 017d2c127b9cbd75d3e31467172ed832f27ef826)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b16a54f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b16a54f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b16a54f

Branch: refs/heads/branch-2
Commit: 2b16a54fbef2893245f392ee058f10c2845f1dc1
Parents: 92125e6
Author: Li Lu 
Authored: Thu Mar 10 13:02:28 2016 -0800
Committer: Li Lu 
Committed: Thu Mar 10 13:04:57 2016 -0800

--
 .../TestOverrideTimelineStoreYarnClient.java| 56 
 1 file changed, 56 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b16a54f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
new file mode 100644
index 000..c190266
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class TestOverrideTimelineStoreYarnClient {
+
+  @Test
+  public void testLifecycleAndOverride() throws Throwable {
+YarnConfiguration conf = new YarnConfiguration();
+try(NoRMStore store = new NoRMStore()) {
+  store.init(conf);
+  store.start();
+  Assert.assertEquals(EntityGroupFSTimelineStore.AppState.ACTIVE,
+  store.getAppState(ApplicationId.newInstance(1, 1)));
+  store.stop();
+}
+  }
+
+  private static class NoRMStore extends EntityGroupFSTimelineStore {
+@Override
+protected YarnClient createAndInitYarnClient(Configuration conf) {
+  return null;
+}
+
+@Override
+protected AppState getAppState(ApplicationId appId)
+throws IOException {
+  return AppState.ACTIVE;
+}
+  }
+}

hadoop git commit: YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test setup, and concurrency.

2016-03-10 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 9a79b738c -> 017d2c127


YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test 
setup, and concurrency.

This commit amends commit d49cfb350454c2dfa2f3eb70f79b6d5030ce7bec with a 
missed test file.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/017d2c12
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/017d2c12
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/017d2c12

Branch: refs/heads/trunk
Commit: 017d2c127b9cbd75d3e31467172ed832f27ef826
Parents: 9a79b73
Author: Li Lu 
Authored: Thu Mar 10 13:02:28 2016 -0800
Committer: Li Lu 
Committed: Thu Mar 10 13:02:28 2016 -0800

--
 .../TestOverrideTimelineStoreYarnClient.java| 56 
 1 file changed, 56 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/017d2c12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
new file mode 100644
index 000..c190266
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class TestOverrideTimelineStoreYarnClient {
+
+  @Test
+  public void testLifecycleAndOverride() throws Throwable {
+YarnConfiguration conf = new YarnConfiguration();
+try(NoRMStore store = new NoRMStore()) {
+  store.init(conf);
+  store.start();
+  Assert.assertEquals(EntityGroupFSTimelineStore.AppState.ACTIVE,
+  store.getAppState(ApplicationId.newInstance(1, 1)));
+  store.stop();
+}
+  }
+
+  private static class NoRMStore extends EntityGroupFSTimelineStore {
+@Override
+protected YarnClient createAndInitYarnClient(Configuration conf) {
+  return null;
+}
+
+@Override
+protected AppState getAppState(ApplicationId appId)
+throws IOException {
+  return AppState.ACTIVE;
+}
+  }
+}

hadoop git commit: HADOOP-12906. AuthenticatedURL should convert a 404/Not Found into an FileNotFoundException. (Steve Loughran via gtcarrera9)

2016-03-10 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 76ef097fd -> 92125e605


HADOOP-12906. AuthenticatedURL should convert a 404/Not Found into an 
FileNotFoundException. (Steve Loughran via gtcarrera9)

(cherry-picked from commit 9a79b738c582bd84727831987b845535625d75fe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92125e60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92125e60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92125e60

Branch: refs/heads/branch-2
Commit: 92125e60579a4e02e18aa11b2325328c34a3ae29
Parents: 76ef097
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Mar 10 11:38:31 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Mar 10 11:39:44 2016 -0800

--
 .../security/authentication/client/AuthenticatedURL.java  | 10 --
 1 file changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/92125e60/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
--
diff --git 
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
 
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
index c50a516..f87d9d8 100644
--- 
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
+++ 
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
@@ -15,6 +15,7 @@ package org.apache.hadoop.security.authentication.client;
 
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
@@ -269,10 +270,15 @@ public class AuthenticatedURL {
   }
 }
   }
+} else if (respCode == HttpURLConnection.HTTP_NOT_FOUND) {
+  token.set(null);
+  throw new FileNotFoundException(conn.getURL().toString());
 } else {
   token.set(null);
-  throw new AuthenticationException("Authentication failed, status: " + 
conn.getResponseCode() +
-", message: " + 
conn.getResponseMessage());
+  throw new AuthenticationException("Authentication failed" +
+  ", URL: " + conn.getURL() +
+  ", status: " + conn.getResponseCode() +
+  ", message: " + conn.getResponseMessage());
 }
   }

hadoop git commit: HADOOP-12906. AuthenticatedURL should convert a 404/Not Found into an FileNotFoundException. (Steve Loughran via gtcarrera9)

2016-03-10 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk d49cfb350 -> 9a79b738c


HADOOP-12906. AuthenticatedURL should convert a 404/Not Found into an 
FileNotFoundException. (Steve Loughran via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a79b738
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a79b738
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a79b738

Branch: refs/heads/trunk
Commit: 9a79b738c582bd84727831987b845535625d75fe
Parents: d49cfb3
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Mar 10 11:38:31 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Mar 10 11:38:31 2016 -0800

--
 .../security/authentication/client/AuthenticatedURL.java  | 10 --
 1 file changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a79b738/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
--
diff --git 
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
 
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
index c50a516..f87d9d8 100644
--- 
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
+++ 
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
@@ -15,6 +15,7 @@ package org.apache.hadoop.security.authentication.client;
 
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
@@ -269,10 +270,15 @@ public class AuthenticatedURL {
   }
 }
   }
+} else if (respCode == HttpURLConnection.HTTP_NOT_FOUND) {
+  token.set(null);
+  throw new FileNotFoundException(conn.getURL().toString());
 } else {
   token.set(null);
-  throw new AuthenticationException("Authentication failed, status: " + 
conn.getResponseCode() +
-", message: " + 
conn.getResponseMessage());
+  throw new AuthenticationException("Authentication failed" +
+  ", URL: " + conn.getURL() +
+  ", status: " + conn.getResponseCode() +
+  ", message: " + conn.getResponseMessage());
 }
   }

hadoop git commit: YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test setup, and concurrency. (Steve Loughran via gtcarrera9)

2016-03-10 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 a7164e90b -> 8e89bb9d3


YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test 
setup, and concurrency. (Steve Loughran via gtcarrera9)

(cherry-picked from commit d49cfb350454c2dfa2f3eb70f79b6d5030ce7bec)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e89bb9d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e89bb9d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e89bb9d

Branch: refs/heads/branch-2.8
Commit: 8e89bb9d380f25ca18bf0bed5e86a3e139977e36
Parents: a7164e9
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Mar 10 10:51:55 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Mar 10 10:58:51 2016 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   6 +
 .../hadoop/yarn/client/api/TimelineClient.java  |   4 +-
 .../api/impl/FileSystemTimelineWriter.java  |  51 ++---
 .../client/api/impl/TimelineClientImpl.java |  13 ++
 .../yarn/client/api/impl/TimelineWriter.java|  40 +++-
 .../timeline/webapp/TimelineWebServices.java|  12 +-
 .../yarn/server/timeline/EntityCacheItem.java   |  40 ++--
 .../timeline/EntityGroupFSTimelineStore.java| 204 ++-
 .../hadoop/yarn/server/timeline/LogInfo.java|  11 +-
 .../TestEntityGroupFSTimelineStore.java |   8 +-
 10 files changed, 279 insertions(+), 110 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e89bb9d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 52aa953..5e1c6fa 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1679,6 +1679,12 @@ public class YarnConfiguration extends Configuration {
   public static final long
   TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT = 7 * 60;
 
+  // This is temporary solution. The configuration will be deleted once we have
+  // the FileSystem API to check whether append operation is supported or not.
+  public static final String TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND
+  = TIMELINE_SERVICE_PREFIX
+  + "entity-file.fs-support-append";
+
   // mark app-history related configs @Private as application history is going
   // to be integrated into the timeline service
   @Private

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e89bb9d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
index 258b9f5..09298b5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.client.api;
 
+import java.io.Flushable;
 import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -41,7 +42,8 @@ import 
org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
  */
 @Public
 @Evolving
-public abstract class TimelineClient extends AbstractService {
+public abstract class TimelineClient extends AbstractService implements
+Flushable {
 
   /**
* Create a timeline client. The current UGI when the user initialize the

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e89bb9d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index aa1f1f8..9e719b7 100644
--- 
a/hadoop-yarn-project/hadoo

hadoop git commit: YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test setup, and concurrency. (Steve Loughran via gtcarrera9)

2016-03-10 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 2e32aa547 -> 76ef097fd


YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test 
setup, and concurrency. (Steve Loughran via gtcarrera9)

(cherry-picked from commit d49cfb350454c2dfa2f3eb70f79b6d5030ce7bec)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76ef097f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76ef097f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76ef097f

Branch: refs/heads/branch-2
Commit: 76ef097fd10af7bf4b876bcfec775f8049ac2034
Parents: 2e32aa5
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Mar 10 10:51:55 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Mar 10 10:56:51 2016 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   6 +
 .../hadoop/yarn/client/api/TimelineClient.java  |   4 +-
 .../api/impl/FileSystemTimelineWriter.java  |  51 ++---
 .../client/api/impl/TimelineClientImpl.java |  13 ++
 .../yarn/client/api/impl/TimelineWriter.java|  40 +++-
 .../timeline/webapp/TimelineWebServices.java|  12 +-
 .../yarn/server/timeline/EntityCacheItem.java   |  40 ++--
 .../timeline/EntityGroupFSTimelineStore.java| 204 ++-
 .../hadoop/yarn/server/timeline/LogInfo.java|  11 +-
 .../TestEntityGroupFSTimelineStore.java |   8 +-
 10 files changed, 279 insertions(+), 110 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ef097f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 4573404..174d0c4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1747,6 +1747,12 @@ public class YarnConfiguration extends Configuration {
   public static final long
   TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT = 7 * 60;
 
+  // This is temporary solution. The configuration will be deleted once we have
+  // the FileSystem API to check whether append operation is supported or not.
+  public static final String TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND
+  = TIMELINE_SERVICE_PREFIX
+  + "entity-file.fs-support-append";
+
   // mark app-history related configs @Private as application history is going
   // to be integrated into the timeline service
   @Private

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ef097f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
index 258b9f5..09298b5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.client.api;
 
+import java.io.Flushable;
 import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -41,7 +42,8 @@ import 
org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
  */
 @Public
 @Evolving
-public abstract class TimelineClient extends AbstractService {
+public abstract class TimelineClient extends AbstractService implements
+Flushable {
 
   /**
* Create a timeline client. The current UGI when the user initialize the

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ef097f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index aa1f1f8..9e719b7 100644
--- 
a/hadoop-yarn-project/hadoop-ya

hadoop git commit: YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test setup, and concurrency. (Steve Loughran via gtcarrera9)

2016-03-10 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 318c9b68b -> d49cfb350


YARN-4696. Improving EntityGroupFSTimelineStore on exception handling, test 
setup, and concurrency. (Steve Loughran via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d49cfb35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d49cfb35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d49cfb35

Branch: refs/heads/trunk
Commit: d49cfb350454c2dfa2f3eb70f79b6d5030ce7bec
Parents: 318c9b6
Author: Li Lu <gtcarre...@apache.org>
Authored: Thu Mar 10 10:51:55 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Thu Mar 10 10:51:55 2016 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   6 +
 .../hadoop/yarn/client/api/TimelineClient.java  |   4 +-
 .../api/impl/FileSystemTimelineWriter.java  |  51 ++---
 .../client/api/impl/TimelineClientImpl.java |  13 ++
 .../yarn/client/api/impl/TimelineWriter.java|  40 +++-
 .../timeline/webapp/TimelineWebServices.java|  12 +-
 .../yarn/server/timeline/EntityCacheItem.java   |  40 ++--
 .../timeline/EntityGroupFSTimelineStore.java| 204 ++-
 .../hadoop/yarn/server/timeline/LogInfo.java|  11 +-
 .../TestEntityGroupFSTimelineStore.java |   8 +-
 10 files changed, 279 insertions(+), 110 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d49cfb35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 61d1d72..ff4b493 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1747,6 +1747,12 @@ public class YarnConfiguration extends Configuration {
   public static final long
   TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT = 7 * 60;
 
+  // This is temporary solution. The configuration will be deleted once we have
+  // the FileSystem API to check whether append operation is supported or not.
+  public static final String TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND
+  = TIMELINE_SERVICE_PREFIX
+  + "entity-file.fs-support-append";
+
   // mark app-history related configs @Private as application history is going
   // to be integrated into the timeline service
   @Private

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d49cfb35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
index 258b9f5..09298b5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.client.api;
 
+import java.io.Flushable;
 import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -41,7 +42,8 @@ import 
org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
  */
 @Public
 @Evolving
-public abstract class TimelineClient extends AbstractService {
+public abstract class TimelineClient extends AbstractService implements
+Flushable {
 
   /**
* Create a timeline client. The current UGI when the user initialize the

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d49cfb35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index aa1f1f8..9e719b7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/File

hadoop git commit: HADOOP-12831. LocalFS/FSOutputSummer NPEs in constructor if bytes per checksum set to 0 (Mingliang Liu via gtcarrera9)

2016-02-27 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 621800859 -> 8324199e5


HADOOP-12831. LocalFS/FSOutputSummer NPEs in constructor if bytes per checksum 
set to 0 (Mingliang Liu via gtcarrera9)

(cherry-picked from commit 7545ce6636066a05763744a817878e03ee87f456)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8324199e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8324199e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8324199e

Branch: refs/heads/branch-2.8
Commit: 8324199e5c2c9393b44ff2040f7f1f6f2df3d452
Parents: 6218008
Author: Li Lu <gtcarre...@apache.org>
Authored: Sat Feb 27 21:59:03 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Sat Feb 27 22:04:23 2016 -0800

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../apache/hadoop/fs/ChecksumFileSystem.java|  4 
 .../hadoop/fs/TestChecksumFileSystem.java   | 23 
 3 files changed, 30 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8324199e/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6b2c2ee..8952d51 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -55,6 +55,9 @@ Release 2.8.0 - UNRELEASED
 (Larry McCay via cnauroth)
 
   IMPROVEMENTS
+
+HADOOP-12831. LocalFS/FSOutputSummer NPEs in constructor if bytes per 
checksum 
+set to 0 (Mingliang Liu via gtcarrera9)
 
 HADOOP-12458. Retries is typoed to spell Retires in parts of
 hadoop-yarn and hadoop-common

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8324199e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index dddf0ce..0e78b65 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -25,6 +25,7 @@ import java.io.InputStream;
 import java.nio.channels.ClosedChannelException;
 import java.util.Arrays;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -61,6 +62,9 @@ public abstract class ChecksumFileSystem extends 
FilterFileSystem {
 if (conf != null) {
   bytesPerChecksum = 
conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY,
 
LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_DEFAULT);
+  Preconditions.checkState(bytesPerChecksum > 0,
+  "bytes per checksum should be positive but was %s",
+  bytesPerChecksum);
 }
   }
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8324199e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
index 0c24ad5..923d219 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
@@ -228,6 +228,29 @@ public class TestChecksumFileSystem {
   }
 
 
+  @Test
+  public void testSetConf() {
+Configuration conf = new Configuration();
+
+conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, 0);
+try {
+  localFs.setConf(conf);
+  fail("Should have failed because zero bytes per checksum is invalid");
+} catch (IllegalStateException ignored) {
+}
+
+conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, -1);
+try {
+  localFs.setConf(conf);
+  fail("Should have failed because negative bytes per checksum is 
invalid");
+} catch (IllegalStateException ignored) {
+}
+
+conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, 
512);
+localFs.setConf(conf);
+
+  }
+
   void verifyRename(Path srcPath, Path ds

hadoop git commit: HADOOP-12831. LocalFS/FSOutputSummer NPEs in constructor if bytes per checksum set to 0 (Mingliang Liu via gtcarrera9)

2016-02-27 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 4d870b1fa -> 090f74c74


HADOOP-12831. LocalFS/FSOutputSummer NPEs in constructor if bytes per checksum 
set to 0 (Mingliang Liu via gtcarrera9)

(cherry-picked from commit 7545ce6636066a05763744a817878e03ee87f456)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/090f74c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/090f74c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/090f74c7

Branch: refs/heads/branch-2
Commit: 090f74c741eee5b15b76235448a4f9f3197822cb
Parents: 4d870b1
Author: Li Lu <gtcarre...@apache.org>
Authored: Sat Feb 27 21:59:03 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Sat Feb 27 22:03:49 2016 -0800

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../apache/hadoop/fs/ChecksumFileSystem.java|  4 
 .../hadoop/fs/TestChecksumFileSystem.java   | 23 
 3 files changed, 30 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/090f74c7/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2ffe9b7..d47d4f2 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -126,6 +126,9 @@ Release 2.8.0 - UNRELEASED
 (Larry McCay via cnauroth)
 
   IMPROVEMENTS
+
+HADOOP-12831. LocalFS/FSOutputSummer NPEs in constructor if bytes per 
checksum 
+set to 0 (Mingliang Liu via gtcarrera9)
 
 HADOOP-12458. Retries is typoed to spell Retires in parts of
 hadoop-yarn and hadoop-common

http://git-wip-us.apache.org/repos/asf/hadoop/blob/090f74c7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index dddf0ce..0e78b65 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -25,6 +25,7 @@ import java.io.InputStream;
 import java.nio.channels.ClosedChannelException;
 import java.util.Arrays;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -61,6 +62,9 @@ public abstract class ChecksumFileSystem extends 
FilterFileSystem {
 if (conf != null) {
   bytesPerChecksum = 
conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY,
 
LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_DEFAULT);
+  Preconditions.checkState(bytesPerChecksum > 0,
+  "bytes per checksum should be positive but was %s",
+  bytesPerChecksum);
 }
   }
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/090f74c7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
index 0c24ad5..923d219 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
@@ -228,6 +228,29 @@ public class TestChecksumFileSystem {
   }
 
 
+  @Test
+  public void testSetConf() {
+Configuration conf = new Configuration();
+
+conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, 0);
+try {
+  localFs.setConf(conf);
+  fail("Should have failed because zero bytes per checksum is invalid");
+} catch (IllegalStateException ignored) {
+}
+
+conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, -1);
+try {
+  localFs.setConf(conf);
+  fail("Should have failed because negative bytes per checksum is 
invalid");
+} catch (IllegalStateException ignored) {
+}
+
+conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, 
512);
+localFs.setConf(conf);
+
+  }
+
   void verifyRename(Path srcPath, Path ds

hadoop git commit: HADOOP-12831. LocalFS/FSOutputSummer NPEs in constructor if bytes per checksum set to 0 (Mingliang Liu via gtcarrera9)

2016-02-27 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 03cfb454f -> 7545ce663


HADOOP-12831. LocalFS/FSOutputSummer NPEs in constructor if bytes per checksum 
set to 0 (Mingliang Liu via gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7545ce66
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7545ce66
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7545ce66

Branch: refs/heads/trunk
Commit: 7545ce6636066a05763744a817878e03ee87f456
Parents: 03cfb45
Author: Li Lu <gtcarre...@apache.org>
Authored: Sat Feb 27 21:59:03 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Sat Feb 27 21:59:03 2016 -0800

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../apache/hadoop/fs/ChecksumFileSystem.java|  4 
 .../hadoop/fs/TestChecksumFileSystem.java   | 23 
 3 files changed, 30 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7545ce66/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index b8ae8df..9f95221 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -780,6 +780,9 @@ Release 2.8.0 - UNRELEASED
 (Larry McCay via cnauroth)
 
   IMPROVEMENTS
+
+HADOOP-12831. LocalFS/FSOutputSummer NPEs in constructor if bytes per 
checksum 
+set to 0 (Mingliang Liu via gtcarrera9)
 
 HADOOP-12458. Retries is typoed to spell Retires in parts of
 hadoop-yarn and hadoop-common

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7545ce66/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index 3b8ecea..c19be3d 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -25,6 +25,7 @@ import java.io.InputStream;
 import java.nio.channels.ClosedChannelException;
 import java.util.Arrays;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -61,6 +62,9 @@ public abstract class ChecksumFileSystem extends 
FilterFileSystem {
 if (conf != null) {
   bytesPerChecksum = 
conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY,
 
LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_DEFAULT);
+  Preconditions.checkState(bytesPerChecksum > 0,
+  "bytes per checksum should be positive but was %s",
+  bytesPerChecksum);
 }
   }
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7545ce66/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
index 0c24ad5..923d219 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
@@ -228,6 +228,29 @@ public class TestChecksumFileSystem {
   }
 
 
+  @Test
+  public void testSetConf() {
+Configuration conf = new Configuration();
+
+conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, 0);
+try {
+  localFs.setConf(conf);
+  fail("Should have failed because zero bytes per checksum is invalid");
+} catch (IllegalStateException ignored) {
+}
+
+conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, -1);
+try {
+  localFs.setConf(conf);
+  fail("Should have failed because negative bytes per checksum is 
invalid");
+} catch (IllegalStateException ignored) {
+}
+
+conf.setInt(LocalFileSystemConfigKeys.LOCAL_FS_BYTES_PER_CHECKSUM_KEY, 
512);
+localFs.setConf(conf);
+
+  }
+
   void verifyRename(Path srcPath, Path dstPath, boolean dstIsDir)
   throws Exception { 
 localFs.delete(srcPath,true);

hadoop git commit: YARN-4680. TimerTasks leak in ATS V1.5 Writer. (Xuan Gong via gtcarrera9)

2016-02-24 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 d4bbdd9d2 -> b22f4db3a


YARN-4680. TimerTasks leak in ATS V1.5 Writer. (Xuan Gong via
gtcarrera9)

(cherry picked from commit 9e0f7b8b69ead629f999aa86c8fb7eb581e175d8)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b22f4db3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b22f4db3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b22f4db3

Branch: refs/heads/branch-2.8
Commit: b22f4db3a0f44476b0a44b1b81f2d1dffcdf57fc
Parents: d4bbdd9
Author: Li Lu <gtcarre...@apache.org>
Authored: Wed Feb 24 13:43:09 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Wed Feb 24 14:08:28 2016 -0800

--
 hadoop-yarn-project/CHANGES.txt |   2 +
 .../hadoop/yarn/conf/YarnConfiguration.java |   6 +
 .../api/impl/FileSystemTimelineWriter.java  | 159 ---
 .../src/main/resources/yarn-default.xml |  39 +
 4 files changed, 180 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b22f4db3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 904cbcd..00f7aaa 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -647,6 +647,8 @@ Release 2.8.0 - UNRELEASED
 
   BUG FIXES
 
+YARN-4680. TimerTasks leak in ATS V1.5 Writer. (Xuan Gong via gtcarrera9)
+
 YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena 
 via devaraj)
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b22f4db3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index f844391..52aa953 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1673,6 +1673,12 @@ public class YarnConfiguration extends Configuration {
   public static final long TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS_DEFAULT =
   5*60;
 
+  public static final String
+  TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS =
+  TIMELINE_SERVICE_CLIENT_PREFIX + "internal-timers-ttl-secs";
+  public static final long
+  TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT = 7 * 60;
+
   // mark app-history related configs @Private as application history is going
   // to be integrated into the timeline service
   @Private

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b22f4db3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 1c295e1..aa1f1f8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -32,6 +32,9 @@ import java.util.Timer;
 import java.util.TimerTask;
 import java.util.Map.Entry;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -44,6 +47,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
@@ -154,8 +158,14 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 YarnConfiguration.TIMELINE_SERV

hadoop git commit: YARN-4680. TimerTasks leak in ATS V1.5 Writer. (Xuan Gong via gtcarrera9)

2016-02-24 Thread gtcarrera9 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 59cd23e27 -> 51162f962


YARN-4680. TimerTasks leak in ATS V1.5 Writer. (Xuan Gong via
gtcarrera9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51162f96
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51162f96
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51162f96

Branch: refs/heads/branch-2
Commit: 51162f9629c9f2cb1cddd4b050d21d4654110242
Parents: 59cd23e
Author: Li Lu <gtcarre...@apache.org>
Authored: Wed Feb 24 13:43:09 2016 -0800
Committer: Li Lu <gtcarre...@apache.org>
Committed: Wed Feb 24 13:52:11 2016 -0800

--
 hadoop-yarn-project/CHANGES.txt |   2 +
 .../hadoop/yarn/conf/YarnConfiguration.java |   6 +
 .../api/impl/FileSystemTimelineWriter.java  | 159 ---
 .../src/main/resources/yarn-default.xml |  39 +
 4 files changed, 180 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/51162f96/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a470ff1..4a53abb 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -816,6 +816,8 @@ Release 2.8.0 - UNRELEASED
 
   BUG FIXES
 
+YARN-4680. TimerTasks leak in ATS V1.5 Writer. (Xuan Gong via gtcarrera9)
+
 YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena 
 via devaraj)
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/51162f96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 92f84d2..3501586 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1729,6 +1729,12 @@ public class YarnConfiguration extends Configuration {
   public static final long TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS_DEFAULT =
   5*60;
 
+  public static final String
+  TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS =
+  TIMELINE_SERVICE_CLIENT_PREFIX + "internal-timers-ttl-secs";
+  public static final long
+  TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT = 7 * 60;
+
   // mark app-history related configs @Private as application history is going
   // to be integrated into the timeline service
   @Private

http://git-wip-us.apache.org/repos/asf/hadoop/blob/51162f96/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index 1c295e1..aa1f1f8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -32,6 +32,9 @@ import java.util.Timer;
 import java.util.TimerTask;
 import java.util.Map.Entry;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -44,6 +47,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
@@ -154,8 +158,14 @@ public class FileSystemTimelineWriter extends 
TimelineWriter{
 YarnConfiguration.TIMELINE_SERV

  1   2   >