hadoop git commit: HADOOP-14026. start-build-env.sh: invalid docker image name (Contributed by Gergő Pásztor via Daniel Templeton)

2017-03-04 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 2288857e3 -> c8b96296b


HADOOP-14026. start-build-env.sh: invalid docker image name (Contributed by 
Gergő Pásztor via Daniel Templeton)

(cherry picked from commit 6757329c43c1d6586c135b5aa00f144dd0d8d2f1)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8b96296
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8b96296
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8b96296

Branch: refs/heads/branch-2
Commit: c8b96296b26fab1b2637ee70676710c1560e96b5
Parents: 2288857
Author: Daniel Templeton 
Authored: Sat Mar 4 11:00:03 2017 -0800
Committer: Daniel Templeton 
Committed: Sat Mar 4 11:02:18 2017 -0800

--
 start-build-env.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8b96296/start-build-env.sh
--
diff --git a/start-build-env.sh b/start-build-env.sh
index 1fa084f..18e3a8c 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -31,7 +31,7 @@ else # boot2docker uid and gid
   GROUP_ID=50
 fi
 
-docker build -t "hadoop-build-${USER_NAME}" - <

hadoop git commit: HADOOP-14026. start-build-env.sh: invalid docker image name (Contributed by Gergő Pásztor via Daniel Templeton)

2017-03-04 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk c1386a90d -> c8bd8ac7f


HADOOP-14026. start-build-env.sh: invalid docker image name (Contributed by 
Gergő Pásztor via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8bd8ac7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8bd8ac7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8bd8ac7

Branch: refs/heads/trunk
Commit: c8bd8ac7f9642b8db5949a914f209d5de3d4ef9b
Parents: c1386a9
Author: Daniel Templeton 
Authored: Sat Mar 4 11:00:03 2017 -0800
Committer: Daniel Templeton 
Committed: Sat Mar 4 11:01:44 2017 -0800

--
 start-build-env.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8bd8ac7/start-build-env.sh
--
diff --git a/start-build-env.sh b/start-build-env.sh
index 1fa084f..18e3a8c 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -31,7 +31,7 @@ else # boot2docker uid and gid
   GROUP_ID=50
 fi
 
-docker build -t "hadoop-build-${USER_NAME}" - <

hadoop git commit: MAPREDUCE-6825. YARNRunner#createApplicationSubmissionContext method is longer than 150 lines (Contributed by Gergely Novák via Daniel Templeton)

2017-02-22 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 6c5fd8277 -> 4c883f331


MAPREDUCE-6825. YARNRunner#createApplicationSubmissionContext method is longer 
than 150 lines (Contributed by Gergely Novák via Daniel Templeton)

(cherry picked from commit 732ee6f0b58a12500198c0d934cc570c7490b520)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c883f33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c883f33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c883f33

Branch: refs/heads/branch-2
Commit: 4c883f331c0870e039d439186f9d6dfef0a0dca7
Parents: 6c5fd82
Author: Daniel Templeton 
Authored: Wed Feb 22 15:38:11 2017 -0800
Committer: Daniel Templeton 
Committed: Wed Feb 22 15:40:12 2017 -0800

--
 .../org/apache/hadoop/mapred/YARNRunner.java| 141 +++
 1 file changed, 86 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c883f33/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 98fe553..228c6af 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -291,8 +291,7 @@ public class YARNRunner implements ClientProtocol {
   throws IOException, InterruptedException {
 
 addHistoryToken(ts);
-
-// Construct necessary information to start the MR AM
+
 ApplicationSubmissionContext appContext =
   createApplicationSubmissionContext(conf, jobSubmitDir, ts);
 
@@ -331,34 +330,15 @@ public class YARNRunner implements ClientProtocol {
 return rsrc;
   }
 
-  public ApplicationSubmissionContext createApplicationSubmissionContext(
-  Configuration jobConf,
-  String jobSubmitDir, Credentials ts) throws IOException {
-ApplicationId applicationId = resMgrDelegate.getApplicationId();
-
-// Setup resource requirements
-Resource capability = recordFactory.newRecordInstance(Resource.class);
-capability.setMemorySize(
-conf.getInt(
-MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB
-)
-);
-capability.setVirtualCores(
-conf.getInt(
-MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES
-)
-);
-LOG.debug("AppMaster capability = " + capability);
-
-// Setup LocalResources
-Map localResources =
-new HashMap();
+  private Map setupLocalResources(Configuration jobConf,
+  String jobSubmitDir) throws IOException {
+Map localResources = new HashMap<>();
 
 Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JOB_CONF_FILE);
 
-URL yarnUrlForJobSubmitDir = 
URL.fromPath(defaultFileContext.getDefaultFileSystem()
-.resolvePath(
-defaultFileContext.makeQualified(new Path(jobSubmitDir;
+URL yarnUrlForJobSubmitDir = URL.fromPath(defaultFileContext
+.getDefaultFileSystem().resolvePath(
+defaultFileContext.makeQualified(new Path(jobSubmitDir;
 LOG.debug("Creating setup context, jobSubmitDir url is "
 + yarnUrlForJobSubmitDir);
 
@@ -371,7 +351,7 @@ public class YARNRunner implements ClientProtocol {
   FileContext.getFileContext(jobJarPath.toUri(), jobConf),
   jobJarPath,
   LocalResourceType.PATTERN);
-  String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, 
+  String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
   JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
   rc.setPattern(pattern);
   localResources.put(MRJobConfig.JOB_JAR, rc);
@@ -392,13 +372,11 @@ public class YARNRunner implements ClientProtocol {
   new Path(jobSubmitDir, s), LocalResourceType.FILE));
 }
 
-// Setup security tokens
-DataOutputBuffer dob = new DataOutputBuffer();
-ts.writeTokenStorageToStream(dob);
-ByteBuffer securityTokens  = ByteBuffer.wrap(dob.getData(), 0, 
dob.getLength());
+return localResources;
+  }
 
-// Setup the command to run the AM
-List vargs = new ArrayList(8);
+

hadoop git commit: MAPREDUCE-6825. YARNRunner#createApplicationSubmissionContext method is longer than 150 lines (Contributed by Gergely Novák via Daniel Templeton)

2017-02-22 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk d150f061f -> 732ee6f0b


MAPREDUCE-6825. YARNRunner#createApplicationSubmissionContext method is longer 
than 150 lines (Contributed by Gergely Novák via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/732ee6f0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/732ee6f0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/732ee6f0

Branch: refs/heads/trunk
Commit: 732ee6f0b58a12500198c0d934cc570c7490b520
Parents: d150f06
Author: Daniel Templeton 
Authored: Wed Feb 22 15:38:11 2017 -0800
Committer: Daniel Templeton 
Committed: Wed Feb 22 15:38:11 2017 -0800

--
 .../org/apache/hadoop/mapred/YARNRunner.java| 141 +++
 1 file changed, 86 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/732ee6f0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 98fe553..228c6af 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -291,8 +291,7 @@ public class YARNRunner implements ClientProtocol {
   throws IOException, InterruptedException {
 
 addHistoryToken(ts);
-
-// Construct necessary information to start the MR AM
+
 ApplicationSubmissionContext appContext =
   createApplicationSubmissionContext(conf, jobSubmitDir, ts);
 
@@ -331,34 +330,15 @@ public class YARNRunner implements ClientProtocol {
 return rsrc;
   }
 
-  public ApplicationSubmissionContext createApplicationSubmissionContext(
-  Configuration jobConf,
-  String jobSubmitDir, Credentials ts) throws IOException {
-ApplicationId applicationId = resMgrDelegate.getApplicationId();
-
-// Setup resource requirements
-Resource capability = recordFactory.newRecordInstance(Resource.class);
-capability.setMemorySize(
-conf.getInt(
-MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB
-)
-);
-capability.setVirtualCores(
-conf.getInt(
-MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES
-)
-);
-LOG.debug("AppMaster capability = " + capability);
-
-// Setup LocalResources
-Map localResources =
-new HashMap();
+  private Map setupLocalResources(Configuration jobConf,
+  String jobSubmitDir) throws IOException {
+Map localResources = new HashMap<>();
 
 Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JOB_CONF_FILE);
 
-URL yarnUrlForJobSubmitDir = 
URL.fromPath(defaultFileContext.getDefaultFileSystem()
-.resolvePath(
-defaultFileContext.makeQualified(new Path(jobSubmitDir;
+URL yarnUrlForJobSubmitDir = URL.fromPath(defaultFileContext
+.getDefaultFileSystem().resolvePath(
+defaultFileContext.makeQualified(new Path(jobSubmitDir;
 LOG.debug("Creating setup context, jobSubmitDir url is "
 + yarnUrlForJobSubmitDir);
 
@@ -371,7 +351,7 @@ public class YARNRunner implements ClientProtocol {
   FileContext.getFileContext(jobJarPath.toUri(), jobConf),
   jobJarPath,
   LocalResourceType.PATTERN);
-  String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, 
+  String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
   JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
   rc.setPattern(pattern);
   localResources.put(MRJobConfig.JOB_JAR, rc);
@@ -392,13 +372,11 @@ public class YARNRunner implements ClientProtocol {
   new Path(jobSubmitDir, s), LocalResourceType.FILE));
 }
 
-// Setup security tokens
-DataOutputBuffer dob = new DataOutputBuffer();
-ts.writeTokenStorageToStream(dob);
-ByteBuffer securityTokens  = ByteBuffer.wrap(dob.getData(), 0, 
dob.getLength());
+return localResources;
+  }
 
-// Setup the command to run the AM
-List vargs = new ArrayList(8);
+  private List setupAMCommand(Configuration jobConf) {
+List vargs = new

hadoop git commit: YARN-6125. The application attempt's diagnostic message should have a maximum size (Contributed by Andras Piros via Daniel Templeton)

2017-02-17 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 5575328bb -> 93219b645


YARN-6125. The application attempt's diagnostic message should have a maximum 
size
(Contributed by Andras Piros via Daniel Templeton)

(cherry picked from commit c7a36e613053ec8b46004b887c2f13535469)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93219b64
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93219b64
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93219b64

Branch: refs/heads/branch-2
Commit: 93219b64545a28749dc6d643de9a931b2e34dff0
Parents: 5575328
Author: Daniel Templeton 
Authored: Fri Feb 17 13:40:58 2017 -0800
Committer: Daniel Templeton 
Committed: Fri Feb 17 14:38:06 2017 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   5 +
 .../src/main/resources/yarn-default.xml |  15 ++
 .../hadoop-yarn-server-resourcemanager/pom.xml  |  16 +-
 .../rmapp/attempt/RMAppAttemptImpl.java | 172 ++-
 .../rmapp/attempt/TestBoundedAppender.java  | 116 +
 .../TestRMAppAttemptImplDiagnostics.java| 111 
 6 files changed, 422 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/93219b64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 02e5c99..6b4d0f4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2519,6 +2519,11 @@ public class YarnConfiguration extends Configuration {
 
   public static final int DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY = 0;
 
+  public static final String APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC =
+  YARN_PREFIX + "app.attempt.diagnostics.limit.kc";
+
+  public static final int DEFAULT_APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC = 64;
+
   @Private
   public static boolean isDistributedNodeLabelConfiguration(Configuration 
conf) {
 return DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE.equals(conf.get(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/93219b64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2c9c95e..9f2af10 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2964,4 +2964,19 @@
 3000
   
 
+  
+
+  Defines the limit of the diagnostics message of an application
+  attempt, in kilo characters (character count * 1024).
+  When using ZooKeeper to store application state behavior, it's
+  important to limit the size of the diagnostic messages to
+  prevent YARN from overwhelming ZooKeeper. In cases where
+  yarn.resourcemanager.state-store.max-completed-applications is set to
+  a large number, it may be desirable to reduce the value of this property
+  to limit the total data stored.
+
+yarn.app.attempt.diagnostics.limit.kc
+64
+  
+
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/93219b64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index 6338929..75ba54a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -53,8 +53,17 @@
   org.apache.hadoop
   hadoop-annotations
 
+
 
-  org.mockito
+  junit
+  junit
+  test
+
+
+org.mockito
   mockito-all
   test
 
@@ -74,11 +83,6 @@
   protobuf-java
 
 
-  junit
-  junit
-  test
-
-
   commons-io
   commons-io

hadoop git commit: YARN-6125. The application attempt's diagnostic message should have a maximum size (Contributed by Andras Piros via Daniel Templeton)

2017-02-17 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 9a9283778 -> c7a36e613


YARN-6125. The application attempt's diagnostic message should have a maximum 
size
(Contributed by Andras Piros via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7a36e61
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7a36e61
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7a36e61

Branch: refs/heads/trunk
Commit: c7a36e613053ec8b46004b887c2f13535469
Parents: 9a92837
Author: Daniel Templeton 
Authored: Fri Feb 17 13:40:58 2017 -0800
Committer: Daniel Templeton 
Committed: Fri Feb 17 13:40:58 2017 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   5 +
 .../src/main/resources/yarn-default.xml |  15 ++
 .../hadoop-yarn-server-resourcemanager/pom.xml  |  16 +-
 .../rmapp/attempt/RMAppAttemptImpl.java | 172 ++-
 .../rmapp/attempt/TestBoundedAppender.java  | 116 +
 .../TestRMAppAttemptImplDiagnostics.java| 111 
 6 files changed, 422 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a36e61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 136227a..094a424 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2619,6 +2619,11 @@ public class YarnConfiguration extends Configuration {
 
   public static final int DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY = 0;
 
+  public static final String APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC =
+  YARN_PREFIX + "app.attempt.diagnostics.limit.kc";
+
+  public static final int DEFAULT_APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC = 64;
+
   @Private
   public static boolean isDistributedNodeLabelConfiguration(Configuration 
conf) {
 return DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE.equals(conf.get(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a36e61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 4ca46f9..53beb5e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3022,4 +3022,19 @@
 3000
   
 
+  
+
+  Defines the limit of the diagnostics message of an application
+  attempt, in kilo characters (character count * 1024).
+  When using ZooKeeper to store application state behavior, it's
+  important to limit the size of the diagnostic messages to
+  prevent YARN from overwhelming ZooKeeper. In cases where
+  yarn.resourcemanager.state-store.max-completed-applications is set to
+  a large number, it may be desirable to reduce the value of this property
+  to limit the total data stored.
+
+yarn.app.attempt.diagnostics.limit.kc
+64
+  
+
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a36e61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index 6985d65..0a85d0c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -52,8 +52,17 @@
   org.apache.hadoop
   hadoop-annotations
 
+
 
-  org.mockito
+  junit
+  junit
+  test
+
+
+org.mockito
   mockito-all
   test
 
@@ -73,11 +82,6 @@
   protobuf-java
 
 
-  junit
-  junit
-  test
-
-
   commons-io
   commons-io

hadoop git commit: YARN-6188. Fix OOM issue with decommissioningNodesWatcher in the case of clusters with large number of nodes (Contributed by Ajay Jadhav via Daniel Templeton)

2017-02-17 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 06386b7e7 -> 10fd5627b


YARN-6188. Fix OOM issue with decommissioningNodesWatcher in the case of 
clusters with
large number of nodes (Contributed by Ajay Jadhav via Daniel Templeton)

(cherry picked from commit 585168cc1bdb87398d4f388f3eeccd1c71114982)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10fd5627
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10fd5627
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10fd5627

Branch: refs/heads/branch-2
Commit: 10fd5627bb53a16d0df30a53801c411d0f3c9823
Parents: 06386b7
Author: Daniel Templeton 
Authored: Fri Feb 17 13:11:43 2017 -0800
Committer: Daniel Templeton 
Committed: Fri Feb 17 13:14:37 2017 -0800

--
 .../yarn/server/resourcemanager/DecommissioningNodesWatcher.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/10fd5627/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
index 376b503..9631803 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
@@ -385,9 +385,9 @@ public class DecommissioningNodesWatcher {
 if (!LOG.isDebugEnabled() || decomNodes.size() == 0) {
   return;
 }
-StringBuilder sb = new StringBuilder();
 long now = mclock.getTime();
 for (DecommissioningNodeContext d : decomNodes.values()) {
+  StringBuilder sb = new StringBuilder();
   DecommissioningNodeStatus s = checkDecommissioningStatus(d.nodeId);
   sb.append(String.format(
   "%n  %-34s %4ds fresh:%3ds containers:%2d %14s",
@@ -413,8 +413,8 @@ public class DecommissioningNodesWatcher {
   (mclock.getTime() - rmApp.getStartTime()) / 1000));
 }
   }
+  LOG.debug("Decommissioning node: " + sb.toString());
 }
-LOG.info("Decommissioning Nodes: " + sb.toString());
   }
 
   // Read possible new DECOMMISSIONING_TIMEOUT_KEY from yarn-site.xml.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6188. Fix OOM issue with decommissioningNodesWatcher in the case of clusters with large number of nodes (Contributed by Ajay Jadhav via Daniel Templeton)

2017-02-17 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 4c26c241a -> 9a9283778


YARN-6188. Fix OOM issue with decommissioningNodesWatcher in the case of 
clusters with
large number of nodes (Contributed by Ajay Jadhav via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a928377
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a928377
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a928377

Branch: refs/heads/trunk
Commit: 9a928377868dfb2dc846c340501b3248eb6ad77f
Parents: 4c26c24
Author: Daniel Templeton 
Authored: Fri Feb 17 13:11:43 2017 -0800
Committer: Daniel Templeton 
Committed: Fri Feb 17 13:13:46 2017 -0800

--
 .../yarn/server/resourcemanager/DecommissioningNodesWatcher.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a928377/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
index 376b503..9631803 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
@@ -385,9 +385,9 @@ public class DecommissioningNodesWatcher {
 if (!LOG.isDebugEnabled() || decomNodes.size() == 0) {
   return;
 }
-StringBuilder sb = new StringBuilder();
 long now = mclock.getTime();
 for (DecommissioningNodeContext d : decomNodes.values()) {
+  StringBuilder sb = new StringBuilder();
   DecommissioningNodeStatus s = checkDecommissioningStatus(d.nodeId);
   sb.append(String.format(
   "%n  %-34s %4ds fresh:%3ds containers:%2d %14s",
@@ -413,8 +413,8 @@ public class DecommissioningNodesWatcher {
   (mclock.getTime() - rmApp.getStartTime()) / 1000));
 }
   }
+  LOG.debug("Decommissioning node: " + sb.toString());
 }
-LOG.info("Decommissioning Nodes: " + sb.toString());
   }
 
   // Read possible new DECOMMISSIONING_TIMEOUT_KEY from yarn-site.xml.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6166. Unnecessary INFO logs in AMRMClientAsyncImpl$CallbackHandlerThread.run (Contributed by Grant Whiteheart via Daniel Templeton)

2017-02-10 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 7b2b6caf1 -> 7c3578ca1


YARN-6166. Unnecessary INFO logs in 
AMRMClientAsyncImpl$CallbackHandlerThread.run (Contributed by Grant Whiteheart 
via Daniel Templeton)

(cherry picked from commit 1238fa71f8e62cecde6e2437d1b83ad3922295f8)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c3578ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c3578ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c3578ca

Branch: refs/heads/branch-2
Commit: 7c3578ca17fb4ff813f8c0c55f345e3e554f9be7
Parents: 7b2b6ca
Author: Daniel Templeton 
Authored: Fri Feb 10 09:44:49 2017 -0800
Committer: Daniel Templeton 
Committed: Fri Feb 10 09:48:50 2017 -0800

--
 .../hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c3578ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
index bba6993..d040345 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
@@ -310,7 +310,8 @@ extends AMRMClientAsync {
   try {
 object = responseQueue.take();
   } catch (InterruptedException ex) {
-LOG.info("Interrupted while waiting for queue", ex);
+LOG.debug("Interrupted while waiting for queue", ex);
+Thread.currentThread().interrupt();
 continue;
   }
   if (object instanceof Throwable) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6166. Unnecessary INFO logs in AMRMClientAsyncImpl$CallbackHandlerThread.run (Contributed by Grant Whiteheart via Daniel Templeton)

2017-02-10 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 3a0a0a485 -> 1238fa71f


YARN-6166. Unnecessary INFO logs in 
AMRMClientAsyncImpl$CallbackHandlerThread.run (Contributed by Grant Whiteheart 
via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1238fa71
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1238fa71
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1238fa71

Branch: refs/heads/trunk
Commit: 1238fa71f8e62cecde6e2437d1b83ad3922295f8
Parents: 3a0a0a4
Author: Daniel Templeton 
Authored: Fri Feb 10 09:44:49 2017 -0800
Committer: Daniel Templeton 
Committed: Fri Feb 10 09:46:11 2017 -0800

--
 .../hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1238fa71/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
index 3dd53d3..3e72d3f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
@@ -313,7 +313,8 @@ extends AMRMClientAsync {
   try {
 object = responseQueue.take();
   } catch (InterruptedException ex) {
-LOG.info("Interrupted while waiting for queue", ex);
+LOG.debug("Interrupted while waiting for queue", ex);
+Thread.currentThread().interrupt();
 continue;
   }
   if (object instanceof Throwable) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6135. Node manager REST API documentation is not up to date (Contributed by Miklos Szegedi via Daniel Templeton)

2017-02-07 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 307ec58ff -> a87e3850b


YARN-6135. Node manager REST API documentation is not up to date (Contributed 
by Miklos Szegedi via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a87e3850
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a87e3850
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a87e3850

Branch: refs/heads/trunk
Commit: a87e3850b9240900bca295ad9c4b2e49d0068e98
Parents: 307ec58
Author: Daniel Templeton 
Authored: Tue Feb 7 16:39:22 2017 -0800
Committer: Daniel Templeton 
Committed: Tue Feb 7 16:41:02 2017 -0800

--
 .../src/site/markdown/NodeManagerRest.md| 108 +--
 1 file changed, 75 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a87e3850/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md
index c7600f0..26b21be 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRest.md
@@ -64,7 +64,10 @@ Both of the following URI's give you the cluster information.
 | totalPmemAllocatedContainersMB | long | The amount of physical memory 
allocated for use by containers in MB |
 | totalVmemAllocatedContainersMB | long | The amount of virtual memory 
allocated for use by containers in MB |
 | totalVCoresAllocatedContainers | long | The number of virtual cores 
allocated for use by containers |
+| vmemCheckEnabled | boolean | Whether virtual memory checking is enabled for 
preemption |
+| pmemCheckEnabled | boolean | Whether physical memory checking is enabled for 
preemption |
 | lastNodeUpdateTime | long | The last timestamp at which the health report 
was received (in ms since epoch) |
+| nmStartupTime | long | The timestamp at which the node was started (in ms 
since epoch) |
 | healthReport | string | The diagnostic health report of the node |
 | nodeHealthy | boolean | true/false indicator of if the node is healthy |
 | nodeManagerVersion | string | Version of the NodeManager |
@@ -93,23 +96,27 @@ Response Body:
 
 ```json
 {
-   "nodeInfo" : {
-  "hadoopVersionBuiltOn" : "Mon Jan  9 14:58:42 UTC 2012",
-  "nodeManagerBuildVersion" : "0.23.1-SNAPSHOT from 1228355 by user1 
source checksum 20647f76c36430e888cc7204826a445c",
-  "lastNodeUpdateTime" : 132666126,
-  "totalVmemAllocatedContainersMB" : 17203,
-  "totalVCoresAllocatedContainers" : 8,
-  "nodeHealthy" : true,
-  "healthReport" : "",
-  "totalPmemAllocatedContainersMB" : 8192,
-  "nodeManagerVersionBuiltOn" : "Mon Jan  9 15:01:59 UTC 2012",
-  "nodeManagerVersion" : "0.23.1-SNAPSHOT",
-  "id" : "host.domain.com:8041",
-  "hadoopBuildVersion" : "0.23.1-SNAPSHOT from 1228292 by user1 source 
checksum 3eba233f2248a089e9b28841a784dd00",
-  "nodeHostName" : "host.domain.com",
-  "hadoopVersion" : "0.23.1-SNAPSHOT"
-   }
+  "nodeInfo": {
+"healthReport": "",
+"totalVmemAllocatedContainersMB": 17203,
+"totalPmemAllocatedContainersMB": 8192,
+"totalVCoresAllocatedContainers": 8,
+"vmemCheckEnabled": false,
+"pmemCheckEnabled": true,
+"lastNodeUpdateTime": 1485814574224,
+"nodeHealthy": true,
+"nodeManagerVersion": "3.0.0",
+"nodeManagerBuildVersion": "3.0.0",
+"nodeManagerVersionBuiltOn": "2017-01-30T17:42Z",
+"hadoopVersion": "3.0.0",
+"hadoopBuildVersion": "3.0.0",
+"hadoopVersionBuiltOn": "2017-01-30T17:39Z",
+"id": "host.domain.com:46077",
+"nodeHostName": "host.domain.com",
+"nmStartupTime": 1485800887841
+  }
 }
+
 ```
 
 **XML response**
@@ -131,20 +138,23 @@ Response Body:
 ```xml
 
 
-  
-  17203
-  8192
-  8
-  1326222386134
-  true
-  0.23.1-SNAPSHOT
-  0.23.1-SNAPSHOT from 1228355 by user1 source 
checksum 20647f76c36430e888cc7204826a445c
-  Mon Jan  9 15:01:59 UTC 
2012
-  0.23.1-SNAPSHOT
-  0.23.1-SNAPSHOT from 1228292 by user1 source checksum 
3eba233f2248a089e9b28841a784dd00
-  Mon Jan  9 14:58:42 UTC 2012
-  host.domain.com:8041
-  host.domain.com
+
+17203
+8192
+8
+false
+true
+1485815774203
+true
+3.0.0
+3.0.0
+2017-01-30T17:42Z
+3.0.0
+3.0.0
+2017-01-30T17:39Z
+host.domain.com:46077
+host.domain.com
+1485800887841
 
 ```
 
@@ -391,7 +401,13 @@ Response Body:
 "containerLogsLink" :

hadoop git commit: MAPREDUCE-6842. Update the links in PiEstimator document (Contributed by Jung Yoo via Daniel Templeton)

2017-02-07 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 ab41da7e6 -> 85261cbe0


MAPREDUCE-6842. Update the links in PiEstimator document (Contributed by Jung 
Yoo via Daniel Templeton)

(cherry picked from commit 0e58b368f001b3bb4031b923b63dce7c81d42d39)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85261cbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85261cbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85261cbe

Branch: refs/heads/branch-2
Commit: 85261cbe0effd34556ab985d8bee71aed19677fd
Parents: ab41da7
Author: Daniel Templeton 
Authored: Tue Feb 7 13:04:28 2017 -0800
Committer: Daniel Templeton 
Committed: Tue Feb 7 13:06:50 2017 -0800

--
 .../src/main/java/org/apache/hadoop/examples/pi/package.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/85261cbe/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
 
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
index 55913a9..ad06b0b 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
@@ -62,11 +62,11 @@ The table on the right are the results computed by distbbp.
 clusters spread over 20 days.
 The remaining 70% was finished over a weekend on Hammer,
 a 30,000-task-capacity cluster, which was also used for the
-http://developer.yahoo.net/blogs/hadoop/2009/05/hadoop_sorts_a_petabyte_in_162.html;>petabyte
 sort benchmark.
+http://yahoohadoop.tumblr.com/post/98338791001/hadoop_sorts_a_petabyte_in_162.html;>petabyte
 sort benchmark.
 The log files are available
 https://issues.apache.org/jira/secure/attachment/12408543/1e15log.zip;>here.
 The result was posted in
-http://developer.yahoo.net/blogs/hadoop/2009/05/hadoop_computes_the_10151st_bi.html;>this
 YDN blog.
+http://yahoohadoop.tumblr.com/post/98338598026/hadoop-computes-the-10-15-1st-bit-of-π;>this
 YDN blog.
 
 
 The second part of Row 15 (D3611)


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: MAPREDUCE-6842. Update the links in PiEstimator document (Contributed by Jung Yoo via Daniel Templeton)

2017-02-07 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 69b23632c -> deb368b1b


MAPREDUCE-6842. Update the links in PiEstimator document (Contributed by Jung 
Yoo via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/deb368b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/deb368b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/deb368b1

Branch: refs/heads/trunk
Commit: deb368b1b676363fd318c9de4fa53cd8700b220a
Parents: 69b2363
Author: Daniel Templeton 
Authored: Tue Feb 7 13:04:28 2017 -0800
Committer: Daniel Templeton 
Committed: Tue Feb 7 13:05:40 2017 -0800

--
 .../src/main/java/org/apache/hadoop/examples/pi/package.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/deb368b1/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
 
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
index 55913a9..ad06b0b 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/package.html
@@ -62,11 +62,11 @@ The table on the right are the results computed by distbbp.
 clusters spread over 20 days.
 The remaining 70% was finished over a weekend on Hammer,
 a 30,000-task-capacity cluster, which was also used for the
-http://developer.yahoo.net/blogs/hadoop/2009/05/hadoop_sorts_a_petabyte_in_162.html;>petabyte
 sort benchmark.
+http://yahoohadoop.tumblr.com/post/98338791001/hadoop_sorts_a_petabyte_in_162.html;>petabyte
 sort benchmark.
 The log files are available
 https://issues.apache.org/jira/secure/attachment/12408543/1e15log.zip;>here.
 The result was posted in
-http://developer.yahoo.net/blogs/hadoop/2009/05/hadoop_computes_the_10151st_bi.html;>this
 YDN blog.
+http://yahoohadoop.tumblr.com/post/98338598026/hadoop-computes-the-10-15-1st-bit-of-π;>this
 YDN blog.
 
 
 The second part of Row 15 (D3611)


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-4658. Typo in o.a.h.yarn.server.resourcemanager.scheduler.fair.TestFairScheduler comment (Contributed by Udai Potluri via Daniel Templeton)

2017-01-31 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 ccf33bc36 -> be14bda5c


YARN-4658. Typo in 
o.a.h.yarn.server.resourcemanager.scheduler.fair.TestFairScheduler comment 
(Contributed by Udai Potluri via Daniel Templeton)

(cherry picked from commit 3e06475307e30377092547dfdebe6c2c6ac6e78f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be14bda5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be14bda5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be14bda5

Branch: refs/heads/branch-2
Commit: be14bda5c0ba2aa241d485d4f0ce03d63eb17f11
Parents: ccf33bc
Author: Daniel Templeton 
Authored: Tue Jan 31 16:25:33 2017 -0800
Committer: Daniel Templeton 
Committed: Tue Jan 31 16:26:41 2017 -0800

--
 .../resourcemanager/scheduler/fair/TestContinuousScheduling.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/be14bda5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
index 4cc99b2..b87efc6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
@@ -260,7 +260,7 @@ public class TestContinuousScheduling extends 
FairSchedulerTestBase {
 FairScheduler spyScheduler = spy(scheduler);
 Assert.assertTrue("Continuous scheduling should be disabled.",
 !spyScheduler.isContinuousSchedulingEnabled());
-// Add one nodes
+// Add one node
 RMNode node1 =
 MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
 "127.0.0.1");


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-4658. Typo in o.a.h.yarn.server.resourcemanager.scheduler.fair.TestFairScheduler comment (Contributed by Udai Potluri via Daniel Templeton)

2017-01-31 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 258991dc5 -> 3e0647530


YARN-4658. Typo in 
o.a.h.yarn.server.resourcemanager.scheduler.fair.TestFairScheduler comment 
(Contributed by Udai Potluri via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e064753
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e064753
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e064753

Branch: refs/heads/trunk
Commit: 3e06475307e30377092547dfdebe6c2c6ac6e78f
Parents: 258991d
Author: Daniel Templeton 
Authored: Tue Jan 31 16:25:33 2017 -0800
Committer: Daniel Templeton 
Committed: Tue Jan 31 16:25:33 2017 -0800

--
 .../resourcemanager/scheduler/fair/TestContinuousScheduling.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e064753/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
index 8bb06e7..1ea0032 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
@@ -261,7 +261,7 @@ public class TestContinuousScheduling extends 
FairSchedulerTestBase {
 FairScheduler spyScheduler = spy(scheduler);
 Assert.assertTrue("Continuous scheduling should be disabled.",
 !spyScheduler.isContinuousSchedulingEnabled());
-// Add one nodes
+// Add one node
 RMNode node1 =
 MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
 "127.0.0.1");


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6103. Log updates for ZKRMStateStore (Contributed by Daniel Sturman via Daniel Templeton)

2017-01-30 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 9f0b81335 -> ccf33bc36


YARN-6103. Log updates for ZKRMStateStore (Contributed by Daniel Sturman via 
Daniel Templeton)

(cherry picked from commit 87852b6ef4b9d973b7b374d41c8860fb1495)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ccf33bc3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ccf33bc3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ccf33bc3

Branch: refs/heads/branch-2
Commit: ccf33bc36cc9d7bfc6ef54bf9129bd4390d9b7a4
Parents: 9f0b813
Author: Daniel Templeton 
Authored: Mon Jan 30 21:44:48 2017 -0800
Committer: Daniel Templeton 
Committed: Mon Jan 30 21:46:38 2017 -0800

--
 .../recovery/ZKRMStateStore.java| 27 
 1 file changed, 16 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccf33bc3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index dd59368..b9863f2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -611,8 +611,10 @@ public class ZKRMStateStore extends RMStateStore {
 } else {
   safeCreate(nodeUpdatePath, appStateData, zkAcl,
   CreateMode.PERSISTENT);
-  LOG.debug(appId + " znode didn't exist. Created a new znode to"
-  + " update the application state.");
+  if (LOG.isDebugEnabled()) {
+LOG.debug(appId + " znode didn't exist. Created a new znode to"
++ " update the application state.");
+  }
 }
   }
 
@@ -656,8 +658,10 @@ public class ZKRMStateStore extends RMStateStore {
 } else {
   safeCreate(nodeUpdatePath, attemptStateData, zkAcl,
   CreateMode.PERSISTENT);
-  LOG.debug(appAttemptId + " znode didn't exist. Created a new znode to"
-  + " update the application attempt state.");
+  if (LOG.isDebugEnabled()) {
+LOG.debug(appAttemptId + " znode didn't exist. Created a new znode to"
++ " update the application attempt state.");
+  }
 }
   }
 
@@ -737,7 +741,9 @@ public class ZKRMStateStore extends RMStateStore {
 } else {
   // in case znode doesn't exist
   addStoreOrUpdateOps(trx, rmDTIdentifier, renewDate, false);
-  LOG.debug("Attempted to update a non-existing znode " + nodeRemovePath);
+  if (LOG.isDebugEnabled()) {
+LOG.debug("Attempted to update a non-existing znode " + 
nodeRemovePath);
+  }
 }
 
 trx.commit();
@@ -754,12 +760,12 @@ public class ZKRMStateStore extends RMStateStore {
 ByteArrayOutputStream seqOs = new ByteArrayOutputStream();
 
 try (DataOutputStream seqOut = new DataOutputStream(seqOs)) {
-  if (LOG.isDebugEnabled()) {
-LOG.debug((isUpdate ? "Storing " : "Updating ") + "RMDelegationToken_"
-+ rmDTIdentifier.getSequenceNumber());
-  }
 
   if (isUpdate) {
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Updating RMDelegationToken_"
+  + rmDTIdentifier.getSequenceNumber());
+}
 trx.setData(nodeCreatePath, identifierData.toByteArray(), -1);
   } else {
 trx.create(nodeCreatePath, identifierData.toByteArray(), zkAcl,
@@ -768,8 +774,7 @@ public class ZKRMStateStore extends RMStateStore {
 seqOut.writeInt(rmDTIdentifier.getSequenceNumber());
 
 if (LOG.isDebugEnabled()) {
-  LOG.debug((isUpdate ? "Storing " : "Updating ")
-  + dtSequenceNumberPath + ". SequenceNumber: "
+  LOG.debug("Storing " + dtSequenceNumberPath + ". SequenceNumber: "
   + rmDTIdentifier.getSequenceNumber());
 }
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6103. Log updates for ZKRMStateStore (Contributed by Daniel Sturman via Daniel Templeton)

2017-01-30 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 11e44bdda -> 87852b6ef


YARN-6103. Log updates for ZKRMStateStore (Contributed by Daniel Sturman via 
Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87852b6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87852b6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87852b6e

Branch: refs/heads/trunk
Commit: 87852b6ef4b9d973b7b374d41c8860fb1495
Parents: 11e44bd
Author: Daniel Templeton 
Authored: Mon Jan 30 21:44:48 2017 -0800
Committer: Daniel Templeton 
Committed: Mon Jan 30 21:44:48 2017 -0800

--
 .../recovery/ZKRMStateStore.java| 27 
 1 file changed, 16 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/87852b6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index cf6380f..1212a91 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -610,8 +610,10 @@ public class ZKRMStateStore extends RMStateStore {
 } else {
   safeCreate(nodeUpdatePath, appStateData, zkAcl,
   CreateMode.PERSISTENT);
-  LOG.debug(appId + " znode didn't exist. Created a new znode to"
-  + " update the application state.");
+  if (LOG.isDebugEnabled()) {
+LOG.debug(appId + " znode didn't exist. Created a new znode to"
++ " update the application state.");
+  }
 }
   }
 
@@ -655,8 +657,10 @@ public class ZKRMStateStore extends RMStateStore {
 } else {
   safeCreate(nodeUpdatePath, attemptStateData, zkAcl,
   CreateMode.PERSISTENT);
-  LOG.debug(appAttemptId + " znode didn't exist. Created a new znode to"
-  + " update the application attempt state.");
+  if (LOG.isDebugEnabled()) {
+LOG.debug(appAttemptId + " znode didn't exist. Created a new znode to"
++ " update the application attempt state.");
+  }
 }
   }
 
@@ -736,7 +740,9 @@ public class ZKRMStateStore extends RMStateStore {
 } else {
   // in case znode doesn't exist
   addStoreOrUpdateOps(trx, rmDTIdentifier, renewDate, false);
-  LOG.debug("Attempted to update a non-existing znode " + nodeRemovePath);
+  if (LOG.isDebugEnabled()) {
+LOG.debug("Attempted to update a non-existing znode " + 
nodeRemovePath);
+  }
 }
 
 trx.commit();
@@ -753,12 +759,12 @@ public class ZKRMStateStore extends RMStateStore {
 ByteArrayOutputStream seqOs = new ByteArrayOutputStream();
 
 try (DataOutputStream seqOut = new DataOutputStream(seqOs)) {
-  if (LOG.isDebugEnabled()) {
-LOG.debug((isUpdate ? "Storing " : "Updating ") + "RMDelegationToken_"
-+ rmDTIdentifier.getSequenceNumber());
-  }
 
   if (isUpdate) {
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Updating RMDelegationToken_"
+  + rmDTIdentifier.getSequenceNumber());
+}
 trx.setData(nodeCreatePath, identifierData.toByteArray(), -1);
   } else {
 trx.create(nodeCreatePath, identifierData.toByteArray(), zkAcl,
@@ -767,8 +773,7 @@ public class ZKRMStateStore extends RMStateStore {
 seqOut.writeInt(rmDTIdentifier.getSequenceNumber());
 
 if (LOG.isDebugEnabled()) {
-  LOG.debug((isUpdate ? "Storing " : "Updating ")
-  + dtSequenceNumberPath + ". SequenceNumber: "
+  LOG.debug("Storing " + dtSequenceNumberPath + ". SequenceNumber: "
   + rmDTIdentifier.getSequenceNumber());
 }
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6000. Make AllocationFileLoaderService.Listener public. (Tao Jie via kasha)

2017-01-26 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 c4eeac02d -> 1b1a25cca


YARN-6000. Make AllocationFileLoaderService.Listener public. (Tao Jie via kasha)

(cherry picked from commit 4d3f73acc0a5cabc748132889dbe670bea178a3f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b1a25cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b1a25cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b1a25cc

Branch: refs/heads/branch-2
Commit: 1b1a25cca653d9a66904cbcd423fdf6994ead128
Parents: c4eeac0
Author: Karthik Kambatla 
Authored: Fri Dec 23 11:40:56 2016 -0800
Committer: Daniel Templeton 
Committed: Thu Jan 26 14:11:50 2017 -0800

--
 .../scheduler/fair/AllocationFileLoaderService.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b1a25cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index a655615..163a265 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -639,7 +639,7 @@ public class AllocationFileLoaderService extends 
AbstractService {
 return defaultPermissions;
   }
 
-  interface Listener {
+  public interface Listener {
 void onReload(AllocationConfiguration info) throws IOException;
   }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-4975. Fair Scheduler: exception thrown when a parent queue marked 'parent' has configured child queues (Contributed by Yufei Gu via Daniel Templeton)

2017-01-26 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 443f2803a -> c4eeac02d


YARN-4975. Fair Scheduler: exception thrown when a parent queue marked 'parent' 
has configured child queues
(Contributed by Yufei Gu via Daniel Templeton)

(cherry picked from commit f85b74ccf9f1c1c1444cc00750b03468cbf40fb9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4eeac02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4eeac02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4eeac02

Branch: refs/heads/branch-2
Commit: c4eeac02deb028437f19b2c4fe59de5bec5d1b13
Parents: 443f280
Author: Daniel Templeton 
Authored: Thu Jan 26 10:31:09 2017 -0800
Committer: Daniel Templeton 
Committed: Thu Jan 26 14:08:46 2017 -0800

--
 .../fair/AllocationFileLoaderService.java   | 26 +++---
 .../fair/TestAllocationFileLoaderService.java   | 88 
 2 files changed, 101 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4eeac02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index 3aecbfd..a655615 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -487,6 +487,7 @@ public class AllocationFileLoaderService extends 
AbstractService {
 Map racls = new HashMap<>();
 NodeList fields = element.getChildNodes();
 boolean isLeaf = true;
+boolean isReservable = false;
 
 for (int j = 0; j < fields.getLength(); j++) {
   Node fieldNode = fields.item(j);
@@ -558,7 +559,7 @@ public class AllocationFileLoaderService extends 
AbstractService {
 racls.put(ReservationACL.SUBMIT_RESERVATIONS,
 new AccessControlList(text));
   } else if ("reservation".equals(field.getTagName())) {
-isLeaf = false;
+isReservable = true;
 reservableQueues.add(queueName);
 configuredQueues.get(FSQueueType.PARENT).add(queueName);
   } else if ("allowPreemptionFrom".equals(field.getTagName())) {
@@ -577,22 +578,21 @@ public class AllocationFileLoaderService extends 
AbstractService {
 isLeaf = false;
   }
 }
-if (isLeaf) {
-  // if a leaf in the alloc file is marked as type='parent'
-  // then store it under 'parent'
-  if ("parent".equals(element.getAttribute("type"))) {
-configuredQueues.get(FSQueueType.PARENT).add(queueName);
-  } else {
-configuredQueues.get(FSQueueType.LEAF).add(queueName);
-  }
+
+// if a leaf in the alloc file is marked as type='parent'
+// then store it as a parent queue
+if (isLeaf && !"parent".equals(element.getAttribute("type"))) {
+  configuredQueues.get(FSQueueType.LEAF).add(queueName);
 } else {
-  if ("parent".equals(element.getAttribute("type"))) {
-throw new AllocationConfigurationException("Both  and " +
-"type=\"parent\" found for queue " + queueName + " which is " +
-"unsupported");
+  if (isReservable) {
+throw new AllocationConfigurationException("The configuration settings"
++ " for " + queueName + " are invalid. A queue element that "
++ "contains child queue elements or that has the type='parent' "
++ "attribute cannot also include a reservation element.");
   }
   configuredQueues.get(FSQueueType.PARENT).add(queueName);
 }
+
 // Set default acls if not defined
 // The root queue defaults to all access
 for (QueueACL acl : QueueACL.values()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4eeac02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java

hadoop git commit: YARN-4997. Update fair scheduler to use pluggable auth provider (Contributed by Tao Jie via Daniel Templeton)

2017-01-26 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 4a48f0e70 -> 443f2803a


YARN-4997. Update fair scheduler to use pluggable auth provider (Contributed by 
Tao Jie via Daniel Templeton)

(cherry picked from commit b3befc021b0e2d63d1a3710ea450797d1129f1f5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/443f2803
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/443f2803
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/443f2803

Branch: refs/heads/branch-2
Commit: 443f2803ac7fa7dd3b571578effc2846177c8bc3
Parents: 4a48f0e
Author: Daniel Templeton 
Authored: Wed Nov 30 09:50:33 2016 -0800
Committer: Daniel Templeton 
Committed: Thu Jan 26 14:07:46 2017 -0800

--
 .../security/YarnAuthorizationProvider.java | 15 +
 .../scheduler/fair/AllocationConfiguration.java | 38 +--
 .../fair/AllocationFileLoaderService.java   | 68 +---
 .../resourcemanager/scheduler/fair/FSQueue.java | 22 +--
 .../scheduler/fair/FairScheduler.java   | 45 +++--
 .../scheduler/fair/TestFairScheduler.java   |  4 +-
 6 files changed, 149 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/443f2803/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
index 4b43ea1..9ae4bd7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.util.List;
 
 /**
@@ -61,6 +62,20 @@ public abstract class YarnAuthorizationProvider {
   }
 
   /**
+   * Destroy the {@link YarnAuthorizationProvider} instance.
+   * This method is called only in Tests.
+   */
+  @VisibleForTesting
+  public static void destroy() {
+synchronized (YarnAuthorizationProvider.class) {
+  if (authorizer != null) {
+LOG.debug(authorizer.getClass().getName() + " is destroyed.");
+authorizer = null;
+  }
+}
+  }
+
+  /**
* Initialize the provider. Invoked on daemon startup. DefaultYarnAuthorizer 
is
* initialized based on configurations.
*/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/443f2803/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index 023d822..81233c8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -17,6 +17,7 @@
 */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -25,13 +26,14 @@ import java.util.Set;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.ReservationACL;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.security.AccessType;
 import

hadoop git commit: YARN-4975. Fair Scheduler: exception thrown when a parent queue marked 'parent' has configured child queues (Contributed by Yufei Gu via Daniel Templeton)

2017-01-26 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 7c1cc30b3 -> f85b74ccf


YARN-4975. Fair Scheduler: exception thrown when a parent queue marked 'parent' 
has configured child queues
(Contributed by Yufei Gu via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f85b74cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f85b74cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f85b74cc

Branch: refs/heads/trunk
Commit: f85b74ccf9f1c1c1444cc00750b03468cbf40fb9
Parents: 7c1cc30
Author: Daniel Templeton 
Authored: Thu Jan 26 10:31:09 2017 -0800
Committer: Daniel Templeton 
Committed: Thu Jan 26 10:31:09 2017 -0800

--
 .../fair/AllocationFileLoaderService.java   | 26 +++---
 .../fair/TestAllocationFileLoaderService.java   | 88 
 2 files changed, 101 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85b74cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index cd4a19b..163a265 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -487,6 +487,7 @@ public class AllocationFileLoaderService extends 
AbstractService {
 Map racls = new HashMap<>();
 NodeList fields = element.getChildNodes();
 boolean isLeaf = true;
+boolean isReservable = false;
 
 for (int j = 0; j < fields.getLength(); j++) {
   Node fieldNode = fields.item(j);
@@ -558,7 +559,7 @@ public class AllocationFileLoaderService extends 
AbstractService {
 racls.put(ReservationACL.SUBMIT_RESERVATIONS,
 new AccessControlList(text));
   } else if ("reservation".equals(field.getTagName())) {
-isLeaf = false;
+isReservable = true;
 reservableQueues.add(queueName);
 configuredQueues.get(FSQueueType.PARENT).add(queueName);
   } else if ("allowPreemptionFrom".equals(field.getTagName())) {
@@ -577,22 +578,21 @@ public class AllocationFileLoaderService extends 
AbstractService {
 isLeaf = false;
   }
 }
-if (isLeaf) {
-  // if a leaf in the alloc file is marked as type='parent'
-  // then store it under 'parent'
-  if ("parent".equals(element.getAttribute("type"))) {
-configuredQueues.get(FSQueueType.PARENT).add(queueName);
-  } else {
-configuredQueues.get(FSQueueType.LEAF).add(queueName);
-  }
+
+// if a leaf in the alloc file is marked as type='parent'
+// then store it as a parent queue
+if (isLeaf && !"parent".equals(element.getAttribute("type"))) {
+  configuredQueues.get(FSQueueType.LEAF).add(queueName);
 } else {
-  if ("parent".equals(element.getAttribute("type"))) {
-throw new AllocationConfigurationException("Both  and " +
-"type=\"parent\" found for queue " + queueName + " which is " +
-"unsupported");
+  if (isReservable) {
+throw new AllocationConfigurationException("The configuration settings"
++ " for " + queueName + " are invalid. A queue element that "
++ "contains child queue elements or that has the type='parent' "
++ "attribute cannot also include a reservation element.");
   }
   configuredQueues.get(FSQueueType.PARENT).add(queueName);
 }
+
 // Set default acls if not defined
 // The root queue defaults to all access
 for (QueueACL acl : QueueACL.values()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85b74cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
--
diff --git

hadoop git commit: MAPREDUCE-6808. Log map attempts as part of shuffle handler audit log (Contributed by Gergő Pásztor via Daniel Templeton)

2017-01-25 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 9e19f758c -> a7463b6c8


MAPREDUCE-6808. Log map attempts as part of shuffle handler audit log 
(Contributed by Gergő Pásztor via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7463b6c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7463b6c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7463b6c

Branch: refs/heads/trunk
Commit: a7463b6c88f698950a2f326030261001aa51b35e
Parents: 9e19f75
Author: Daniel Templeton 
Authored: Wed Jan 25 14:30:50 2017 -0800
Committer: Daniel Templeton 
Committed: Wed Jan 25 14:32:40 2017 -0800

--
 .../src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7463b6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 9547062..15a1b89 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -947,7 +947,7 @@ public class ShuffleHandler extends AuxiliaryService {
   // to turn it on please enable this audit log
   // on log4j.properties by uncommenting the setting
   if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) +
+AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
  " reducer " + reduceQ.get(0));
   }
   int reduceId;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6012. Remove node label (removeFromClusterNodeLabels) document is missing (Contributed by Ying Zhang via Daniel Templeton)

2017-01-23 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk ce832059d -> a2c50127d


YARN-6012. Remove node label (removeFromClusterNodeLabels) document is missing 
(Contributed by Ying Zhang via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2c50127
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2c50127
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2c50127

Branch: refs/heads/trunk
Commit: a2c50127d12b0aefcf1b050e0ab0d2df2abb30b3
Parents: ce83205
Author: Daniel Templeton 
Authored: Mon Jan 23 17:40:44 2017 -0800
Committer: Daniel Templeton 
Committed: Mon Jan 23 17:42:54 2017 -0800

--
 .../hadoop-yarn-site/src/site/markdown/NodeLabel.md  | 8 
 1 file changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c50127/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
index a87658d..73c9f41 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
@@ -20,6 +20,7 @@ YARN Node Labels
 * [Configuration](#Configuration)
 * [Setting up ResourceManager to enable Node 
Labels](#Setting_up_ResourceManager_to_enable_Node_Labels)
 * [Add/modify node labels list to 
YARN](#Add/modify_node_labels_list_to_YARN)
+* [Remove node labels from YARN](#Remove_node_labels_from_YARN)
 * [Add/modify node-to-labels mapping to 
YARN](#Add/modify_node-to-labels_mapping_to_YARN)
 * [Configuration of Schedulers for node 
labels](#Configuration_of_Schedulers_for_node_labels)
 * [Specifying node label for 
application](#Specifying_node_label_for_application)
@@ -86,6 +87,13 @@ Notes:
 * If user don’t specify “(exclusive=…)”, exclusive will be 
```true``` by default.
 * Run ```yarn cluster --list-node-labels``` to check added node labels are 
visible in the cluster.
 
+###Remove node labels from YARN
+
+* Remove cluster node labels:
+* To remove one or more node labels, execute the following command: 
```yarn rmadmin -removeFromClusterNodeLabels "[,,...]"```. The 
command argument should be a comma-separated list of node labels to remove.
+* It is not allowed to remove a label which has been associated with 
queues, i.e., one or more queues have access to this label.
+* To verify if specified node labels have been successfully removed, run 
```yarn cluster --list-node-labels```.
+
 ###Add/modify node-to-labels mapping to YARN
 
 * Configuring nodes to labels mapping in **Centralized** NodeLabel setup


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6028. Add document for container metrics (Contributed by Weiwei Yang via Daniel Templeton)

2017-01-20 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 bed700e98 -> be7e4a080


YARN-6028. Add document for container metrics (Contributed by Weiwei Yang via 
Daniel Templeton)

(cherry picked from commit f5839fd6df77786353160b41eb4b5ceea9380017)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be7e4a08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be7e4a08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be7e4a08

Branch: refs/heads/branch-2
Commit: be7e4a080558adb1c83c82b456ac3e0abbb9eb9b
Parents: bed700e
Author: Daniel Templeton 
Authored: Thu Jan 19 12:45:17 2017 -0800
Committer: Daniel Templeton 
Committed: Fri Jan 20 10:07:49 2017 -0800

--
 .../hadoop-common/src/site/markdown/Metrics.md  | 53 +++-
 1 file changed, 51 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/be7e4a08/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
--
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 53d1095..e52a6a7 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -443,6 +443,57 @@ NodeManagerMetrics shows the statistics of the containers 
in the node. Each metr
 | `goodLocalDirsDiskUtilizationPerc` | Current disk utilization percentage 
across all good local directories |
 | `goodLogDirsDiskUtilizationPerc` | Current disk utilization percentage 
across all good log directories |
 
+ContainerMetrics
+--
+
+ContainerMetrics shows the resource utilization statistics of a container. 
Each metrics record contains tags such as ContainerPid and Hostname as 
additional information along with metrics.
+
+| Name | Description |
+|: |: |
+| `pMemLimitMBs` | Physical memory limit of the container in MB |
+| `vMemLimitMBs` | Virtual memory limit of the container in MB |
+| `vCoreLimit` | CPU limit of the container in number of vcores |
+| `launchDurationMs` | Container launch duration in msec  |
+| `localizationDurationMs` | Container localization duration in msec |
+| `StartTime` | Time in msec when container starts |
+| `FinishTime` | Time in msec when container finishes |
+| `ExitCode` | Container's exit code |
+| `PMemUsageMBsNumUsage` | Total number of physical memory used metrics |
+| `PMemUsageMBsAvgMBs` | Average physical memory used in MB |
+| `PMemUsageMBsStdevMBs` | Standard deviation of the physical memory used in 
MB |
+| `PMemUsageMBsMinMBs` | Minimum physical memory used in MB |
+| `PMemUsageMBsMaxMBs` | Maximum physical memory used in MB |
+| `PMemUsageMBsIMinMBs` | Minimum physical memory used in MB of current 
_interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PMemUsageMBsIMaxMBs` | Maximum physical memory used in MB of current 
_interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PMemUsageMBsINumUsage` | Total number of physical memory used metrics in 
current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PCpuUsagePercentNumUsage` | Total number of physical CPU cores percent used 
metrics |
+| `PCpuUsagePercentAvgPercents` | Average physical CPU cores percent used |
+| `PCpuUsagePercentStdevPercents` | Standard deviation of physical CPU cores 
percent used |
+| `PCpuUsagePercentMinPercents` | Minimum physical CPU cores percent used|
+| `PCpuUsagePercentMaxPercents` | Maximum physical CPU cores percent used |
+| `PCpuUsagePercentIMinPercents` | Minimum physical CPU cores percent used in 
current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PCpuUsagePercentIMaxPercents` | Maximum physical CPU cores percent used in 
current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PCpuUsagePercentINumUsage` | Total number of physical CPU cores used 
metrics in current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `MilliVcoreUsageNumUsage` | Total number of vcores used metrics |
+| `MilliVcoreUsageAvgMilliVcores` | 1000 times the average vcores used |
+| `MilliVcoreUsageStdevMilliVcores` | 1000 times the standard deviation of 
vcores used |
+| `MilliVcoreUsageMinMilliVcores` | 1000 times the minimum vcores used |
+| `MilliVcoreUsageMaxMilliVcores` | 1000 times the maximum vcores used |
+| `MilliVcoreUsageIMinMilliVcores` | 1000 times the average vcores used in 
current

[hadoop] Git Push Summary

2017-01-20 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/HEAD [deleted] d85fb264a

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6028. Add document for container metrics (Contributed by Weiwei Yang via Daniel Templeton)

2017-01-19 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/HEAD [created] d85fb264a


YARN-6028. Add document for container metrics (Contributed by Weiwei Yang via 
Daniel Templeton)

(cherry picked from commit f5839fd6df77786353160b41eb4b5ceea9380017)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d85fb264
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d85fb264
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d85fb264

Branch: refs/heads/HEAD
Commit: d85fb264a017d9db052b5ad08dd0028589d39611
Parents: d374087
Author: Daniel Templeton 
Authored: Thu Jan 19 12:45:17 2017 -0800
Committer: Daniel Templeton 
Committed: Thu Jan 19 12:46:53 2017 -0800

--
 .../hadoop-common/src/site/markdown/Metrics.md  | 53 +++-
 1 file changed, 51 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d85fb264/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
--
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 53d1095..e52a6a7 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -443,6 +443,57 @@ NodeManagerMetrics shows the statistics of the containers 
in the node. Each metr
 | `goodLocalDirsDiskUtilizationPerc` | Current disk utilization percentage 
across all good local directories |
 | `goodLogDirsDiskUtilizationPerc` | Current disk utilization percentage 
across all good log directories |
 
+ContainerMetrics
+--
+
+ContainerMetrics shows the resource utilization statistics of a container. 
Each metrics record contains tags such as ContainerPid and Hostname as 
additional information along with metrics.
+
+| Name | Description |
+|: |: |
+| `pMemLimitMBs` | Physical memory limit of the container in MB |
+| `vMemLimitMBs` | Virtual memory limit of the container in MB |
+| `vCoreLimit` | CPU limit of the container in number of vcores |
+| `launchDurationMs` | Container launch duration in msec  |
+| `localizationDurationMs` | Container localization duration in msec |
+| `StartTime` | Time in msec when container starts |
+| `FinishTime` | Time in msec when container finishes |
+| `ExitCode` | Container's exit code |
+| `PMemUsageMBsNumUsage` | Total number of physical memory used metrics |
+| `PMemUsageMBsAvgMBs` | Average physical memory used in MB |
+| `PMemUsageMBsStdevMBs` | Standard deviation of the physical memory used in 
MB |
+| `PMemUsageMBsMinMBs` | Minimum physical memory used in MB |
+| `PMemUsageMBsMaxMBs` | Maximum physical memory used in MB |
+| `PMemUsageMBsIMinMBs` | Minimum physical memory used in MB of current 
_interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PMemUsageMBsIMaxMBs` | Maximum physical memory used in MB of current 
_interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PMemUsageMBsINumUsage` | Total number of physical memory used metrics in 
current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PCpuUsagePercentNumUsage` | Total number of physical CPU cores percent used 
metrics |
+| `PCpuUsagePercentAvgPercents` | Average physical CPU cores percent used |
+| `PCpuUsagePercentStdevPercents` | Standard deviation of physical CPU cores 
percent used |
+| `PCpuUsagePercentMinPercents` | Minimum physical CPU cores percent used|
+| `PCpuUsagePercentMaxPercents` | Maximum physical CPU cores percent used |
+| `PCpuUsagePercentIMinPercents` | Minimum physical CPU cores percent used in 
current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PCpuUsagePercentIMaxPercents` | Maximum physical CPU cores percent used in 
current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PCpuUsagePercentINumUsage` | Total number of physical CPU cores used 
metrics in current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `MilliVcoreUsageNumUsage` | Total number of vcores used metrics |
+| `MilliVcoreUsageAvgMilliVcores` | 1000 times the average vcores used |
+| `MilliVcoreUsageStdevMilliVcores` | 1000 times the standard deviation of 
vcores used |
+| `MilliVcoreUsageMinMilliVcores` | 1000 times the minimum vcores used |
+| `MilliVcoreUsageMaxMilliVcores` | 1000 times the maximum vcores used |
+| `MilliVcoreUsageIMinMilliVcores` | 1000 times the average vcores used in 
current _interval_ (the

hadoop git commit: YARN-6028. Add document for container metrics (Contributed by Weiwei Yang via Daniel Templeton)

2017-01-19 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk efc8faa1b -> f5839fd6d


YARN-6028. Add document for container metrics (Contributed by Weiwei Yang via 
Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5839fd6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5839fd6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5839fd6

Branch: refs/heads/trunk
Commit: f5839fd6df77786353160b41eb4b5ceea9380017
Parents: efc8faa
Author: Daniel Templeton 
Authored: Thu Jan 19 12:45:17 2017 -0800
Committer: Daniel Templeton 
Committed: Thu Jan 19 12:45:17 2017 -0800

--
 .../hadoop-common/src/site/markdown/Metrics.md  | 53 +++-
 1 file changed, 51 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5839fd6/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
--
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 9a19a9b..32d081a 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -446,6 +446,57 @@ NodeManagerMetrics shows the statistics of the containers 
in the node. Each metr
 | `goodLocalDirsDiskUtilizationPerc` | Current disk utilization percentage 
across all good local directories |
 | `goodLogDirsDiskUtilizationPerc` | Current disk utilization percentage 
across all good log directories |
 
+ContainerMetrics
+--
+
+ContainerMetrics shows the resource utilization statistics of a container. 
Each metrics record contains tags such as ContainerPid and Hostname as 
additional information along with metrics.
+
+| Name | Description |
+|: |: |
+| `pMemLimitMBs` | Physical memory limit of the container in MB |
+| `vMemLimitMBs` | Virtual memory limit of the container in MB |
+| `vCoreLimit` | CPU limit of the container in number of vcores |
+| `launchDurationMs` | Container launch duration in msec  |
+| `localizationDurationMs` | Container localization duration in msec |
+| `StartTime` | Time in msec when container starts |
+| `FinishTime` | Time in msec when container finishes |
+| `ExitCode` | Container's exit code |
+| `PMemUsageMBsNumUsage` | Total number of physical memory used metrics |
+| `PMemUsageMBsAvgMBs` | Average physical memory used in MB |
+| `PMemUsageMBsStdevMBs` | Standard deviation of the physical memory used in 
MB |
+| `PMemUsageMBsMinMBs` | Minimum physical memory used in MB |
+| `PMemUsageMBsMaxMBs` | Maximum physical memory used in MB |
+| `PMemUsageMBsIMinMBs` | Minimum physical memory used in MB of current 
_interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PMemUsageMBsIMaxMBs` | Maximum physical memory used in MB of current 
_interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PMemUsageMBsINumUsage` | Total number of physical memory used metrics in 
current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PCpuUsagePercentNumUsage` | Total number of physical CPU cores percent used 
metrics |
+| `PCpuUsagePercentAvgPercents` | Average physical CPU cores percent used |
+| `PCpuUsagePercentStdevPercents` | Standard deviation of physical CPU cores 
percent used |
+| `PCpuUsagePercentMinPercents` | Minimum physical CPU cores percent used|
+| `PCpuUsagePercentMaxPercents` | Maximum physical CPU cores percent used |
+| `PCpuUsagePercentIMinPercents` | Minimum physical CPU cores percent used in 
current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PCpuUsagePercentIMaxPercents` | Maximum physical CPU cores percent used in 
current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `PCpuUsagePercentINumUsage` | Total number of physical CPU cores used 
metrics in current _interval_ (the time of _interval_ is specified by 
yarn.nodemanager.container-metrics.period-ms) |
+| `MilliVcoreUsageNumUsage` | Total number of vcores used metrics |
+| `MilliVcoreUsageAvgMilliVcores` | 1000 times the average vcores used |
+| `MilliVcoreUsageStdevMilliVcores` | 1000 times the standard deviation of 
vcores used |
+| `MilliVcoreUsageMinMilliVcores` | 1000 times the minimum vcores used |
+| `MilliVcoreUsageMaxMilliVcores` | 1000 times the maximum vcores used |
+| `MilliVcoreUsageIMinMilliVcores` | 1000 times the average vcores used in 
current _interval_ (the time of _interval_ is specified by

hadoop git commit: YARN-6071. Fix incompatible API change on AM-RM protocol due to YARN-3866 (trunk only) (Contributed by Wangda Tan via Daniel Templeton)

2017-01-17 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 78b487bde -> 695f110f9


YARN-6071. Fix incompatible API change on AM-RM protocol due to YARN-3866 
(trunk only)
(Contributed by Wangda Tan via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/695f110f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/695f110f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/695f110f

Branch: refs/heads/trunk
Commit: 695f110f937b35e92d635b9e93740d3b5af179a6
Parents: 78b487b
Author: Daniel Templeton 
Authored: Tue Jan 17 12:55:47 2017 -0800
Committer: Daniel Templeton 
Committed: Tue Jan 17 12:55:47 2017 -0800

--
 .../apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java  | 4 ++--
 .../hadoop-yarn-api/src/main/proto/yarn_service_protos.proto | 2 +-
 .../org/apache/hadoop/yarn/server/resourcemanager/MockAM.java| 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/695f110f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
index 0786794..ae0891e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
@@ -77,8 +77,8 @@ public abstract class AllocateRequest {
   public static AllocateRequest newInstance(int responseID, float appProgress,
   List resourceAsk,
   List containersToBeReleased,
-  ResourceBlacklistRequest resourceBlacklistRequest,
-  List updateRequests) {
+  List updateRequests,
+  ResourceBlacklistRequest resourceBlacklistRequest) {
 return AllocateRequest.newBuilder().responseId(responseID)
 .progress(appProgress).askList(resourceAsk)
 .releaseList(containersToBeReleased)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/695f110f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index aed1580..df3c852 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -109,11 +109,11 @@ message AllocateResponseProto {
   optional int32 num_cluster_nodes = 7;
   optional PreemptionMessageProto preempt = 8;
   repeated NMTokenProto nm_tokens = 9;
-  repeated UpdatedContainerProto updated_containers = 10;
   optional hadoop.common.TokenProto am_rm_token = 12;
   optional PriorityProto application_priority = 13;
   optional string collector_addr = 14;
   repeated UpdateContainerErrorProto update_errors = 15;
+  repeated UpdatedContainerProto updated_containers = 16;
 }
 
 enum SchedulerResourceTypes {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/695f110f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
index fbeca7b..2451c1e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
@@ -247,14 +247,14 @@ public class MockAM {
   public AllocateResponse sendContainerResizingRequest(
   List updateRequests) throws Exception {
 final AllocateRequest req = AllocateRequest.newInstance(0, 0F, null, null,
-null, updateRequests);
+updateRequests, null);

hadoop git commit: YARN-5849. Automatically create YARN control group for pre-mounted cgroups (Contributed by Miklos Szegedi via Daniel Templeton)

2017-01-11 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 5f336512d -> e6f13fe5d


YARN-5849. Automatically create YARN control group for pre-mounted cgroups 
(Contributed by Miklos Szegedi via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6f13fe5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6f13fe5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6f13fe5

Branch: refs/heads/trunk
Commit: e6f13fe5d1df8918ffc680d18f9d5576f38893a6
Parents: 5f33651
Author: Daniel Templeton 
Authored: Wed Jan 11 14:48:52 2017 -0800
Committer: Daniel Templeton 
Committed: Wed Jan 11 16:00:01 2017 -0800

--
 .../src/main/resources/yarn-default.xml |   8 +-
 .../CGroupsBlkioResourceHandlerImpl.java|   2 +-
 .../CGroupsCpuResourceHandlerImpl.java  |   2 +-
 .../linux/resources/CGroupsHandler.java |  58 ---
 .../linux/resources/CGroupsHandlerImpl.java | 171 +++
 .../CGroupsMemoryResourceHandlerImpl.java   |   2 +-
 .../TrafficControlBandwidthHandlerImpl.java |   3 +-
 .../TestCGroupsBlkioResourceHandlerImpl.java|   2 +-
 .../TestCGroupsCpuResourceHandlerImpl.java  |   6 +-
 .../linux/resources/TestCGroupsHandlerImpl.java | 153 -
 .../TestCGroupsMemoryResourceHandlerImpl.java   |   2 +-
 .../TestTrafficControlBandwidthHandlerImpl.java |   2 +-
 .../src/site/markdown/NodeManagerCgroups.md |   2 +-
 13 files changed, 334 insertions(+), 79 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6f13fe5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 3c30ed3..f287db0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1458,9 +1458,11 @@
 
   
 The cgroups hierarchy under which to place YARN proccesses 
(cannot contain commas).
-If yarn.nodemanager.linux-container-executor.cgroups.mount is false (that 
is, if cgroups have
-been pre-configured), then this cgroups hierarchy must already exist and 
be writable by the
-NodeManager user, otherwise the NodeManager may fail.
+If yarn.nodemanager.linux-container-executor.cgroups.mount is false
+(that is, if cgroups have been pre-configured) and the Yarn user has write
+access to the parent directory, then the directory will be created.
+If the directory already exists, the administrator has to give Yarn
+write permissions to it recursively.
 Only used when the LCE resources handler is set to the 
CgroupsLCEResourcesHandler.
 yarn.nodemanager.linux-container-executor.cgroups.hierarchy
 /hadoop-yarn

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6f13fe5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
index e7eea1f..e0b43d3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
@@ -120,7 +120,7 @@ public class CGroupsBlkioResourceHandlerImpl implements 
DiskResourceHandler {
 // if bootstrap is called on this class, disk is already enabled
 // so no need to check again
 this.cGroupsHandler
-  .mountCGroupController(CGroupsHandler.CGroupController.BLKIO);
+  .initializeCGroupController(CGroupsHandler.CGroupController.BLKIO);
 return null;
   }

hadoop git commit: YARN-5849. Automatically create YARN control group for pre-mounted cgroups (Contributed by Miklos Szegedi via Daniel Templeton)

2017-01-11 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b089e122b -> 718ae2d47


YARN-5849. Automatically create YARN control group for pre-mounted cgroups 
(Contributed by Miklos Szegedi via Daniel Templeton)

(cherry picked from commit 0fffebe51a91023c4949ab560f5c828f1b568072)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/718ae2d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/718ae2d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/718ae2d4

Branch: refs/heads/branch-2
Commit: 718ae2d4773fd2771c64e2f55a9eed1c4ffe3c07
Parents: b089e12
Author: Daniel Templeton 
Authored: Wed Jan 11 14:48:52 2017 -0800
Committer: Daniel Templeton 
Committed: Wed Jan 11 15:36:18 2017 -0800

--
 .../src/main/resources/yarn-default.xml |   8 +-
 .../CGroupsBlkioResourceHandlerImpl.java|   2 +-
 .../CGroupsCpuResourceHandlerImpl.java  |   2 +-
 .../linux/resources/CGroupsHandler.java |  58 ---
 .../linux/resources/CGroupsHandlerImpl.java | 171 +++
 .../CGroupsMemoryResourceHandlerImpl.java   |   2 +-
 .../TrafficControlBandwidthHandlerImpl.java |   3 +-
 .../TestCGroupsBlkioResourceHandlerImpl.java|   2 +-
 .../TestCGroupsCpuResourceHandlerImpl.java  |   6 +-
 .../linux/resources/TestCGroupsHandlerImpl.java | 153 -
 .../TestCGroupsMemoryResourceHandlerImpl.java   |   2 +-
 .../TestTrafficControlBandwidthHandlerImpl.java |   2 +-
 .../src/site/markdown/NodeManagerCgroups.md |   2 +-
 13 files changed, 334 insertions(+), 79 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/718ae2d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index d6ea32b..310cdb4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1424,9 +1424,11 @@
 
   
 The cgroups hierarchy under which to place YARN proccesses 
(cannot contain commas).
-If yarn.nodemanager.linux-container-executor.cgroups.mount is false (that 
is, if cgroups have
-been pre-configured), then this cgroups hierarchy must already exist and 
be writable by the
-NodeManager user, otherwise the NodeManager may fail.
+If yarn.nodemanager.linux-container-executor.cgroups.mount is false
+(that is, if cgroups have been pre-configured) and the Yarn user has write
+access to the parent directory, then the directory will be created.
+If the directory already exists, the administrator has to give Yarn
+write permissions to it recursively.
 Only used when the LCE resources handler is set to the 
CgroupsLCEResourcesHandler.
 yarn.nodemanager.linux-container-executor.cgroups.hierarchy
 /hadoop-yarn

http://git-wip-us.apache.org/repos/asf/hadoop/blob/718ae2d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
index e7eea1f..e0b43d3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
@@ -120,7 +120,7 @@ public class CGroupsBlkioResourceHandlerImpl implements 
DiskResourceHandler {
 // if bootstrap is called on this class, disk is already enabled
 // so no need to check again
 this.cGroupsHandler
-  .mountCGroupController(CGroupsHandler.CGroupController.BLKIO);
+  .initializeCGroupController(CGroupsHandler.CGroupController.BLKIO);
 return null;
   }

hadoop git commit: YARN-5554. MoveApplicationAcrossQueues does not check user permission on the target queue (Contributed by Wilfred Spiegelenburg via Daniel Templeton)

2017-01-11 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk e648b6e13 -> 797993942


YARN-5554. MoveApplicationAcrossQueues does not check user permission on the 
target queue
(Contributed by Wilfred Spiegelenburg via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79799394
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79799394
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79799394

Branch: refs/heads/trunk
Commit: 7979939428ad5df213846e11bc1489bdf94ed9f8
Parents: e648b6e
Author: Daniel Templeton 
Authored: Tue Jan 10 16:32:16 2017 -0800
Committer: Daniel Templeton 
Committed: Wed Jan 11 14:34:10 2017 -0800

--
 .../server/resourcemanager/ClientRMService.java |  39 ++-
 .../security/QueueACLsManager.java  |  69 -
 .../resourcemanager/TestClientRMService.java| 258 ++-
 3 files changed, 359 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/79799394/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index add522b..c375887 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1186,20 +1186,35 @@ public class ClientRMService extends AbstractService 
implements
   + callerUGI.getShortUserName() + " cannot perform operation "
   + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId));
 }
-
+
+String targetQueue = request.getTargetQueue();
+if (!accessToTargetQueueAllowed(callerUGI, application, targetQueue)) {
+  RMAuditLogger.logFailure(callerUGI.getShortUserName(),
+  AuditConstants.MOVE_APP_REQUEST, "Target queue doesn't exist or user"
+  + " doesn't have permissions to submit to target queue: "
+  + targetQueue, "ClientRMService",
+  AuditConstants.UNAUTHORIZED_USER, applicationId);
+  throw RPCUtil.getRemoteException(new AccessControlException("User "
+  + callerUGI.getShortUserName() + " cannot submit applications to"
+  + " target queue or the target queue doesn't exist: "
+  + targetQueue + " while moving " + applicationId));
+}
+
 // Moves only allowed when app is in a state that means it is tracked by
 // the scheduler. Introducing SUBMITTED state also to this list as there
 // could be a corner scenario that app may not be in Scheduler in SUBMITTED
 // state.
 if (!ACTIVE_APP_STATES.contains(application.getState())) {
-  String msg = "App in " + application.getState() + " state cannot be 
moved.";
+  String msg = "App in " + application.getState() +
+  " state cannot be moved.";
   RMAuditLogger.logFailure(callerUGI.getShortUserName(),
   AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService", msg);
   throw new YarnException(msg);
 }
 
 try {
-  this.rmAppManager.moveApplicationAcrossQueue(applicationId, 
request.getTargetQueue());
+  this.rmAppManager.moveApplicationAcrossQueue(applicationId,
+  request.getTargetQueue());
 } catch (YarnException ex) {
   RMAuditLogger.logFailure(callerUGI.getShortUserName(),
   AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService",
@@ -1214,6 +1229,24 @@ public class ClientRMService extends AbstractService 
implements
 return response;
   }
 
+  /**
+   * Check if the submission of an application to the target queue is allowed.
+   * @param callerUGI the caller UGI
+   * @param application the application to move
+   * @param targetQueue the queue to move the application to
+   * @return true if submission is allowed, false otherwise
+   */
+  private boolean accessToTargetQueueAllowed(UserGroupInformation callerUGI,
+  RMApp application, String targetQueue) {
+return
+queueACLsManager.checkAccess(callerUGI,
+QueueACL.SUBMIT_APPLICATIONS, application,
+Server.getRemoteAddress(), null, targetQueue) ||
+

hadoop git commit: YARN-6073. Misuse of format specifier in Preconditions.checkArgument (Contributed by Yuanbo Liu via Daniel Templeton)

2017-01-09 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b60057709 -> 74bf9c6e4


YARN-6073. Misuse of format specifier in Preconditions.checkArgument 
(Contributed by Yuanbo Liu via Daniel Templeton)

(cherry picked from commit 6332a318bc1e2e9d73d7159eab26347bb3f1f9b3)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74bf9c6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74bf9c6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74bf9c6e

Branch: refs/heads/branch-2
Commit: 74bf9c6e4b982cecbd3aa0d345c2c2144f4c6f0e
Parents: b600577
Author: Daniel Templeton 
Authored: Mon Jan 9 22:12:58 2017 -0800
Committer: Daniel Templeton 
Committed: Mon Jan 9 22:14:46 2017 -0800

--
 .../main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/74bf9c6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 2508c3e..4826013 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -685,7 +685,7 @@ public class RMAdminCLI extends HAAdmin {
   }
   
   int nLabels = map.get(nodeId).size();
-  Preconditions.checkArgument(nLabels <= 1, "%d labels specified on 
host=%s"
+  Preconditions.checkArgument(nLabels <= 1, "%s labels specified on 
host=%s"
   + ", please note that we do not support specifying multiple"
   + " labels on a single host for now.", nLabels, nodeIdStr);
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6073. Misuse of format specifier in Preconditions.checkArgument (Contributed by Yuanbo Liu via Daniel Templeton)

2017-01-09 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 945db55f2 -> 6332a318b


YARN-6073. Misuse of format specifier in Preconditions.checkArgument 
(Contributed by Yuanbo Liu via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6332a318
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6332a318
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6332a318

Branch: refs/heads/trunk
Commit: 6332a318bc1e2e9d73d7159eab26347bb3f1f9b3
Parents: 945db55
Author: Daniel Templeton 
Authored: Mon Jan 9 22:12:58 2017 -0800
Committer: Daniel Templeton 
Committed: Mon Jan 9 22:12:58 2017 -0800

--
 .../main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6332a318/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 2508c3e..4826013 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -685,7 +685,7 @@ public class RMAdminCLI extends HAAdmin {
   }
   
   int nLabels = map.get(nodeId).size();
-  Preconditions.checkArgument(nLabels <= 1, "%d labels specified on 
host=%s"
+  Preconditions.checkArgument(nLabels <= 1, "%s labels specified on 
host=%s"
   + ", please note that we do not support specifying multiple"
   + " labels on a single host for now.", nLabels, nodeIdStr);
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6022. Revert changes of AbstractResourceRequest (Contributed by Wangda Tan via Daniel Templeton)

2017-01-07 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk a59df1575 -> ac16400e1


YARN-6022. Revert changes of AbstractResourceRequest (Contributed by Wangda Tan 
via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac16400e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac16400e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac16400e

Branch: refs/heads/trunk
Commit: ac16400e1fb85a4186e5bf5bbc9cf204735ae74f
Parents: a59df15
Author: Daniel Templeton 
Authored: Sat Jan 7 08:23:06 2017 -0800
Committer: Daniel Templeton 
Committed: Sat Jan 7 08:24:30 2017 -0800

--
 .../api/records/AbstractResourceRequest.java| 55 
 .../yarn/api/records/ResourceRequest.java   | 19 ++-
 .../api/records/UpdateContainerRequest.java | 18 ++-
 .../server/scheduler/SchedulerRequestKey.java   |  1 -
 .../server/resourcemanager/RMAppManager.java|  2 +-
 .../server/resourcemanager/RMServerUtils.java   |  2 +-
 .../scheduler/AbstractYarnScheduler.java|  7 ++-
 .../scheduler/SchedulerUtils.java   | 18 ---
 .../scheduler/YarnScheduler.java|  6 +--
 .../scheduler/fair/FairScheduler.java   |  5 +-
 10 files changed, 55 insertions(+), 78 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
deleted file mode 100644
index 819a607..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-/**
- * {@code AbstractResourceRequest} represents a generic resource request made
- * by an application to the {@code ResourceManager}.
- * 
- * It includes:
- * 
- *   {@link Resource} capability required for each request.
- * 
- *
- * @see Resource
- */
-@Public
-@Unstable
-public abstract class AbstractResourceRequest {
-
-  /**
-   * Set the Resource capability of the request
-   * @param capability Resource capability of the request
-   */
-  @Public
-  @Stable
-  public abstract void setCapability(Resource capability);
-
-  /**
-   * Get the Resource capability of the request.
-   * @return Resource capability of the request
-   */
-  @Public
-  @Stable
-  public abstract Resource getCapability();
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index ce85b21..5bedc87 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -58,8 +58,7 @@ import org.apache.hadoop.yarn.util.Records;
  */
 @Public
 @Stable
-public abstract class ResourceRequest extends AbstractResourceRequest
-

hadoop git commit: YARN-5991. Yarn Distributed Shell does not print throwable t to App Master When failed to start container (Contributed by Jim Frankola via Daniel Templeton)

2017-01-06 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 2977bc6a1 -> 71a4acf74


YARN-5991. Yarn Distributed Shell does not print throwable t to App Master When 
failed to start container
(Contributed by Jim Frankola via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71a4acf7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71a4acf7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71a4acf7

Branch: refs/heads/trunk
Commit: 71a4acf74bc9ca34f0e57835c9d6e3efbe7c0567
Parents: 2977bc6
Author: Daniel Templeton 
Authored: Fri Jan 6 14:26:08 2017 -0800
Committer: Daniel Templeton 
Committed: Fri Jan 6 14:26:08 2017 -0800

--
 .../yarn/applications/distributedshell/ApplicationMaster.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a4acf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 17dae6b..5a06ef6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -1001,7 +1001,7 @@ public class ApplicationMaster {
 
 @Override
 public void onStartContainerError(ContainerId containerId, Throwable t) {
-  LOG.error("Failed to start Container " + containerId);
+  LOG.error("Failed to start Container " + containerId, t);
   containers.remove(containerId);
   applicationMaster.numCompletedContainers.incrementAndGet();
   applicationMaster.numFailedContainers.incrementAndGet();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: MAPREDUCE-6715. Fix Several Unsafe Practices (Contributed by Yufei Gu via Daniel Templeton)

2017-01-05 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 9c61858ee -> d1aa844dc


MAPREDUCE-6715. Fix Several Unsafe Practices (Contributed by Yufei Gu via 
Daniel Templeton)

(cherry picked from commit 0b8a7c18ddbe73b356b3c9baf4460659ccaee095)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1aa844d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1aa844d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1aa844d

Branch: refs/heads/branch-2
Commit: d1aa844dc690ae43f4e73667d765bee2dc45d7bc
Parents: 9c61858
Author: Daniel Templeton 
Authored: Thu Jan 5 17:55:05 2017 -0800
Committer: Daniel Templeton 
Committed: Thu Jan 5 17:57:14 2017 -0800

--
 .../org/apache/hadoop/mapred/CleanupQueue.java  |  7 +++-
 .../java/org/apache/hadoop/mapred/MapTask.java  | 10 -
 .../mapreduce/lib/output/TextOutputFormat.java  | 15 
 .../task/reduce/ShuffleSchedulerImpl.java   | 40 +++-
 .../hadoop/examples/dancing/Pentomino.java  |  5 +++
 .../hadoop/examples/terasort/TeraScheduler.java | 16 
 6 files changed, 57 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1aa844d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
index 456ed7c..2282b54 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
@@ -136,7 +136,12 @@ class CleanupQueue {
 LOG.debug("DELETED " + context.fullPath);
   }
 } catch (InterruptedException t) {
-  LOG.warn("Interrupted deletion of " + context.fullPath);
+  if (context == null) {
+LOG.warn("Interrupted deletion of an invalid path: Path deletion "
++ "context is null.");
+  } else {
+LOG.warn("Interrupted deletion of " + context.fullPath);
+  }
   return;
 } catch (Exception e) {
   LOG.warn("Error deleting path " + context.fullPath + ": " + e);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1aa844d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
index e839aaf..306c728 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
@@ -411,8 +411,14 @@ public class MapTask extends Task {
 LOG.warn(msg, e);
   }
 }
-throw new IOException("Initialization of all the collectors failed. " +
-  "Error in last collector was :" + lastException.getMessage(), 
lastException);
+
+if (lastException != null) {
+  throw new IOException("Initialization of all the collectors failed. " +
+  "Error in last collector was:" + lastException.toString(),
+  lastException);
+} else {
+  throw new IOException("Initialization of all the collectors failed.");
+}
   }
 
   @SuppressWarnings("unchecked")

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1aa844d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
index 1c8ea72..2e49f68 100644
---

hadoop git commit: MAPREDUCE-6715. Fix Several Unsafe Practices (Contributed by Yufei Gu via Daniel Templeton)

2017-01-05 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 5d182949b -> 0b8a7c18d


MAPREDUCE-6715. Fix Several Unsafe Practices (Contributed by Yufei Gu via 
Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b8a7c18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b8a7c18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b8a7c18

Branch: refs/heads/trunk
Commit: 0b8a7c18ddbe73b356b3c9baf4460659ccaee095
Parents: 5d18294
Author: Daniel Templeton 
Authored: Thu Jan 5 17:55:05 2017 -0800
Committer: Daniel Templeton 
Committed: Thu Jan 5 17:56:08 2017 -0800

--
 .../org/apache/hadoop/mapred/CleanupQueue.java  |  7 +++-
 .../java/org/apache/hadoop/mapred/MapTask.java  | 10 -
 .../mapreduce/lib/output/TextOutputFormat.java  | 15 
 .../task/reduce/ShuffleSchedulerImpl.java   | 40 +++-
 .../hadoop/examples/dancing/Pentomino.java  |  5 +++
 .../hadoop/examples/terasort/TeraScheduler.java | 16 
 6 files changed, 57 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b8a7c18/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
index 456ed7c..2282b54 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java
@@ -136,7 +136,12 @@ class CleanupQueue {
 LOG.debug("DELETED " + context.fullPath);
   }
 } catch (InterruptedException t) {
-  LOG.warn("Interrupted deletion of " + context.fullPath);
+  if (context == null) {
+LOG.warn("Interrupted deletion of an invalid path: Path deletion "
++ "context is null.");
+  } else {
+LOG.warn("Interrupted deletion of " + context.fullPath);
+  }
   return;
 } catch (Exception e) {
   LOG.warn("Error deleting path " + context.fullPath + ": " + e);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b8a7c18/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
index 3753fba..9ec0914 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
@@ -411,8 +411,14 @@ public class MapTask extends Task {
 LOG.warn(msg, e);
   }
 }
-throw new IOException("Initialization of all the collectors failed. " +
-  "Error in last collector was :" + lastException.getMessage(), 
lastException);
+
+if (lastException != null) {
+  throw new IOException("Initialization of all the collectors failed. " +
+  "Error in last collector was:" + lastException.toString(),
+  lastException);
+} else {
+  throw new IOException("Initialization of all the collectors failed.");
+}
   }
 
   @SuppressWarnings("unchecked")

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b8a7c18/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java
index 1c8ea72..2e49f68 100644
---

hadoop git commit: YARN-5931. Document timeout interfaces CLI and REST APIs (Contributed by Rohith Sharma K S via Daniel Templeton)

2016-12-30 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 ec646b1a9 -> 8cb7aa2b5


YARN-5931. Document timeout interfaces CLI and REST APIs (Contributed by Rohith 
Sharma K S via Daniel Templeton)

(cherry picked from commit 165d01a73e5f328108a9e876c7a751a3c2cec4a8)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cb7aa2b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cb7aa2b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cb7aa2b

Branch: refs/heads/branch-2
Commit: 8cb7aa2b528e7642abe73366ef441799551c4685
Parents: ec646b1
Author: Daniel Templeton 
Authored: Fri Dec 30 08:41:52 2016 -0800
Committer: Daniel Templeton 
Committed: Fri Dec 30 08:56:03 2016 -0800

--
 .../yarn/api/records/ApplicationTimeout.java|  12 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |   2 +-
 .../src/main/resources/yarn-default.xml |   2 +-
 .../rmapp/monitor/RMAppLifetimeMonitor.java |   4 +-
 .../src/site/markdown/ResourceManagerRest.md| 250 +++
 .../src/site/markdown/YarnCommands.md   |   1 +
 6 files changed, 266 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cb7aa2b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
index 4beb11e..444a270 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
@@ -28,9 +28,19 @@ import org.apache.hadoop.yarn.util.Records;
  * 
  * {@link ApplicationTimeoutType} of the timeout type.
  * Expiry time in ISO8601 standard with format
- * -MM-dd'T'HH:mm:ss.SSSZ.
+ * -MM-dd'T'HH:mm:ss.SSSZ or "UNLIMITED".
  * Remaining time in seconds.
  * 
+ * The possible values for {ExpiryTime, RemainingTimeInSeconds} are
+ * 
+ * {UNLIMITED,-1} : Timeout is not configured for given timeout type
+ * (LIFETIME).
+ * {ISO8601 date string, 0} : Timeout is configured and application has
+ * completed.
+ * {ISO8601 date string, greater than zero} : Timeout is configured and
+ * application is RUNNING. Application will be timed out after configured
+ * value.
+ * 
  */
 @Public
 @Unstable

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cb7aa2b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7b70347..efa8b9d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1512,7 +1512,7 @@ public class YarnConfiguration extends Configuration {
   RM_PREFIX + "application-timeouts.monitor.interval-ms";
 
   public static final long DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS =
-  6;
+  3000;
 
   /**
* Interval of time the linux container executor should try cleaning up

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cb7aa2b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b0dc83c..79df3ba 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2995,7 +2995,7 @@
 The RMAppLifetimeMonitor Service uses this value as monitor interval
 
 yarn.resourcemanager.application-timeouts.monitor.interval-ms
-6
+3000

hadoop git commit: YARN-5931. Document timeout interfaces CLI and REST APIs (Contributed by Rohith Sharma K S via Daniel Templeton)

2016-12-30 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk c6a5b689d -> 165d01a73


YARN-5931. Document timeout interfaces CLI and REST APIs (Contributed by Rohith 
Sharma K S via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/165d01a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/165d01a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/165d01a7

Branch: refs/heads/trunk
Commit: 165d01a73e5f328108a9e876c7a751a3c2cec4a8
Parents: c6a5b68
Author: Daniel Templeton 
Authored: Fri Dec 30 08:41:52 2016 -0800
Committer: Daniel Templeton 
Committed: Fri Dec 30 08:41:52 2016 -0800

--
 .../yarn/api/records/ApplicationTimeout.java|  12 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |   2 +-
 .../src/main/resources/yarn-default.xml |   2 +-
 .../rmapp/monitor/RMAppLifetimeMonitor.java |   4 +-
 .../src/site/markdown/ResourceManagerRest.md| 250 +++
 .../src/site/markdown/YarnCommands.md   |   1 +
 6 files changed, 266 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/165d01a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
index 4beb11e..444a270 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java
@@ -28,9 +28,19 @@ import org.apache.hadoop.yarn.util.Records;
  * 
  * {@link ApplicationTimeoutType} of the timeout type.
  * Expiry time in ISO8601 standard with format
- * -MM-dd'T'HH:mm:ss.SSSZ.
+ * -MM-dd'T'HH:mm:ss.SSSZ or "UNLIMITED".
  * Remaining time in seconds.
  * 
+ * The possible values for {ExpiryTime, RemainingTimeInSeconds} are
+ * 
+ * {UNLIMITED,-1} : Timeout is not configured for given timeout type
+ * (LIFETIME).
+ * {ISO8601 date string, 0} : Timeout is configured and application has
+ * completed.
+ * {ISO8601 date string, greater than zero} : Timeout is configured and
+ * application is RUNNING. Application will be timed out after configured
+ * value.
+ * 
  */
 @Public
 @Unstable

http://git-wip-us.apache.org/repos/asf/hadoop/blob/165d01a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index bf0a0ab..d0ade22 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1521,7 +1521,7 @@ public class YarnConfiguration extends Configuration {
   RM_PREFIX + "application-timeouts.monitor.interval-ms";
 
   public static final long DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS =
-  6;
+  3000;
 
   /**
* Interval of time the linux container executor should try cleaning up

http://git-wip-us.apache.org/repos/asf/hadoop/blob/165d01a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 9d73308..3c30ed3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2996,7 +2996,7 @@
 The RMAppLifetimeMonitor Service uses this value as monitor interval
 
 yarn.resourcemanager.application-timeouts.monitor.interval-ms
-6
+3000
   
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/165d01a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/monitor/RMAppLifetimeMonitor.java

hadoop git commit: MAPREDUCE-5155. Race condition in test case TestFetchFailure cause it to fail (Contributed by Haibo Chen via Daniel Templeton)

2016-12-29 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk e76995755 -> 95c2c24cd


MAPREDUCE-5155. Race condition in test case TestFetchFailure cause it to fail 
(Contributed by Haibo Chen via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95c2c24c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95c2c24c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95c2c24c

Branch: refs/heads/trunk
Commit: 95c2c24cd0007ece9d7cd0fbe4c5932828e639d4
Parents: e769957
Author: Daniel Templeton 
Authored: Thu Dec 29 10:49:29 2016 -0800
Committer: Daniel Templeton 
Committed: Thu Dec 29 10:50:47 2016 -0800

--
 .../hadoop/mapreduce/v2/app/TestFetchFailure.java  | 17 +++--
 1 file changed, 15 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/95c2c24c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
index 8d25079..cb2a29e 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
 
+import com.google.common.base.Supplier;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.TaskCompletionEvent;
 import org.apache.hadoop.mapreduce.Counters;
@@ -45,6 +46,7 @@ import 
org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptFetchFailureEv
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
 import 
org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.junit.Assert;
 import org.junit.Test;
@@ -79,8 +81,19 @@ public class TestFetchFailure {
 
 // wait for map success
 app.waitForState(mapTask, TaskState.SUCCEEDED);
-
-TaskAttemptCompletionEvent[] events = 
+
+final int checkIntervalMillis = 10;
+final int waitForMillis = 800;
+GenericTestUtils.waitFor(new Supplier() {
+  @Override
+  public Boolean get() {
+TaskAttemptCompletionEvent[] events = job
+.getTaskAttemptCompletionEvents(0, 100);
+return events.length >= 1;
+  }
+}, checkIntervalMillis, waitForMillis);
+
+TaskAttemptCompletionEvent[] events =
   job.getTaskAttemptCompletionEvents(0, 100);
 Assert.assertEquals("Num completion events not correct",
 1, events.length);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HDFS-11278. Add missing @Test annotation for TestSafeMode.testSafeModeUtils() (Contributed by Lukas Majercak via Daniel Templeton)

2016-12-28 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 5f36b6226 -> 747be79b2


HDFS-11278. Add missing @Test annotation for TestSafeMode.testSafeModeUtils() 
(Contributed by Lukas Majercak via Daniel Templeton)

(cherry picked from commit ce3613c9656e57efdc73edd1787bd9622d1b716e)
(cherry picked from commit af266c8c85971da18c2675c5aa638907b5016fc5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/747be79b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/747be79b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/747be79b

Branch: refs/heads/branch-2.8
Commit: 747be79b227166393e1f6b37ce9b30d3cf88f5ce
Parents: 5f36b62
Author: Daniel Templeton 
Authored: Wed Dec 28 20:09:00 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Dec 28 20:11:56 2016 -0800

--
 .../src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java  | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/747be79b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
index e1b958e..d5d576d 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
@@ -482,6 +482,7 @@ public class TestSafeMode {
* Tests some utility methods that surround the SafeMode's state.
* @throws IOException when there's an issue connecting to the test DFS.
*/
+  @Test
   public void testSafeModeUtils() throws IOException {
 dfs = cluster.getFileSystem();
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HDFS-11278. Add missing @Test annotation for TestSafeMode.testSafeModeUtils() (Contributed by Lukas Majercak via Daniel Templeton)

2016-12-28 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 f9b48206d -> af266c8c8


HDFS-11278. Add missing @Test annotation for TestSafeMode.testSafeModeUtils() 
(Contributed by Lukas Majercak via Daniel Templeton)

(cherry picked from commit ce3613c9656e57efdc73edd1787bd9622d1b716e)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af266c8c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af266c8c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af266c8c

Branch: refs/heads/branch-2
Commit: af266c8c85971da18c2675c5aa638907b5016fc5
Parents: f9b4820
Author: Daniel Templeton 
Authored: Wed Dec 28 20:09:00 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Dec 28 20:11:09 2016 -0800

--
 .../src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java  | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/af266c8c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
index 13ca148..e10ea7d 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
@@ -482,6 +482,7 @@ public class TestSafeMode {
* Tests some utility methods that surround the SafeMode's state.
* @throws IOException when there's an issue connecting to the test DFS.
*/
+  @Test
   public void testSafeModeUtils() throws IOException {
 dfs = cluster.getFileSystem();
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HDFS-11278. Add missing @Test annotation for TestSafeMode.testSafeModeUtils() (Contributed by Lukas Majercak via Daniel Templeton)

2016-12-28 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk f216276d2 -> ce3613c96


HDFS-11278. Add missing @Test annotation for TestSafeMode.testSafeModeUtils() 
(Contributed by Lukas Majercak via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce3613c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce3613c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce3613c9

Branch: refs/heads/trunk
Commit: ce3613c9656e57efdc73edd1787bd9622d1b716e
Parents: f216276
Author: Daniel Templeton 
Authored: Wed Dec 28 20:09:00 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Dec 28 20:09:00 2016 -0800

--
 .../src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java  | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce3613c9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
index 4940d45..ada61b9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
@@ -482,6 +482,7 @@ public class TestSafeMode {
* Tests some utility methods that surround the SafeMode's state.
* @throws IOException when there's an issue connecting to the test DFS.
*/
+  @Test
   public void testSafeModeUtils() throws IOException {
 dfs = cluster.getFileSystem();
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-13932. Fix indefinite article in comments (Contributed by LiXin Ge via Daniel Templeton)

2016-12-28 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b3231b480 -> 47f0a0e15


HADOOP-13932. Fix indefinite article in comments (Contributed by LiXin Ge via 
Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47f0a0e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47f0a0e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47f0a0e1

Branch: refs/heads/branch-2
Commit: 47f0a0e15b62372233e4726962d74a6f6592d4fa
Parents: b3231b4
Author: Daniel Templeton 
Authored: Wed Dec 28 07:03:51 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Dec 28 07:03:51 2016 -0800

--
 .../namenode/FSImagePreTransactionalStorageInspector.java| 2 +-
 .../hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md  | 8 
 hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/LibHdfs.md | 2 +-
 .../dev-support/jdiff/Apache_Hadoop_YARN_API_2.6.0.xml   | 2 +-
 .../dev-support/jdiff/Apache_Hadoop_YARN_API_2.7.2.xml   | 2 +-
 .../src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java| 4 ++--
 .../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java  | 2 +-
 .../hadoop-yarn-common/src/main/resources/yarn-default.xml   | 8 
 .../resourcemanager/recovery/FileSystemRMStateStore.java | 2 +-
 .../resourcemanager/scheduler/fair/FSQueueMetrics.java   | 2 +-
 .../java/org/apache/hadoop/yarn/server/MiniYARNCluster.java  | 2 +-
 11 files changed, 18 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/47f0a0e1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
index 3cd5ae1..6469385 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
@@ -41,7 +41,7 @@ import 
org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
 import org.apache.hadoop.io.IOUtils;
 
 /**
- * Inspects a FSImage storage directory in the "old" (pre-HDFS-1073) format.
+ * Inspects an FSImage storage directory in the "old" (pre-HDFS-1073) format.
  * This format has the following data files:
  *   - fsimage
  *   - fsimage.ckpt (when checkpoint is being uploaded)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/47f0a0e1/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md 
b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
index fcbe438..aeaad7c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
@@ -67,7 +67,7 @@ Note that rolling upgrade is supported only from Hadoop-2.4.0 
onwards.
 
 ### Upgrade without Downtime
 
-In a HA cluster, there are two or more *NameNodes (NNs)*, many *DataNodes 
(DNs)*,
+In an HA cluster, there are two or more *NameNodes (NNs)*, many *DataNodes 
(DNs)*,
 a few *JournalNodes (JNs)* and a few *ZooKeeperNodes (ZKNs)*.
 *JNs* is relatively stable and does not require upgrade when upgrading HDFS in 
most of the cases.
 In the rolling upgrade procedure described here,
@@ -78,7 +78,7 @@ Upgrading *JNs* and *ZKNs* may incur cluster downtime.
 
 Suppose there are two namenodes *NN1* and *NN2*,
 where *NN1* and *NN2* are respectively in active and standby states.
-The following are the steps for upgrading a HA cluster:
+The following are the steps for upgrading an HA cluster:
 
 1. Prepare Rolling Upgrade
 1. Run "[`hdfs dfsadmin -rollingUpgrade 
prepare`](#dfsadmin_-rollingUpgrade)"
@@ -135,7 +135,7 @@ However, datanodes can still be upgraded in a rolling 
manner.
 
 In a non-HA cluster, there are a *NameNode (NN)*, a *SecondaryNameNode (SNN)*
 and many *DataNodes (DNs)*.
-The procedure for upgrading a non-HA cluster is similar to upgrading a HA 
cluster
+The procedure for upgrading a non-HA cluster is similar to upgrading an HA 
cluster
 except that Step 2 "Upgrade Active and Standby *NNs*" is changed to below:
 
 * Upgrade *NN* and *SNN*
@@ -180,7 +180,7 @@ are not changed between these two releases.
 
 ### Downgrade without Downtime
 
-In a

hadoop git commit: HADOOP-13932. Fix indefinite article in comments (Contributed by LiXin Ge via Daniel Templeton)

2016-12-27 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk ac1e5d4f7 -> e216e8e23


HADOOP-13932. Fix indefinite article in comments (Contributed by LiXin Ge via 
Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e216e8e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e216e8e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e216e8e2

Branch: refs/heads/trunk
Commit: e216e8e2334519b7c833d99586218e99a39265f3
Parents: ac1e5d4
Author: Daniel Templeton 
Authored: Tue Dec 27 11:12:05 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Dec 27 13:03:16 2016 -0800

--
 .../namenode/FSImagePreTransactionalStorageInspector.java| 2 +-
 .../hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md  | 8 
 hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/LibHdfs.md | 2 +-
 .../dev-support/jdiff/Apache_Hadoop_YARN_API_2.6.0.xml   | 2 +-
 .../dev-support/jdiff/Apache_Hadoop_YARN_API_2.7.2.xml   | 2 +-
 .../src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java| 4 ++--
 .../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java  | 2 +-
 .../hadoop-yarn-common/src/main/resources/yarn-default.xml   | 8 
 .../resourcemanager/recovery/FileSystemRMStateStore.java | 2 +-
 .../resourcemanager/scheduler/fair/FSQueueMetrics.java   | 2 +-
 .../java/org/apache/hadoop/yarn/server/MiniYARNCluster.java  | 2 +-
 11 files changed, 18 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e216e8e2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
index 3cd5ae1..6469385 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImagePreTransactionalStorageInspector.java
@@ -41,7 +41,7 @@ import 
org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
 import org.apache.hadoop.io.IOUtils;
 
 /**
- * Inspects a FSImage storage directory in the "old" (pre-HDFS-1073) format.
+ * Inspects an FSImage storage directory in the "old" (pre-HDFS-1073) format.
  * This format has the following data files:
  *   - fsimage
  *   - fsimage.ckpt (when checkpoint is being uploaded)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e216e8e2/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md 
b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
index f9b76b8..889d7a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
@@ -65,7 +65,7 @@ Note that rolling upgrade is supported only from Hadoop-2.4.0 
onwards.
 
 ### Upgrade without Downtime
 
-In a HA cluster, there are two or more *NameNodes (NNs)*, many *DataNodes 
(DNs)*,
+In an HA cluster, there are two or more *NameNodes (NNs)*, many *DataNodes 
(DNs)*,
 a few *JournalNodes (JNs)* and a few *ZooKeeperNodes (ZKNs)*.
 *JNs* is relatively stable and does not require upgrade when upgrading HDFS in 
most of the cases.
 In the rolling upgrade procedure described here,
@@ -76,7 +76,7 @@ Upgrading *JNs* and *ZKNs* may incur cluster downtime.
 
 Suppose there are two namenodes *NN1* and *NN2*,
 where *NN1* and *NN2* are respectively in active and standby states.
-The following are the steps for upgrading a HA cluster:
+The following are the steps for upgrading an HA cluster:
 
 1. Prepare Rolling Upgrade
 1. Run "[`hdfs dfsadmin -rollingUpgrade 
prepare`](#dfsadmin_-rollingUpgrade)"
@@ -133,7 +133,7 @@ However, datanodes can still be upgraded in a rolling 
manner.
 
 In a non-HA cluster, there are a *NameNode (NN)*, a *SecondaryNameNode (SNN)*
 and many *DataNodes (DNs)*.
-The procedure for upgrading a non-HA cluster is similar to upgrading a HA 
cluster
+The procedure for upgrading a non-HA cluster is similar to upgrading an HA 
cluster
 except that Step 2 "Upgrade Active and Standby *NNs*" is changed to below:
 
 * Upgrade *NN* and *SNN*
@@ -175,7 +175,7 @@ A newer release is downgradable to the pre-upgrade release
 only if both the namenode layout

hadoop git commit: HADOOP-13940. Document the missing envvars commands (Contributed by Yiqun Lin via Daniel Templeton)

2016-12-27 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 cd331556f -> 28b605dd1


HADOOP-13940. Document the missing envvars commands (Contributed by Yiqun Lin 
via Daniel Templeton)

(cherry picked from commit d432173737a532b7106682116013b751ba162ff0)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28b605dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28b605dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28b605dd

Branch: refs/heads/branch-2
Commit: 28b605dd1a34ed569aa3d4ab696109a038055b83
Parents: cd33155
Author: Daniel Templeton 
Authored: Tue Dec 27 07:16:37 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Dec 27 07:19:37 2016 -0800

--
 .../hadoop-common/src/site/markdown/CommandsManual.md  | 6 ++
 .../src/site/markdown/MapredCommands.md| 6 ++
 .../hadoop-yarn-site/src/site/markdown/YarnCommands.md | 6 ++
 3 files changed, 18 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/28b605dd/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index ecd5c20..cf6521e 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -158,6 +158,12 @@ Usage: `hadoop CLASSNAME`
 
 Runs the class named `CLASSNAME`.
 
+### `envvars`
+
+Usage: `hadoop envvars`
+
+Display computed Hadoop environment variables.
+
 Administration Commands
 ---
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/28b605dd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
index 2925abd..a3a6b1f 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
@@ -140,6 +140,12 @@ Prints the version.
 
 Usage: `mapred version`
 
+### `envvars`
+
+Usage: `mapred envvars`
+
+Display computed Hadoop environment variables.
+
 Administration Commands
 ---
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/28b605dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index a916a94..eefbafa 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -162,6 +162,12 @@ Usage: `yarn version`
 
 Prints the Hadoop version.
 
+### `envvars`
+
+Usage: `yarn envvars`
+
+Display computed Hadoop environment variables.
+
 Administration Commands
 ---
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-13940. Document the missing envvars commands (Contributed by Yiqun Lin via Daniel Templeton)

2016-12-27 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk ea547529c -> ded2d08f3


HADOOP-13940. Document the missing envvars commands (Contributed by Yiqun Lin 
via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ded2d08f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ded2d08f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ded2d08f

Branch: refs/heads/trunk
Commit: ded2d08f33c25daf17dbf3e5ff0ddfcf9980c6aa
Parents: ea54752
Author: Daniel Templeton 
Authored: Tue Dec 27 07:16:37 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Dec 27 07:18:08 2016 -0800

--
 .../hadoop-common/src/site/markdown/CommandsManual.md  | 6 ++
 .../src/site/markdown/MapredCommands.md| 6 ++
 .../hadoop-yarn-site/src/site/markdown/YarnCommands.md | 6 ++
 3 files changed, 18 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ded2d08f/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index ef76810..696848b 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -223,6 +223,12 @@ Usage: `hadoop CLASSNAME`
 
 Runs the class named `CLASSNAME`. The class must be part of a package.
 
+### `envvars`
+
+Usage: `hadoop envvars`
+
+Display computed Hadoop environment variables.
+
 Administration Commands
 ---
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ded2d08f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
index f312d31..6b7de2b 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
@@ -140,6 +140,12 @@ Prints the version.
 
 Usage: `mapred version`
 
+### `envvars`
+
+Usage: `mapred envvars`
+
+Display computed Hadoop environment variables.
+
 Administration Commands
 ---
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ded2d08f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 1d51b1f..56096f5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -166,6 +166,12 @@ Usage: `yarn version`
 
 Prints the Hadoop version.
 
+### `envvars`
+
+Usage: `yarn envvars`
+
+Display computed Hadoop environment variables.
+
 Administration Commands
 ---
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5136. Error in handling event type APP_ATTEMPT_REMOVED to the scheduler (Contributed by Wilfred Spiegelenburg via Daniel Templeton)

2016-12-07 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk ab923a53f -> 9f5d2c4ff


YARN-5136. Error in handling event type APP_ATTEMPT_REMOVED to the scheduler
(Contributed by Wilfred Spiegelenburg via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f5d2c4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f5d2c4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f5d2c4f

Branch: refs/heads/trunk
Commit: 9f5d2c4fff6d31acc8b422b52462ef4927c4eea1
Parents: ab923a5
Author: Daniel Templeton 
Authored: Wed Dec 7 11:12:14 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Dec 7 11:12:14 2016 -0800

--
 .../scheduler/fair/FairScheduler.java   | 32 +--
 .../scheduler/fair/TestFairScheduler.java   | 89 
 2 files changed, 115 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f5d2c4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 03df5d4..e790bc2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -633,8 +633,7 @@ public class FairScheduler extends
   RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) {
 try {
   writeLock.lock();
-  LOG.info(
-  "Application " + applicationAttemptId + " is done." + " finalState="
+  LOG.info("Application " + applicationAttemptId + " is done. finalState="
   + rmAppAttemptFinalState);
   FSAppAttempt attempt = getApplicationAttempt(applicationAttemptId);
 
@@ -644,6 +643,13 @@ public class FairScheduler extends
 return;
   }
 
+  // Check if the attempt is already stopped and don't stop it twice.
+  if (attempt.isStopped()) {
+LOG.info("Application " + applicationAttemptId + " has already been "
++ "stopped!");
+return;
+  }
+
   // Release all the running containers
   for (RMContainer rmContainer : attempt.getLiveContainers()) {
 if (keepContainers && rmContainer.getState().equals(
@@ -1521,6 +1527,13 @@ public class FairScheduler extends
   try {
 attempt.getWriteLock().lock();
 FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue();
+// Check if the attempt is already stopped: don't move stopped app
+// attempt. The attempt has already been removed from all queues.
+if (attempt.isStopped()) {
+  LOG.info("Application " + appId + " is stopped and can't be moved!");
+  throw new YarnException("Application " + appId
+  + " is stopped and can't be moved!");
+}
 String destQueueName = handleMoveToPlanQueue(queueName);
 FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false);
 if (targetQueue == null) {
@@ -1617,16 +1630,23 @@ public class FairScheduler extends
* operations will be atomic.
*/
   private void executeMove(SchedulerApplication app,
-  FSAppAttempt attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue) {
-boolean wasRunnable = oldQueue.removeApp(attempt);
+  FSAppAttempt attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue)
+  throws YarnException {
+// Check current runs state. Do not remove the attempt from the queue until
+// after the check has been performed otherwise it could remove the app
+// from a queue without moving it to a new queue.
+boolean wasRunnable = oldQueue.isRunnableApp(attempt);
 // if app was not runnable before, it may be runnable now
 boolean nowRunnable = maxRunningEnforcer.canAppBeRunnable(newQueue,
 attempt);
 if (wasRunnable && !nowRunnable) {
-  throw new IllegalStateException("Should have already verified that app "
+  throw new YarnException("Should have already verified that app "
   + attempt.getApplicationId() + " would be runnable in new queue");
 }
-
+
+//

hadoop git commit: YARN-5136. Error in handling event type APP_ATTEMPT_REMOVED to the scheduler (Contributed by Wilfred Spiegelenburg via Daniel Templeton)

2016-12-07 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 602c99844 -> d8b496151


YARN-5136. Error in handling event type APP_ATTEMPT_REMOVED to the scheduler
(Contributed by Wilfred Spiegelenburg via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8b49615
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8b49615
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8b49615

Branch: refs/heads/branch-2
Commit: d8b4961515ae81619b9689140ce02b1b1cd1
Parents: 602c998
Author: Daniel Templeton 
Authored: Wed Dec 7 10:59:11 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Dec 7 11:01:09 2016 -0800

--
 .../scheduler/fair/FairScheduler.java   | 32 +--
 .../scheduler/fair/TestFairScheduler.java   | 89 
 2 files changed, 115 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8b49615/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 9b473f0..6350ac9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -820,8 +820,7 @@ public class FairScheduler extends
   RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) {
 try {
   writeLock.lock();
-  LOG.info(
-  "Application " + applicationAttemptId + " is done." + " finalState="
+  LOG.info("Application " + applicationAttemptId + " is done. finalState="
   + rmAppAttemptFinalState);
   FSAppAttempt attempt = getApplicationAttempt(applicationAttemptId);
 
@@ -831,6 +830,13 @@ public class FairScheduler extends
 return;
   }
 
+  // Check if the attempt is already stopped and don't stop it twice.
+  if (attempt.isStopped()) {
+LOG.info("Application " + applicationAttemptId + " has already been "
++ "stopped!");
+return;
+  }
+
   // Release all the running containers
   for (RMContainer rmContainer : attempt.getLiveContainers()) {
 if (keepContainers && rmContainer.getState().equals(
@@ -1673,6 +1679,13 @@ public class FairScheduler extends
   try {
 attempt.getWriteLock().lock();
 FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue();
+// Check if the attempt is already stopped: don't move stopped app
+// attempt. The attempt has already been removed from all queues.
+if (attempt.isStopped()) {
+  LOG.info("Application " + appId + " is stopped and can't be moved!");
+  throw new YarnException("Application " + appId
+  + " is stopped and can't be moved!");
+}
 String destQueueName = handleMoveToPlanQueue(queueName);
 FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false);
 if (targetQueue == null) {
@@ -1769,16 +1782,23 @@ public class FairScheduler extends
* operations will be atomic.
*/
   private void executeMove(SchedulerApplication app,
-  FSAppAttempt attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue) {
-boolean wasRunnable = oldQueue.removeApp(attempt);
+  FSAppAttempt attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue)
+  throws YarnException {
+// Check current runs state. Do not remove the attempt from the queue until
+// after the check has been performed otherwise it could remove the app
+// from a queue without moving it to a new queue.
+boolean wasRunnable = oldQueue.isRunnableApp(attempt);
 // if app was not runnable before, it may be runnable now
 boolean nowRunnable = maxRunningEnforcer.canAppBeRunnable(newQueue,
 attempt);
 if (wasRunnable && !nowRunnable) {
-  throw new IllegalStateException("Should have already verified that app "
+  throw new YarnException("Should have already verified that app "
   + attempt.getApplicationId() + " would be runnable in new queue");
 }
-
+
+

hadoop git commit: YARN-5929. Missing scheduling policy in the FS queue metric. (Contributed by Yufei Gu via Daniel Templeton)

2016-12-02 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 c37e814fa -> 1216bad0f


YARN-5929. Missing scheduling policy in the FS queue metric. (Contributed by 
Yufei Gu via Daniel Templeton)

(cherry picked from commit 9b076327eac9da244f94e226816465e7d17fda54)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1216bad0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1216bad0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1216bad0

Branch: refs/heads/branch-2
Commit: 1216bad0f71c300f737fae11ecb62f4fd9310726
Parents: c37e814
Author: Daniel Templeton 
Authored: Fri Dec 2 13:35:09 2016 -0800
Committer: Daniel Templeton 
Committed: Fri Dec 2 14:02:35 2016 -0800

--
 .../scheduler/fair/FSQueueMetrics.java  | 32 +++--
 .../scheduler/fair/TestFSQueueMetrics.java  | 69 
 2 files changed, 97 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1216bad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
index a970815..ca375f2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.annotation.Metric;
@@ -169,6 +170,12 @@ public class FSQueueMetrics extends QueueMetrics {
 amResourceUsageVCores.set(resource.getVirtualCores());
   }
 
+  /**
+   * Get the scheduling policy.
+   *
+   * @return the scheduling policy
+   */
+  @Metric("Scheduling policy")
   public String getSchedulingPolicy() {
 return schedulingPolicy;
   }
@@ -181,21 +188,38 @@ public class FSQueueMetrics extends QueueMetrics {
   static FSQueueMetrics forQueue(String queueName, Queue parent,
   boolean enableUserMetrics, Configuration conf) {
 MetricsSystem ms = DefaultMetricsSystem.instance();
+return forQueue(ms, queueName, parent, enableUserMetrics, conf);
+  }
+
+  /**
+   * Get the FS queue metric for the given queue. Create one and register it to
+   * metrics system if there isn't one for the queue.
+   *
+   * @param ms the metric system
+   * @param queueName queue name
+   * @param parent parent queue
+   * @param enableUserMetrics  if user metrics is needed
+   * @param conf configuration
+   * @return a FSQueueMetrics object
+   */
+  @VisibleForTesting
+  public synchronized
+  static FSQueueMetrics forQueue(MetricsSystem ms, String queueName,
+  Queue parent, boolean enableUserMetrics, Configuration conf) {
 QueueMetrics metrics = queueMetrics.get(queueName);
 if (metrics == null) {
   metrics = new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, 
conf)
   .tag(QUEUE_INFO, queueName);
-  
+
   // Register with the MetricsSystems
   if (ms != null) {
 metrics = ms.register(
-sourceName(queueName).toString(), 
-"Metrics for queue: " + queueName, metrics);
+sourceName(queueName).toString(),
+"Metrics for queue: " + queueName, metrics);
   }
   queueMetrics.put(queueName, metrics);
 }
 
 return (FSQueueMetrics)metrics;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1216bad0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueMetrics.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueMetrics.java

hadoop git commit: YARN-5929. Missing scheduling policy in the FS queue metric. (Contributed by Yufei Gu via Daniel Templeton)

2016-12-02 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 2ff84a004 -> 5bd18c49b


YARN-5929. Missing scheduling policy in the FS queue metric. (Contributed by 
Yufei Gu via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5bd18c49
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5bd18c49
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5bd18c49

Branch: refs/heads/trunk
Commit: 5bd18c49bd5075fa20d24363dceea7828e3fa266
Parents: 2ff84a0
Author: Daniel Templeton 
Authored: Fri Dec 2 13:35:09 2016 -0800
Committer: Daniel Templeton 
Committed: Fri Dec 2 13:55:42 2016 -0800

--
 .../scheduler/fair/FSQueueMetrics.java  | 32 +++--
 .../scheduler/fair/TestFSQueueMetrics.java  | 69 
 2 files changed, 97 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd18c49/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
index a970815..ca375f2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.annotation.Metric;
@@ -169,6 +170,12 @@ public class FSQueueMetrics extends QueueMetrics {
 amResourceUsageVCores.set(resource.getVirtualCores());
   }
 
+  /**
+   * Get the scheduling policy.
+   *
+   * @return the scheduling policy
+   */
+  @Metric("Scheduling policy")
   public String getSchedulingPolicy() {
 return schedulingPolicy;
   }
@@ -181,21 +188,38 @@ public class FSQueueMetrics extends QueueMetrics {
   static FSQueueMetrics forQueue(String queueName, Queue parent,
   boolean enableUserMetrics, Configuration conf) {
 MetricsSystem ms = DefaultMetricsSystem.instance();
+return forQueue(ms, queueName, parent, enableUserMetrics, conf);
+  }
+
+  /**
+   * Get the FS queue metric for the given queue. Create one and register it to
+   * metrics system if there isn't one for the queue.
+   *
+   * @param ms the metric system
+   * @param queueName queue name
+   * @param parent parent queue
+   * @param enableUserMetrics  if user metrics is needed
+   * @param conf configuration
+   * @return a FSQueueMetrics object
+   */
+  @VisibleForTesting
+  public synchronized
+  static FSQueueMetrics forQueue(MetricsSystem ms, String queueName,
+  Queue parent, boolean enableUserMetrics, Configuration conf) {
 QueueMetrics metrics = queueMetrics.get(queueName);
 if (metrics == null) {
   metrics = new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, 
conf)
   .tag(QUEUE_INFO, queueName);
-  
+
   // Register with the MetricsSystems
   if (ms != null) {
 metrics = ms.register(
-sourceName(queueName).toString(), 
-"Metrics for queue: " + queueName, metrics);
+sourceName(queueName).toString(),
+"Metrics for queue: " + queueName, metrics);
   }
   queueMetrics.put(queueName, metrics);
 }
 
 return (FSQueueMetrics)metrics;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd18c49/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueMetrics.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueMetrics.java

hadoop git commit: YARN-5901. Fix race condition in TestGetGroups beforeclass setup() (Contributed by Haibo Chen via Daniel Templeton)

2016-12-02 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 be969e591 -> eb301bb3e


YARN-5901. Fix race condition in TestGetGroups beforeclass setup() (Contributed 
by Haibo Chen via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb301bb3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb301bb3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb301bb3

Branch: refs/heads/branch-2
Commit: eb301bb3ecb9ed63cfd609b24d6eb909d94bd0d1
Parents: be969e5
Author: Daniel Templeton 
Authored: Fri Dec 2 12:05:29 2016 -0800
Committer: Daniel Templeton 
Committed: Fri Dec 2 12:05:29 2016 -0800

--
 .../hadoop/yarn/client/TestGetGroups.java   | 36 +---
 1 file changed, 24 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb301bb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestGetGroups.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestGetGroups.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestGetGroups.java
index e947ece..71f02ed 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestGetGroups.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestGetGroups.java
@@ -20,16 +20,21 @@ package org.apache.hadoop.yarn.client;
 
 import java.io.IOException;
 import java.io.PrintStream;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
 import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.service.ServiceStateChangeListener;
 import org.apache.hadoop.tools.GetGroupsTestBase;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
@@ -42,30 +47,37 @@ public class TestGetGroups extends GetGroupsTestBase {
   private static Configuration conf;
   
   @BeforeClass
-  public static void setUpResourceManager() throws IOException, 
InterruptedException {
+  public static void setUpResourceManager() throws InterruptedException {
 conf = new YarnConfiguration();
 resourceManager = new ResourceManager() {
   @Override
   protected void doSecureLogin() throws IOException {
   };
 };
+
+// a reliable way to wait for resource manager to start
+final CountDownLatch rmStartedSignal = new CountDownLatch(1);
+ServiceStateChangeListener rmStateChangeListener =
+new ServiceStateChangeListener() {
+  @Override
+  public void stateChanged(Service service) {
+if (service.getServiceState() == STATE.STARTED) {
+  rmStartedSignal.countDown();
+}
+  }
+};
+resourceManager.registerServiceListener(rmStateChangeListener);
+
 resourceManager.init(conf);
 new Thread() {
   public void run() {
 resourceManager.start();
   };
 }.start();
-int waitCount = 0;
-while (resourceManager.getServiceState() == STATE.INITED
-&& waitCount++ < 10) {
-  LOG.info("Waiting for RM to start...");
-  Thread.sleep(1000);
-}
-if (resourceManager.getServiceState() != STATE.STARTED) {
-  throw new IOException(
-  "ResourceManager failed to start. Final state is "
-  + resourceManager.getServiceState());
-}
+
+boolean rmStarted = rmStartedSignal.await(6L, TimeUnit.MILLISECONDS);
+Assert.assertTrue("ResourceManager failed to start up.", rmStarted);
+
 LOG.info("ResourceManager RMAdmin address: " +
 conf.get(YarnConfiguration.RM_ADMIN_ADDRESS));
   }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5901. Fix race condition in TestGetGroups beforeclass setup() (Contributed by Haibo Chen via Daniel Templeton)

2016-12-01 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 19f373a46 -> 2d77dc727


YARN-5901. Fix race condition in TestGetGroups beforeclass setup() (Contributed 
by Haibo Chen via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d77dc72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d77dc72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d77dc72

Branch: refs/heads/trunk
Commit: 2d77dc727d9b5e56009bbc36643d85500efcbca5
Parents: 19f373a
Author: Daniel Templeton 
Authored: Thu Dec 1 15:57:39 2016 -0800
Committer: Daniel Templeton 
Committed: Thu Dec 1 15:57:39 2016 -0800

--
 .../hadoop/yarn/client/TestGetGroups.java   | 36 +---
 1 file changed, 24 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d77dc72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestGetGroups.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestGetGroups.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestGetGroups.java
index e947ece..da0258c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestGetGroups.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestGetGroups.java
@@ -20,16 +20,21 @@ package org.apache.hadoop.yarn.client;
 
 import java.io.IOException;
 import java.io.PrintStream;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
 import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.service.ServiceStateChangeListener;
 import org.apache.hadoop.tools.GetGroupsTestBase;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
@@ -42,30 +47,37 @@ public class TestGetGroups extends GetGroupsTestBase {
   private static Configuration conf;
   
   @BeforeClass
-  public static void setUpResourceManager() throws IOException, 
InterruptedException {
+  public static void setUpResourceManager() throws InterruptedException {
 conf = new YarnConfiguration();
 resourceManager = new ResourceManager() {
   @Override
   protected void doSecureLogin() throws IOException {
   };
 };
+
+// a reliable way to wait for resource manager to start
+CountDownLatch rmStartedSignal = new CountDownLatch(1);
+ServiceStateChangeListener rmStateChangeListener =
+new ServiceStateChangeListener() {
+  @Override
+  public void stateChanged(Service service) {
+if (service.getServiceState() == STATE.STARTED) {
+  rmStartedSignal.countDown();
+}
+  }
+};
+resourceManager.registerServiceListener(rmStateChangeListener);
+
 resourceManager.init(conf);
 new Thread() {
   public void run() {
 resourceManager.start();
   };
 }.start();
-int waitCount = 0;
-while (resourceManager.getServiceState() == STATE.INITED
-&& waitCount++ < 10) {
-  LOG.info("Waiting for RM to start...");
-  Thread.sleep(1000);
-}
-if (resourceManager.getServiceState() != STATE.STARTED) {
-  throw new IOException(
-  "ResourceManager failed to start. Final state is "
-  + resourceManager.getServiceState());
-}
+
+boolean rmStarted = rmStartedSignal.await(6L, TimeUnit.MILLISECONDS);
+Assert.assertTrue("ResourceManager failed to start up.", rmStarted);
+
 LOG.info("ResourceManager RMAdmin address: " +
 conf.get(YarnConfiguration.RM_ADMIN_ADDRESS));
   }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5942. "Overridden" is misspelled as "overriden" in FairScheduler.md (Contributed by Heather Sutherland via Daniel Templeton)

2016-11-30 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 20c544d11 -> 3013b028b


YARN-5942. "Overridden" is misspelled as "overriden" in FairScheduler.md
(Contributed by Heather Sutherland via Daniel Templeton)

(cherry picked from commit 57194889fc39a36dc0697b29eafc4ac07ee42d00)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3013b028
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3013b028
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3013b028

Branch: refs/heads/branch-2
Commit: 3013b028baaaec79270f73c6ac008e0c4036b02d
Parents: 20c544d
Author: Daniel Templeton 
Authored: Wed Nov 30 11:22:21 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Nov 30 11:24:44 2016 -0800

--
 .../hadoop-yarn-site/src/site/markdown/FairScheduler.md  | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3013b028/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index 3a15f1c..91e7d8d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -127,13 +127,13 @@ The allocation file must be in XML format. The format 
contains five types of ele
 
 * **A defaultFairSharePreemptionThreshold element**: which sets the fair share 
preemption threshold for the root queue; overridden by 
fairSharePreemptionThreshold element in root queue.
 
-* **A queueMaxAppsDefault element**: which sets the default running app limit 
for queues; overriden by maxRunningApps element in each queue.
+* **A queueMaxAppsDefault element**: which sets the default running app limit 
for queues; overridden by maxRunningApps element in each queue.
 
-* **A queueMaxResourcesDefault element**: which sets the default max resource 
limit for queue; overriden by maxResources element in each queue.
+* **A queueMaxResourcesDefault element**: which sets the default max resource 
limit for queue; overridden by maxResources element in each queue.
 
-* **A queueMaxAMShareDefault element**: which sets the default AM resource 
limit for queue; overriden by maxAMShare element in each queue.
+* **A queueMaxAMShareDefault element**: which sets the default AM resource 
limit for queue; overridden by maxAMShare element in each queue.
 
-* **A defaultQueueSchedulingPolicy element**: which sets the default 
scheduling policy for queues; overriden by the schedulingPolicy element in each 
queue if specified. Defaults to "fair".
+* **A defaultQueueSchedulingPolicy element**: which sets the default 
scheduling policy for queues; overridden by the schedulingPolicy element in 
each queue if specified. Defaults to "fair".
 
 * **A queuePlacementPolicy element**: which contains a list of rule elements 
that tell the scheduler how to place incoming apps into queues. Rules are 
applied in the order that they are listed. Rules may take arguments. All rules 
accept the "create" argument, which indicates whether the rule can create a new 
queue. "Create" defaults to true; if set to false and the rule would place the 
app in a queue that is not configured in the allocations file, we continue on 
to the next rule. The last rule must be one that can never issue a continue. 
Valid rules are:
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5942. "Overridden" is misspelled as "overriden" in FairScheduler.md (Contributed by Heather Sutherland via Daniel Templeton)

2016-11-30 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk be5a75709 -> 4fca94fbd


YARN-5942. "Overridden" is misspelled as "overriden" in FairScheduler.md
(Contributed by Heather Sutherland via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4fca94fb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4fca94fb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4fca94fb

Branch: refs/heads/trunk
Commit: 4fca94fbdad16e845e670758939aabb7a97154d9
Parents: be5a757
Author: Daniel Templeton 
Authored: Wed Nov 30 11:22:21 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Nov 30 11:23:51 2016 -0800

--
 .../hadoop-yarn-site/src/site/markdown/FairScheduler.md  | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fca94fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index ecbb309..ae4c3ab 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -129,13 +129,13 @@ The allocation file must be in XML format. The format 
contains five types of ele
 
 * **A defaultFairSharePreemptionThreshold element**: which sets the fair share 
preemption threshold for the root queue; overridden by 
fairSharePreemptionThreshold element in root queue.
 
-* **A queueMaxAppsDefault element**: which sets the default running app limit 
for queues; overriden by maxRunningApps element in each queue.
+* **A queueMaxAppsDefault element**: which sets the default running app limit 
for queues; overridden by maxRunningApps element in each queue.
 
-* **A queueMaxResourcesDefault element**: which sets the default max resource 
limit for queue; overriden by maxResources element in each queue.
+* **A queueMaxResourcesDefault element**: which sets the default max resource 
limit for queue; overridden by maxResources element in each queue.
 
-* **A queueMaxAMShareDefault element**: which sets the default AM resource 
limit for queue; overriden by maxAMShare element in each queue.
+* **A queueMaxAMShareDefault element**: which sets the default AM resource 
limit for queue; overridden by maxAMShare element in each queue.
 
-* **A defaultQueueSchedulingPolicy element**: which sets the default 
scheduling policy for queues; overriden by the schedulingPolicy element in each 
queue if specified. Defaults to "fair".
+* **A defaultQueueSchedulingPolicy element**: which sets the default 
scheduling policy for queues; overridden by the schedulingPolicy element in 
each queue if specified. Defaults to "fair".
 
 * **A queuePlacementPolicy element**: which contains a list of rule elements 
that tell the scheduler how to place incoming apps into queues. Rules are 
applied in the order that they are listed. Rules may take arguments. All rules 
accept the "create" argument, which indicates whether the rule can create a new 
queue. "Create" defaults to true; if set to false and the rule would place the 
app in a queue that is not configured in the allocations file, we continue on 
to the next rule. The last rule must be one that can never issue a continue. 
Valid rules are:
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-4997. Update fair scheduler to use pluggable auth provider (Contributed by Tao Jie via Daniel Templeton)

2016-11-30 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 625df87c7 -> b3befc021


YARN-4997. Update fair scheduler to use pluggable auth provider (Contributed by 
Tao Jie via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3befc02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3befc02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3befc02

Branch: refs/heads/trunk
Commit: b3befc021b0e2d63d1a3710ea450797d1129f1f5
Parents: 625df87
Author: Daniel Templeton 
Authored: Wed Nov 30 09:50:33 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Nov 30 09:50:33 2016 -0800

--
 .../security/YarnAuthorizationProvider.java | 15 +
 .../scheduler/fair/AllocationConfiguration.java | 38 +--
 .../fair/AllocationFileLoaderService.java   | 68 +---
 .../resourcemanager/scheduler/fair/FSQueue.java | 22 +--
 .../scheduler/fair/FairScheduler.java   | 45 +++--
 .../scheduler/fair/TestFairScheduler.java   |  4 +-
 6 files changed, 149 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3befc02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
index 4b43ea1..9ae4bd7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.util.List;
 
 /**
@@ -61,6 +62,20 @@ public abstract class YarnAuthorizationProvider {
   }
 
   /**
+   * Destroy the {@link YarnAuthorizationProvider} instance.
+   * This method is called only in Tests.
+   */
+  @VisibleForTesting
+  public static void destroy() {
+synchronized (YarnAuthorizationProvider.class) {
+  if (authorizer != null) {
+LOG.debug(authorizer.getClass().getName() + " is destroyed.");
+authorizer = null;
+  }
+}
+  }
+
+  /**
* Initialize the provider. Invoked on daemon startup. DefaultYarnAuthorizer 
is
* initialized based on configurations.
*/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3befc02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index c771887..7bd2616 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -17,6 +17,7 @@
 */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -25,13 +26,14 @@ import java.util.Set;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.ReservationACL;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.security.AccessType;
 import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
 import

hadoop git commit: YARN-4395. Typo in comment in ClientServiceDelegate (Contributed by Alison Yu via Daniel Templeton)

2016-11-29 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk dd9a96cb1 -> 6d8b4f6c2


YARN-4395. Typo in comment in ClientServiceDelegate (Contributed by Alison Yu 
via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d8b4f6c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d8b4f6c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d8b4f6c

Branch: refs/heads/trunk
Commit: 6d8b4f6c2791f861a55ac78c2950f783693e912a
Parents: dd9a96c
Author: Daniel Templeton 
Authored: Tue Nov 29 15:30:22 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Nov 29 15:30:22 2016 -0800

--
 .../java/org/apache/hadoop/mapred/ClientServiceDelegate.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d8b4f6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
index eac8dbc..72339e5 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
@@ -335,8 +335,8 @@ public class ClientServiceDelegate {
   throw new IOException(e.getTargetException());
 }
 
-// if it's AM shut down, do not decrement maxClientRetry as we wait for
-// AM to be restarted.
+// if its AM shut down, do not decrement maxClientRetry while we wait
+// for its AM to be restarted.
 if (!usingAMProxy.get()) {
   maxClientRetry--;
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5851. TestContainerManagerSecurity testContainerManager[1] failed (Contributed by Haibo Chen via Daniel Templeton)

2016-11-29 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 803aba03f -> c22ceb0a9


YARN-5851. TestContainerManagerSecurity testContainerManager[1] failed 
(Contributed by Haibo Chen via Daniel Templeton)

(cherry picked from commit dd9a96cb175d63f7c5909cd98f2dc9af267a5864)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c22ceb0a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c22ceb0a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c22ceb0a

Branch: refs/heads/branch-2
Commit: c22ceb0a9bb8b991d2182fcb128c25a818c0f16f
Parents: 803aba0
Author: Daniel Templeton 
Authored: Tue Nov 29 14:22:19 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Nov 29 14:23:42 2016 -0800

--
 .../apache/hadoop/yarn/server/TestContainerManagerSecurity.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c22ceb0a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 408c1cc..98cb365 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -105,6 +105,7 @@ public class TestContainerManagerSecurity extends 
KerberosSecurityTestcase {
 testRootDir.mkdirs();
 httpSpnegoKeytabFile.deleteOnExit();
 getKdc().createPrincipal(httpSpnegoKeytabFile, httpSpnegoPrincipal);
+UserGroupInformation.setConfiguration(conf);
 
 yarnCluster =
 new MiniYARNCluster(TestContainerManagerSecurity.class.getName(), 1, 1,
@@ -148,7 +149,6 @@ public class TestContainerManagerSecurity extends 
KerberosSecurityTestcase {
   
   public TestContainerManagerSecurity(Configuration conf) {
 conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 10L);
-UserGroupInformation.setConfiguration(conf);
 this.conf = conf;
   }
   


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5851. TestContainerManagerSecurity testContainerManager[1] failed (Contributed by Haibo Chen via Daniel Templeton)

2016-11-29 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 3b9d3acd2 -> dd9a96cb1


YARN-5851. TestContainerManagerSecurity testContainerManager[1] failed 
(Contributed by Haibo Chen via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd9a96cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd9a96cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd9a96cb

Branch: refs/heads/trunk
Commit: dd9a96cb175d63f7c5909cd98f2dc9af267a5864
Parents: 3b9d3ac
Author: Daniel Templeton 
Authored: Tue Nov 29 14:22:19 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Nov 29 14:22:19 2016 -0800

--
 .../apache/hadoop/yarn/server/TestContainerManagerSecurity.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd9a96cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 408c1cc..98cb365 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -105,6 +105,7 @@ public class TestContainerManagerSecurity extends 
KerberosSecurityTestcase {
 testRootDir.mkdirs();
 httpSpnegoKeytabFile.deleteOnExit();
 getKdc().createPrincipal(httpSpnegoKeytabFile, httpSpnegoPrincipal);
+UserGroupInformation.setConfiguration(conf);
 
 yarnCluster =
 new MiniYARNCluster(TestContainerManagerSecurity.class.getName(), 1, 1,
@@ -148,7 +149,6 @@ public class TestContainerManagerSecurity extends 
KerberosSecurityTestcase {
   
   public TestContainerManagerSecurity(Configuration conf) {
 conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 10L);
-UserGroupInformation.setConfiguration(conf);
 this.conf = conf;
   }
   


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5890. FairScheduler should log information about AM-resource-usage and max-AM-share for queues (Contributed by Yufei Gu via Daniel Templeton)

2016-11-29 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b8bebb860 -> 803aba03f


YARN-5890. FairScheduler should log information about AM-resource-usage and 
max-AM-share for queues
(Contributed by Yufei Gu via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/803aba03
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/803aba03
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/803aba03

Branch: refs/heads/branch-2
Commit: 803aba03f20b5d9b67d59c326aa6fb7a02ac1376
Parents: b8bebb8
Author: Daniel Templeton 
Authored: Tue Nov 29 14:14:43 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Nov 29 14:14:43 2016 -0800

--
 .../scheduler/fair/FSLeafQueue.java |  39 --
 .../resourcemanager/scheduler/fair/FSQueue.java |   7 +
 .../scheduler/fair/FSQueueMetrics.java  |  60 
 .../scheduler/fair/TestFairScheduler.java   | 137 +++
 4 files changed, 231 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/803aba03/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index c393759..190c475 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -79,6 +79,7 @@ public class FSLeafQueue extends FSQueue {
 this.lastTimeAtFairShareThreshold = scheduler.getClock().getTime();
 activeUsersManager = new ActiveUsersManager(getMetrics());
 amResourceUsage = Resource.newInstance(0, 0);
+getMetrics().setAMResourceUsage(amResourceUsage);
   }
   
   public void addApp(FSAppAttempt app, boolean runnable) {
@@ -130,6 +131,7 @@ public class FSLeafQueue extends FSQueue {
 // running an unmanaged AM.
 if (runnable && app.isAmRunning()) {
   Resources.subtractFrom(amResourceUsage, app.getAMResource());
+  getMetrics().setAMResourceUsage(amResourceUsage);
 }
 
 return runnable;
@@ -473,18 +475,14 @@ public class FSLeafQueue extends FSQueue {
   }
 
   /**
-   * Check whether this queue can run this application master under the
-   * maxAMShare limit.
-   * @param amResource
-   * @return true if this queue can run
+   * Compute the maximum resource AM can use. The value is the result of
+   * multiplying FairShare and maxAMShare. If FairShare is zero, use
+   * min(maxShare, available resource) instead to prevent zero value for
+   * maximum AM resource since it forbids any job running in the queue.
+   *
+   * @return the maximum resource AM can use
*/
-  public boolean canRunAppAM(Resource amResource) {
-if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
-  return true;
-}
-
-// If FairShare is zero, use min(maxShare, available resource) to compute
-// maxAMResource
+  private Resource computeMaxAMResource() {
 Resource maxResource = Resources.clone(getFairShare());
 if (maxResource.getMemorySize() == 0) {
   maxResource.setMemorySize(
@@ -498,7 +496,23 @@ public class FSLeafQueue extends FSQueue {
   getMaxShare().getVirtualCores()));
 }
 
-Resource maxAMResource = Resources.multiply(maxResource, maxAMShare);
+return Resources.multiply(maxResource, maxAMShare);
+  }
+
+  /**
+   * Check whether this queue can run the Application Master under the
+   * maxAMShare limit.
+   *
+   * @param amResource resources required to run the AM
+   * @return true if this queue can run
+   */
+  public boolean canRunAppAM(Resource amResource) {
+if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
+  return true;
+}
+
+Resource maxAMResource = computeMaxAMResource();
+getMetrics().setMaxAMShare(maxAMResource);
 Resource ifRunAMResource = Resources.add(amResourceUsage, amResource);
 return Resources.fitsIn(ifRunAMResource, maxAMResource);
   }
@@ -506,6 +520,7 @@ public class FSLeafQueue extends FSQueue {
   public void addAMResourceUsage(Resource amResource) {
 if (amResource != null) {

hadoop git commit: YARN-5890. FairScheduler should log information about AM-resource-usage and max-AM-share for queues (Contributed by Yufei Gu via Daniel Templeton)

2016-11-29 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk b407d5319 -> 3b9d3acd2


YARN-5890. FairScheduler should log information about AM-resource-usage and 
max-AM-share for queues
(Contributed by Yufei Gu via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b9d3acd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b9d3acd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b9d3acd

Branch: refs/heads/trunk
Commit: 3b9d3acd203cef4d861c5182fc4dccc55128d347
Parents: b407d53
Author: Daniel Templeton 
Authored: Tue Nov 29 12:44:08 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Nov 29 12:46:05 2016 -0800

--
 .../scheduler/fair/FSLeafQueue.java |  42 --
 .../resourcemanager/scheduler/fair/FSQueue.java |   7 +
 .../scheduler/fair/FSQueueMetrics.java  |  60 
 .../scheduler/fair/TestFairScheduler.java   | 137 +++
 4 files changed, 232 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b9d3acd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 343e9c3..2754616 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -81,6 +81,7 @@ public class FSLeafQueue extends FSQueue {
 this.lastTimeAtMinShare = scheduler.getClock().getTime();
 activeUsersManager = new ActiveUsersManager(getMetrics());
 amResourceUsage = Resource.newInstance(0, 0);
+getMetrics().setAMResourceUsage(amResourceUsage);
   }
   
   void addApp(FSAppAttempt app, boolean runnable) {
@@ -132,6 +133,7 @@ public class FSLeafQueue extends FSQueue {
 // running an unmanaged AM.
 if (runnable && app.isAmRunning()) {
   Resources.subtractFrom(amResourceUsage, app.getAMResource());
+  getMetrics().setAMResourceUsage(amResourceUsage);
 }
 
 return runnable;
@@ -468,19 +470,14 @@ public class FSLeafQueue extends FSQueue {
   }
 
   /**
-   * Check whether this queue can run this application master under the
-   * maxAMShare limit.
-   *
-   * @param amResource resources required to run the AM
-   * @return true if this queue can run
-   */
-  boolean canRunAppAM(Resource amResource) {
-if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
-  return true;
-}
-
-// If FairShare is zero, use min(maxShare, available resource) to compute
-// maxAMResource
+  * Compute the maximum resource AM can use. The value is the result of
+  * multiplying FairShare and maxAMShare. If FairShare is zero, use
+  * min(maxShare, available resource) instead to prevent zero value for
+  * maximum AM resource since it forbids any job running in the queue.
+  *
+  * @return the maximum resource AM can use
+  */
+  private Resource computeMaxAMResource() {
 Resource maxResource = Resources.clone(getFairShare());
 if (maxResource.getMemorySize() == 0) {
   maxResource.setMemorySize(
@@ -494,7 +491,23 @@ public class FSLeafQueue extends FSQueue {
   getMaxShare().getVirtualCores()));
 }
 
-Resource maxAMResource = Resources.multiply(maxResource, maxAMShare);
+return Resources.multiply(maxResource, maxAMShare);
+  }
+
+  /**
+   * Check whether this queue can run the Application Master under the
+   * maxAMShare limit.
+   *
+   * @param amResource resources required to run the AM
+   * @return true if this queue can run
+   */
+  public boolean canRunAppAM(Resource amResource) {
+if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
+  return true;
+}
+
+Resource maxAMResource = computeMaxAMResource();
+getMetrics().setMaxAMShare(maxAMResource);
 Resource ifRunAMResource = Resources.add(amResourceUsage, amResource);
 return Resources.fitsIn(ifRunAMResource, maxAMResource);
   }
@@ -502,6 +515,7 @@ public class FSLeafQueue extends FSQueue {
   void addAMResourceUsage(Resource amResource) {
 if (amResource != null) {

hadoop git commit: YARN-5774. MR Job stuck in ACCEPTED status without any progress in Fair Scheduler if set yarn.scheduler.minimum-allocation-mb to 0. (Contributed by Yufei Gu via Daniel Templeton)

2016-11-29 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 0c895e8a6 -> b8bebb860


YARN-5774. MR Job stuck in ACCEPTED status without any progress in Fair 
Scheduler
if set yarn.scheduler.minimum-allocation-mb to 0. (Contributed by Yufei Gu via 
Daniel Templeton)

(cherry picked from commit 25f9872be63423ada6a18481eaad2888e731fdac)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8bebb86
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8bebb86
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8bebb86

Branch: refs/heads/branch-2
Commit: b8bebb8607ed8603d2fac05ef3e63ce94f9c3660
Parents: 0c895e8
Author: Daniel Templeton 
Authored: Tue Nov 29 09:40:49 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Nov 29 10:12:10 2016 -0800

--
 .../api/records/AbstractResourceRequest.java| 55 ++
 .../yarn/api/records/ResourceRequest.java   | 19 +
 .../api/records/UpdateContainerRequest.java | 18 +
 .../resource/DefaultResourceCalculator.java | 19 +++--
 .../resource/DominantResourceCalculator.java| 25 ++-
 .../yarn/util/resource/ResourceCalculator.java  | 17 +
 .../util/resource/TestResourceCalculator.java   | 79 
 .../server/resourcemanager/RMAppManager.java|  6 +-
 .../server/resourcemanager/RMServerUtils.java   |  9 +--
 .../scheduler/AbstractYarnScheduler.java| 20 +
 .../scheduler/SchedulerUtils.java   | 59 +++
 .../scheduler/YarnScheduler.java|  8 ++
 .../scheduler/capacity/CapacityScheduler.java   |  4 +-
 .../scheduler/fair/FairScheduler.java   | 40 --
 .../scheduler/fifo/FifoScheduler.java   |  4 +-
 .../scheduler/TestSchedulerUtils.java   | 22 +++---
 16 files changed, 258 insertions(+), 146 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8bebb86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
new file mode 100644
index 000..819a607
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * {@code AbstractResourceRequest} represents a generic resource request made
+ * by an application to the {@code ResourceManager}.
+ * 
+ * It includes:
+ * 
+ *   {@link Resource} capability required for each request.
+ * 
+ *
+ * @see Resource
+ */
+@Public
+@Unstable
+public abstract class AbstractResourceRequest {
+
+  /**
+   * Set the Resource capability of the request
+   * @param capability Resource capability of the request
+   */
+  @Public
+  @Stable
+  public abstract void setCapability(Resource capability);
+
+  /**
+   * Get the Resource capability of the request.
+   * @return Resource capability of the request
+   */
+  @Public
+  @Stable
+  public abstract Resource getCapability();
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8bebb86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java

hadoop git commit: YARN-5774. MR Job stuck in ACCEPTED status without any progress in Fair Scheduler if set yarn.scheduler.minimum-allocation-mb to 0. (Contributed by Yufei Gu via Daniel Templeton)

2016-11-29 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 00096dcc0 -> 25f9872be


YARN-5774. MR Job stuck in ACCEPTED status without any progress in Fair 
Scheduler
if set yarn.scheduler.minimum-allocation-mb to 0. (Contributed by Yufei Gu via 
Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25f9872b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25f9872b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25f9872b

Branch: refs/heads/trunk
Commit: 25f9872be63423ada6a18481eaad2888e731fdac
Parents: 00096dc
Author: Daniel Templeton 
Authored: Tue Nov 29 09:40:49 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Nov 29 09:40:49 2016 -0800

--
 .../api/records/AbstractResourceRequest.java| 55 ++
 .../yarn/api/records/ResourceRequest.java   | 19 +
 .../api/records/UpdateContainerRequest.java | 18 +
 .../resource/DefaultResourceCalculator.java | 19 +++--
 .../resource/DominantResourceCalculator.java| 25 ++-
 .../yarn/util/resource/ResourceCalculator.java  | 17 +
 .../util/resource/TestResourceCalculator.java   | 79 
 .../server/resourcemanager/RMAppManager.java|  6 +-
 .../server/resourcemanager/RMServerUtils.java   |  9 +--
 .../scheduler/AbstractYarnScheduler.java| 20 +
 .../scheduler/SchedulerUtils.java   | 59 +++
 .../scheduler/YarnScheduler.java|  8 ++
 .../scheduler/capacity/CapacityScheduler.java   |  4 +-
 .../scheduler/fair/FairScheduler.java   | 40 --
 .../scheduler/fifo/FifoScheduler.java   |  4 +-
 .../scheduler/TestSchedulerUtils.java   | 22 +++---
 16 files changed, 258 insertions(+), 146 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
new file mode 100644
index 000..819a607
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * {@code AbstractResourceRequest} represents a generic resource request made
+ * by an application to the {@code ResourceManager}.
+ * 
+ * It includes:
+ * 
+ *   {@link Resource} capability required for each request.
+ * 
+ *
+ * @see Resource
+ */
+@Public
+@Unstable
+public abstract class AbstractResourceRequest {
+
+  /**
+   * Set the Resource capability of the request
+   * @param capability Resource capability of the request
+   */
+  @Public
+  @Stable
+  public abstract void setCapability(Resource capability);
+
+  /**
+   * Get the Resource capability of the request.
+   * @return Resource capability of the request
+   */
+  @Public
+  @Stable
+  public abstract Resource getCapability();
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f9872b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java

hadoop git commit: YARN-5725. Test uncaught exception in TestContainersMonitorResourceChange.testContainersResourceChange when setting IP and host (Contributed by Miklos Szegedi via Daniel Templeton)

2016-11-28 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk a2b1ff025 -> 62b42ef5d


YARN-5725. Test uncaught exception in 
TestContainersMonitorResourceChange.testContainersResourceChange
when setting IP and host (Contributed by Miklos Szegedi via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62b42ef5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62b42ef5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62b42ef5

Branch: refs/heads/trunk
Commit: 62b42ef5dd04d516d33bf0890ac5cd49f8184a73
Parents: a2b1ff0
Author: Daniel Templeton 
Authored: Mon Nov 28 15:16:43 2016 -0800
Committer: Daniel Templeton 
Committed: Mon Nov 28 15:19:52 2016 -0800

--
 .../monitor/ContainersMonitorImpl.java  | 36 +---
 .../TestContainersMonitorResourceChange.java| 16 +++--
 2 files changed, 36 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/62b42ef5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index dda55d2..718604d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -536,15 +536,21 @@ public class ContainersMonitorImpl extends 
AbstractService implements
   }
 
   Container container = context.getContainers().get(containerId);
-  String[] ipAndHost = containerExecutor.getIpAndHost(container);
-  if (ipAndHost != null && ipAndHost[0] != null
-  && ipAndHost[1] != null) {
-container.setIpAndHost(ipAndHost);
-LOG.info(containerId + "'s ip = " + ipAndHost[0]
-+ ", and hostname = " + ipAndHost[1]);
+
+  if (container != null) {
+String[] ipAndHost = containerExecutor.getIpAndHost(container);
+
+if ((ipAndHost != null) && (ipAndHost[0] != null) &&
+(ipAndHost[1] != null)) {
+  container.setIpAndHost(ipAndHost);
+  LOG.info(containerId + "'s ip = " + ipAndHost[0]
+  + ", and hostname = " + ipAndHost[1]);
+} else {
+  LOG.info("Can not get both ip and hostname: "
+  + Arrays.toString(ipAndHost));
+}
   } else {
-LOG.info("Can not get both ip and hostname: " + Arrays
-.toString(ipAndHost));
+LOG.info(containerId + " is missing. Not setting ip and hostname");
   }
 }
   }
@@ -683,11 +689,15 @@ public class ContainersMonitorImpl extends 
AbstractService implements
 long currentPmemUsage, float cpuUsagePercentPerCore) {
   ContainerImpl container =
   (ContainerImpl) context.getContainers().get(containerId);
-  NMTimelinePublisher nmMetricsPublisher =
-  container.getNMTimelinePublisher();
-  if (nmMetricsPublisher != null) {
-nmMetricsPublisher.reportContainerResourceUsage(container,
-currentPmemUsage, cpuUsagePercentPerCore);
+  if (container != null) {
+NMTimelinePublisher nmMetricsPublisher =
+container.getNMTimelinePublisher();
+if (nmMetricsPublisher != null) {
+  nmMetricsPublisher.reportContainerResourceUsage(container,
+  currentPmemUsage, cpuUsagePercentPerCore);
+}
+  } else {
+LOG.info(containerId + " does not exist to report");
   }
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/62b42ef5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
--
diff --git

hadoop git commit: YARN-5722. FairScheduler hides group resolution exceptions when assigning queue (Contributed by Wilfred Spiegelenburg via Daniel Templeton)

2016-11-22 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 2207a84a7 -> 06f5d27bd


YARN-5722. FairScheduler hides group resolution exceptions when assigning queue 
(Contributed by Wilfred Spiegelenburg via Daniel Templeton)

(cherry picked from commit 83cc7263af632939dc3b2ee58d8f03f98ed4d96a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06f5d27b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06f5d27b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06f5d27b

Branch: refs/heads/branch-2
Commit: 06f5d27bd6c86096011be3b8762e8507766824b2
Parents: 2207a84
Author: Daniel Templeton 
Authored: Tue Nov 22 11:16:00 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Nov 22 11:17:39 2016 -0800

--
 .../yarn/server/resourcemanager/scheduler/fair/FairScheduler.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f5d27b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 1d04710..354f936 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -767,7 +767,8 @@ public class FairScheduler extends
 } catch (InvalidQueueNameException qne) {
   appRejectMsg = qne.getMessage();
 } catch (IOException ioe) {
-  appRejectMsg = "Error assigning app to queue " + queueName;
+  // IOException should only happen for a user without groups
+  appRejectMsg = "Error assigning app to a queue: " + ioe.getMessage();
 }
 
 if (appRejectMsg != null && rmApp != null) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5885. [YARN-4752] Cleanup YARN-4752 for merge (Contributed by Karthik Kambatla via Daniel Templeton)

2016-11-16 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-4752 389ac4668 -> dd24d316d


YARN-5885. [YARN-4752] Cleanup YARN-4752 for merge (Contributed by Karthik 
Kambatla via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd24d316
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd24d316
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd24d316

Branch: refs/heads/YARN-4752
Commit: dd24d316dfcf233257605d6ed464a19687fa97d1
Parents: 389ac46
Author: Daniel Templeton 
Authored: Wed Nov 16 17:18:33 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Nov 16 17:18:33 2016 -0800

--
 .../scheduler/fair/FSAppAttempt.java|  61 +--
 .../scheduler/fair/FSLeafQueue.java |  37 -
 .../scheduler/fair/FSPreemptionThread.java  |  21 ++-
 .../scheduler/fair/FSSchedulerNode.java |   8 +-
 .../scheduler/fair/FairScheduler.java   |   7 -
 .../scheduler/fair/TestFSAppStarvation.java |  48 --
 .../scheduler/fair/TestFSLeafQueue.java | 164 +--
 .../fair/TestQueueManagerRealScheduler.java | 128 +++
 8 files changed, 257 insertions(+), 217 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd24d316/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index f5bc2cd..39f4a3d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -28,6 +28,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -81,12 +82,14 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   private Resource fairShare = Resources.createResource(0, 0);
 
   // Preemption related variables
-  private Resource fairshareStarvation = Resources.none();
-  private Resource minshareStarvation = Resources.none();
   private final Resource preemptedResources = 
Resources.clone(Resources.none());
   private final Set containersToPreempt = new HashSet<>();
+  private Resource fairshareStarvation = Resources.none();
   private long lastTimeAtFairShare;
 
+  // minShareStarvation attributed to this application by the leaf queue
+  private Resource minshareStarvation = Resources.none();
+
   // Used to record node reservation by an app.
   // Key = RackName, Value = Set of Nodes reserved by app on rack
   private Map reservations = new HashMap<>();
@@ -149,7 +152,7 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
 
   // Remove from the list of containers
   liveContainers.remove(rmContainer.getContainerId());
-  removePreemption(rmContainer);
+  untrackContainerForPreemption(rmContainer);
 
   Resource containerResource = rmContainer.getContainer().getResource();
   RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER,
@@ -510,26 +513,42 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   }
 
   // Preemption related methods
+
+  /**
+   * Get overall starvation - fairshare and attributed minshare.
+   *
+   * @return total starvation attributed to this application
+   */
   Resource getStarvation() {
 return Resources.add(fairshareStarvation, minshareStarvation);
   }
 
+  /**
+   * Set the minshare attributed to this application. To be called only from
+   * {@link FSLeafQueue#updateStarvedApps}.
+   *
+   * @param starvation minshare starvation attributed to this app
+   */
   void setMinshareStarvation(Resource starvation) {
 this.minshareStarvation = starvation;
   }
 
+  /**
+   * Reset the minshare starvation attributed to this application. To be
+   * called only from {@link FSLeafQueue#updateStarvedApps}
+   */
   void resetMinshareStarvation() {

hadoop git commit: YARN-5819. [YARN-4752] Verify fairshare and minshare preemption (Contributed by Karthik Kambatla via Daniel Templeton)

2016-11-15 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-4752 214067405 -> 389ac4668


YARN-5819. [YARN-4752] Verify fairshare and minshare preemption (Contributed by 
Karthik Kambatla via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/389ac466
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/389ac466
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/389ac466

Branch: refs/heads/YARN-4752
Commit: 389ac46687e96a507d705ede52e6fec2850c09ba
Parents: 2140674
Author: Daniel Templeton 
Authored: Tue Nov 15 09:17:56 2016 -0800
Committer: Daniel Templeton 
Committed: Tue Nov 15 09:17:56 2016 -0800

--
 .../scheduler/fair/FSAppAttempt.java|  38 ++-
 .../scheduler/fair/FSPreemptionThread.java  |  23 +-
 .../scheduler/fair/FSSchedulerNode.java |  34 +++
 .../scheduler/fair/FairSchedulerTestBase.java   |  37 ++-
 .../scheduler/fair/TestFSAppStarvation.java |  21 +-
 .../fair/TestFairSchedulerPreemption.java   | 275 +++
 6 files changed, 389 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/389ac466/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 6b88bd0..f5bc2cd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -83,7 +83,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
   // Preemption related variables
   private Resource fairshareStarvation = Resources.none();
   private Resource minshareStarvation = Resources.none();
-  private Resource preemptedResources = Resources.createResource(0);
+  private final Resource preemptedResources = 
Resources.clone(Resources.none());
   private final Set containersToPreempt = new HashSet<>();
   private long lastTimeAtFairShare;
 
@@ -149,7 +149,7 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
 
   // Remove from the list of containers
   liveContainers.remove(rmContainer.getContainerId());
-  containersToPreempt.remove(rmContainer);
+  removePreemption(rmContainer);
 
   Resource containerResource = rmContainer.getContainer().getResource();
   RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER,
@@ -524,7 +524,17 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
 
   void addPreemption(RMContainer container) {
 containersToPreempt.add(container);
-Resources.addTo(preemptedResources, container.getAllocatedResource());
+synchronized (preemptedResources) {
+  Resources.addTo(preemptedResources, container.getAllocatedResource());
+}
+  }
+
+  void removePreemption(RMContainer container) {
+synchronized (preemptedResources) {
+  Resources.subtractFrom(preemptedResources,
+  container.getAllocatedResource());
+}
+containersToPreempt.remove(container);
   }
 
   Set getPreemptionContainers() {
@@ -533,7 +543,9 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   
 
   private Resource getPreemptedResources() {
-return preemptedResources;
+synchronized (preemptedResources) {
+  return preemptedResources;
+}
   }
 
   boolean canContainerBePreempted(RMContainer container) {
@@ -545,20 +557,30 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   return false;
 }
 
+if (containersToPreempt.contains(container)) {
+  // The container is already under consideration for preemption
+  return false;
+}
+
 // Check if any of the parent queues are not preemptable
 // TODO (KK): Propagate the "preemptable" flag all the way down to the app
 // to avoid recursing up every time.
-FSQueue queue = getQueue();
-while (!queue.getQueueName().equals("root")) {
-  if (!queue.isPreemptable()) {
+for (FSQueue q = getQueue();
+!q.getQueueName().equals("root");
+q = q.getParent()) {
+  if

[hadoop] Git Push Summary

2016-11-09 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/+branch-2.6.0 [deleted] e3496499e

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] Git Push Summary

2016-11-09 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6.0-DT [deleted] 37b2bc8c4

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] Git Push Summary [forced push!] [Forced Update!]

2016-11-09 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6.0 37b2bc8c4 -> e3496499e (forced update)

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] Git Push Summary

2016-11-09 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/+branch-2.6.0 [created] e3496499e

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] Git Push Summary

2016-11-09 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6.0-DT [created] 37b2bc8c4

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: Revert "CLOUDERA-BUILD. Set the Fair Scheduler as the default scheduler."

2016-11-09 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6.0 44fdd1414 -> 37b2bc8c4


Revert "CLOUDERA-BUILD. Set the Fair Scheduler as the default scheduler."

This reverts commit 44fdd141468f72082c36d24151327ada19da8e46.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37b2bc8c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37b2bc8c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37b2bc8c

Branch: refs/heads/branch-2.6.0
Commit: 37b2bc8c469902c7de3836a333f388a65977c650
Parents: 44fdd14
Author: Daniel Templeton 
Authored: Wed Nov 9 13:20:51 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Nov 9 13:20:51 2016 -0800

--
 .../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java   | 2 +-
 .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/37b2bc8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6b991e7..83cbfc5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -331,7 +331,7 @@ public class YarnConfiguration extends Configuration {
 RM_PREFIX + "scheduler.class";
  
   public static final String DEFAULT_RM_SCHEDULER = 
-  
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler";
+  
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler";
 
   /** RM set next Heartbeat interval for NM */
   public static final String RM_NM_HEARTBEAT_INTERVAL_MS =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/37b2bc8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index d47ddca..9d37196 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -224,7 +224,7 @@
   
 The class to use as the resource scheduler.
 yarn.resourcemanager.scheduler.class
-
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler
+
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
   
 
   


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[3/3] hadoop git commit: YARN-4329. [YARN-5437] Allow fetching exact reason as to why a submitted app is in ACCEPTED state in Fair Scheduler (Contributed by Yufei Gu)

2016-11-09 Thread templedf 
YARN-4329. [YARN-5437] Allow fetching exact reason as to why a submitted app
is in ACCEPTED state in Fair Scheduler (Contributed by Yufei Gu)

(cherry picked from commit 59ee8b7a88603e94b5661a8d5d088f7aa99fe049)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42147bbb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42147bbb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42147bbb

Branch: refs/heads/branch-2
Commit: 42147bbbf73996504704d563e541fa1efb1b3b42
Parents: ede9091
Author: Daniel Templeton 
Authored: Wed Nov 9 13:11:37 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Nov 9 13:18:28 2016 -0800

--
 .../scheduler/fair/FSAppAttempt.java| 71 +++-
 .../scheduler/fair/FairScheduler.java   |  4 +-
 .../scheduler/fair/MaxRunningAppsEnforcer.java  | 50 --
 .../fair/TestMaxRunningAppsEnforcer.java|  2 +-
 4 files changed, 103 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/42147bbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index bc988c9..0686bc2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -766,8 +766,18 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
 // The desired container won't fit here, so reserve
 if (isReservable(capability) &&
 reserve(request, node, reservedContainer, type, schedulerKey)) {
+  if (isWaitingForAMContainer()) {
+updateAMDiagnosticMsg(capability,
+" exceed the available resources of the node and the request is"
++ " reserved");
+  }
   return FairScheduler.CONTAINER_RESERVED;
 } else {
+  if (isWaitingForAMContainer()) {
+updateAMDiagnosticMsg(capability,
+" exceed the available resources of the node and the request 
cannot"
++ " be reserved");
+  }
   if (LOG.isDebugEnabled()) {
 LOG.debug("Couldn't creating reservation for " +
 getName() + ",at priority " +  request.getPriority());
@@ -920,23 +930,31 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
 ResourceRequest rackRequest = getResourceRequest(key, node.getRackName());
 ResourceRequest nodeRequest = getResourceRequest(key, node.getNodeName());
 
-return
-// There must be outstanding requests at the given priority:
+boolean ret = true;
+if (!(// There must be outstanding requests at the given priority:
 anyRequest != null && anyRequest.getNumContainers() > 0 &&
-// If locality relaxation is turned off at *-level, there must be a
-// non-zero request for the node's rack:
-(anyRequest.getRelaxLocality() ||
-(rackRequest != null && rackRequest.getNumContainers() > 0)) &&
-// If locality relaxation is turned off at rack-level, there must 
be a
-// non-zero request at the node:
-(rackRequest == null || rackRequest.getRelaxLocality() ||
-(nodeRequest != null && nodeRequest.getNumContainers() > 0)) &&
-// The requested container must be able to fit on the node:
-Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null,
-anyRequest.getCapability(),
-node.getRMNode().getTotalCapability()) &&
-// The requested container must fit in queue maximum share:
-getQueue().fitsInMaxShare(anyRequest.getCapability());
+// If locality relaxation is turned off at *-level, there must be a
+// non-zero request for the node's rack:
+(anyRequest.getRelaxLocality() ||
+(rackRequest != null && rackRequest.getNumContainers() > 0)) &&
+// If locality relaxation is turned off at rack-level, there must be a
+// non-zero request at the node:
+(rackRequest ==

[1/3] hadoop git commit: CLOUDERA-BUILD. Set the Fair Scheduler as the default scheduler.

2016-11-09 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 ede909144 -> 42147bbbf
  refs/heads/branch-2.6.0 e3496499e -> 44fdd1414
  refs/heads/trunk 822ae88f7 -> 59ee8b7a8


CLOUDERA-BUILD. Set the Fair Scheduler as the default scheduler.

(cherry picked from commit 2608afcdbb6efa3bbf496c2b70a7e2ccde4dd0ba)
(cherry picked from commit 30f7dad019e7de2b974360de97a9e841ea5663cc)
(cherry picked from commit e55941cca0971855b19a0d42c0ec510d4d464027)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44fdd141
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44fdd141
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44fdd141

Branch: refs/heads/branch-2.6.0
Commit: 44fdd141468f72082c36d24151327ada19da8e46
Parents: e349649
Author: Sandy Ryza 
Authored: Fri Jun 21 16:35:24 2013 -0700
Committer: Daniel Templeton 
Committed: Sun Sep 4 07:24:08 2016 -0700

--
 .../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java   | 2 +-
 .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/44fdd141/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 83cbfc5..6b991e7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -331,7 +331,7 @@ public class YarnConfiguration extends Configuration {
 RM_PREFIX + "scheduler.class";
  
   public static final String DEFAULT_RM_SCHEDULER = 
-  
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler";
+  
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler";
 
   /** RM set next Heartbeat interval for NM */
   public static final String RM_NM_HEARTBEAT_INTERVAL_MS =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44fdd141/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 9d37196..d47ddca 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -224,7 +224,7 @@
   
 The class to use as the resource scheduler.
 yarn.resourcemanager.scheduler.class
-
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
+
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler
   
 
   


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[2/3] hadoop git commit: YARN-4329. [YARN-5437] Allow fetching exact reason as to why a submitted app is in ACCEPTED state in Fair Scheduler (Contributed by Yufei Gu)

2016-11-09 Thread templedf 
YARN-4329. [YARN-5437] Allow fetching exact reason as to why a submitted app
is in ACCEPTED state in Fair Scheduler (Contributed by Yufei Gu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59ee8b7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59ee8b7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59ee8b7a

Branch: refs/heads/trunk
Commit: 59ee8b7a88603e94b5661a8d5d088f7aa99fe049
Parents: 822ae88
Author: Daniel Templeton 
Authored: Wed Nov 9 13:11:37 2016 -0800
Committer: Daniel Templeton 
Committed: Wed Nov 9 13:11:37 2016 -0800

--
 .../scheduler/fair/FSAppAttempt.java| 71 +++-
 .../scheduler/fair/FairScheduler.java   |  4 +-
 .../scheduler/fair/MaxRunningAppsEnforcer.java  | 50 --
 .../fair/TestMaxRunningAppsEnforcer.java|  2 +-
 4 files changed, 103 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/59ee8b7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 11922d9..df20117 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -766,8 +766,18 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
 // The desired container won't fit here, so reserve
 if (isReservable(capability) &&
 reserve(request, node, reservedContainer, type, schedulerKey)) {
+  if (isWaitingForAMContainer()) {
+updateAMDiagnosticMsg(capability,
+" exceed the available resources of the node and the request is"
++ " reserved");
+  }
   return FairScheduler.CONTAINER_RESERVED;
 } else {
+  if (isWaitingForAMContainer()) {
+updateAMDiagnosticMsg(capability,
+" exceed the available resources of the node and the request 
cannot"
++ " be reserved");
+  }
   if (LOG.isDebugEnabled()) {
 LOG.debug("Couldn't creating reservation for " +
 getName() + ",at priority " +  request.getPriority());
@@ -920,23 +930,31 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
 ResourceRequest rackRequest = getResourceRequest(key, node.getRackName());
 ResourceRequest nodeRequest = getResourceRequest(key, node.getNodeName());
 
-return
-// There must be outstanding requests at the given priority:
+boolean ret = true;
+if (!(// There must be outstanding requests at the given priority:
 anyRequest != null && anyRequest.getNumContainers() > 0 &&
-// If locality relaxation is turned off at *-level, there must be a
-// non-zero request for the node's rack:
-(anyRequest.getRelaxLocality() ||
-(rackRequest != null && rackRequest.getNumContainers() > 0)) &&
-// If locality relaxation is turned off at rack-level, there must 
be a
-// non-zero request at the node:
-(rackRequest == null || rackRequest.getRelaxLocality() ||
-(nodeRequest != null && nodeRequest.getNumContainers() > 0)) &&
-// The requested container must be able to fit on the node:
-Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null,
-anyRequest.getCapability(),
-node.getRMNode().getTotalCapability()) &&
-// The requested container must fit in queue maximum share:
-getQueue().fitsInMaxShare(anyRequest.getCapability());
+// If locality relaxation is turned off at *-level, there must be a
+// non-zero request for the node's rack:
+(anyRequest.getRelaxLocality() ||
+(rackRequest != null && rackRequest.getNumContainers() > 0)) &&
+// If locality relaxation is turned off at rack-level, there must be a
+// non-zero request at the node:
+(rackRequest == null || rackRequest.getRelaxLocality() ||
+(nodeRequest != null &&

hadoop git commit: YARN-5821. Drop left-over preemption-related code and clean up method visibilities in the Schedulable hierarchy (Contributed by Karthik Kambatla via Daniel Templeton)

2016-11-03 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-4752 5ad5085be -> b425ca250


YARN-5821. Drop left-over preemption-related code and clean up method 
visibilities in the Schedulable hierarchy
(Contributed by Karthik Kambatla via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b425ca25
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b425ca25
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b425ca25

Branch: refs/heads/YARN-4752
Commit: b425ca25044c7b1a3b786be1993788b8250c6356
Parents: 5ad5085
Author: Daniel Templeton 
Authored: Thu Nov 3 14:50:09 2016 -0700
Committer: Daniel Templeton 
Committed: Thu Nov 3 14:58:24 2016 -0700

--
 .../scheduler/fair/FSAppAttempt.java|  76 --
 .../scheduler/fair/FSLeafQueue.java | 102 +++
 .../scheduler/fair/FSParentQueue.java   |  46 ++---
 .../resourcemanager/scheduler/fair/FSQueue.java |  20 ++--
 .../scheduler/fair/Schedulable.java |  29 +++---
 .../scheduler/fair/FakeSchedulable.java |   5 -
 6 files changed, 62 insertions(+), 216 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b425ca25/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 39c8e13..60f701b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -18,12 +18,10 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
-import java.io.Serializable;
 import java.text.DecimalFormat;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.Comparator;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -81,7 +79,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
   private FairScheduler scheduler;
   private FSQueue fsQueue;
   private Resource fairShare = Resources.createResource(0, 0);
-  private RMContainerComparator comparator = new RMContainerComparator();
 
   // Preemption related variables
   private Resource fairshareStarvation = Resources.none();
@@ -121,7 +118,7 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
 this.resourceWeights = new ResourceWeights();
   }
 
-  public ResourceWeights getResourceWeights() {
+  ResourceWeights getResourceWeights() {
 return resourceWeights;
   }
 
@@ -132,7 +129,7 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
 return queue.getMetrics();
   }
 
-  public void containerCompleted(RMContainer rmContainer,
+  void containerCompleted(RMContainer rmContainer,
   ContainerStatus containerStatus, RMContainerEventType event) {
 try {
   writeLock.lock();
@@ -461,7 +458,7 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
* @param schedulerKey Scheduler Key
* @param level NodeType
*/
-  public void resetAllowedLocalityLevel(
+  void resetAllowedLocalityLevel(
   SchedulerRequestKey schedulerKey, NodeType level) {
 NodeType old;
 try {
@@ -483,45 +480,45 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   }
 
   // Preemption related methods
-  public Resource getStarvation() {
+  Resource getStarvation() {
 return Resources.add(fairshareStarvation, minshareStarvation);
   }
 
-  public void setMinshareStarvation(Resource starvation) {
+  void setMinshareStarvation(Resource starvation) {
 this.minshareStarvation = starvation;
   }
 
-  public void resetMinshareStarvation() {
+  void resetMinshareStarvation() {
 this.minshareStarvation = Resources.none();
   }
 
-  public void addPreemption(RMContainer container) {
+  void addPreemption(RMContainer container) {
 containersToPreempt.add(container);
 Resources.addTo(preemptedResources, container.getAllocatedResource());
   }
 
-  public Set getPreemptionContainers() {
+  Set

hadoop git commit: HADOOP-13667. Fix typing mistake of inline document in hadoop-metrics2.properties (Contributed by Rui Gao via Daniel Templeton)

2016-11-02 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk cb5cc0dc5 -> 0dc2a6a3a


HADOOP-13667. Fix typing mistake of inline document in 
hadoop-metrics2.properties (Contributed by Rui Gao via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0dc2a6a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0dc2a6a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0dc2a6a3

Branch: refs/heads/trunk
Commit: 0dc2a6a3a50add18e959475032264b04aff18fc4
Parents: cb5cc0d
Author: Daniel Templeton 
Authored: Wed Nov 2 06:52:27 2016 -0700
Committer: Daniel Templeton 
Committed: Wed Nov 2 06:53:33 2016 -0700

--
 .../src/main/conf/hadoop-metrics2.properties| 22 +---
 1 file changed, 19 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0dc2a6a3/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties 
b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties
index 0c09228..16fdcf0 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties
+++ 
b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties
@@ -50,9 +50,25 @@
 # If '*' all tags are used. If specifiying multiple tags separate them with 
 # commas. Note that the last segment of the property name is the context name.
 #
-#*.sink.ganglia.tagsForPrefix.jvm=ProcesName
-#*.sink.ganglia.tagsForPrefix.dfs=
-#*.sink.ganglia.tagsForPrefix.rpc=
+# A typical use of tags is separating the metrics by the HDFS rpc port
+# and HDFS service rpc port.
+# For example:
+#   With following HDFS configuration:
+#   dfs.namenode.rpc-address is set as namenodeAddress:9110
+#   dfs.namenode.servicerpc-address is set as namenodeAddress:9111
+#   If no tags are used, following metric would be gathered:
+#   rpc.rpc.NumOpenConnections
+#   If using "*.sink.ganglia.tagsForPrefix.rpc=port",
+#   following metrics would be gathered:
+#   rpc.rpc.port=9110.NumOpenConnections
+#   rpc.rpc.port=9111.NumOpenConnections
+#
+#*.sink.ganglia.tagsForPrefix.jvm=ProcessName
+#*.sink.ganglia.tagsForPrefix.dfs=HAState,IsOutOfSync
+#*.sink.ganglia.tagsForPrefix.rpc=port
+#*.sink.ganglia.tagsForPrefix.rpcdetailed=port
+#*.sink.ganglia.tagsForPrefix.metricssystem=*
+#*.sink.ganglia.tagsForPrefix.ugi=*
 #*.sink.ganglia.tagsForPrefix.mapred=
 
 #namenode.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5800. Delete LinuxContainerExecutor comment from yarn-default.xml (Contributed by Jan Hentschel via Daniel Templeton)

2016-10-31 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk a1761a841 -> 773c60bd7


YARN-5800. Delete LinuxContainerExecutor comment from yarn-default.xml 
(Contributed by Jan Hentschel via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/773c60bd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/773c60bd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/773c60bd

Branch: refs/heads/trunk
Commit: 773c60bd7bd00651dc3016799b424b9bd2233eb3
Parents: a1761a8
Author: Daniel Templeton 
Authored: Mon Oct 31 14:47:48 2016 -0700
Committer: Daniel Templeton 
Committed: Mon Oct 31 14:47:48 2016 -0700

--
 .../hadoop-yarn-common/src/main/resources/yarn-default.xml  | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/773c60bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 6c247b0..c265e86 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -965,7 +965,6 @@
 who will execute(launch) the containers.
 yarn.nodemanager.container-executor.class
 
org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor
-
   
 
   


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5793. Trim configuration values in DockerLinuxContainerRuntime (Contributed by Tianyin Xu via Daniel Templeton)

2016-10-31 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 c4ccafdaf -> 9d13a1316


YARN-5793. Trim configuration values in DockerLinuxContainerRuntime 
(Contributed by Tianyin Xu via Daniel Templeton)

(cherry picked from commit f3eb4c3c738204e099cbaa03471497c46530efbf)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d13a131
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d13a131
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d13a131

Branch: refs/heads/branch-2
Commit: 9d13a13160678506c28302f010a1728e722e79cc
Parents: c4ccafd
Author: Daniel Templeton 
Authored: Mon Oct 31 13:51:07 2016 -0700
Committer: Daniel Templeton 
Committed: Mon Oct 31 14:01:15 2016 -0700

--
 .../linux/runtime/DockerLinuxContainerRuntime.java| 14 --
 1 file changed, 8 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d13a131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index a3aff2f..b70a4e1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -219,9 +219,10 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
 dockerClient = new DockerClient(conf);
 allowedNetworks.clear();
 allowedNetworks.addAll(Arrays.asList(
-conf.getStrings(YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
+conf.getTrimmedStrings(
+YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
 YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS)));
-defaultNetwork = conf.get(
+defaultNetwork = conf.getTrimmed(
 YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK,
 YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK);
 
@@ -237,7 +238,7 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
   throw new ContainerExecutionException(message);
 }
 
-privilegedContainersAcl = new AccessControlList(conf.get(
+privilegedContainersAcl = new AccessControlList(conf.getTrimmed(
 YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL,
 YarnConfiguration.DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL));
   }
@@ -439,9 +440,10 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
 LOCALIZED_RESOURCES);
 @SuppressWarnings("unchecked")
 List userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
-Set capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
-YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
-YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
+Set capabilities = new HashSet<>(Arrays.asList(
+conf.getTrimmedStrings(
+YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
 
 @SuppressWarnings("unchecked")
 DockerRunCommand runCommand = new DockerRunCommand(containerIdStr,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5793. Trim configuration values in DockerLinuxContainerRuntime (Contributed by Tianyin Xu via Daniel Templeton)

2016-10-31 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 2528bea67 -> f3eb4c3c7


YARN-5793. Trim configuration values in DockerLinuxContainerRuntime 
(Contributed by Tianyin Xu via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3eb4c3c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3eb4c3c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3eb4c3c

Branch: refs/heads/trunk
Commit: f3eb4c3c738204e099cbaa03471497c46530efbf
Parents: 2528bea
Author: Daniel Templeton 
Authored: Mon Oct 31 13:51:07 2016 -0700
Committer: Daniel Templeton 
Committed: Mon Oct 31 13:51:07 2016 -0700

--
 .../linux/runtime/DockerLinuxContainerRuntime.java| 14 --
 1 file changed, 8 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3eb4c3c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index a3aff2f..b70a4e1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -219,9 +219,10 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
 dockerClient = new DockerClient(conf);
 allowedNetworks.clear();
 allowedNetworks.addAll(Arrays.asList(
-conf.getStrings(YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
+conf.getTrimmedStrings(
+YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
 YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS)));
-defaultNetwork = conf.get(
+defaultNetwork = conf.getTrimmed(
 YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK,
 YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK);
 
@@ -237,7 +238,7 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
   throw new ContainerExecutionException(message);
 }
 
-privilegedContainersAcl = new AccessControlList(conf.get(
+privilegedContainersAcl = new AccessControlList(conf.getTrimmed(
 YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL,
 YarnConfiguration.DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL));
   }
@@ -439,9 +440,10 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
 LOCALIZED_RESOURCES);
 @SuppressWarnings("unchecked")
 List userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
-Set capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
-YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
-YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
+Set capabilities = new HashSet<>(Arrays.asList(
+conf.getTrimmedStrings(
+YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
 
 @SuppressWarnings("unchecked")
 DockerRunCommand runCommand = new DockerRunCommand(containerIdStr,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-4396. Log the trace information on FSAppAttempt#assignContainer (Contributed by Yiqun Li via Daniel Templeton)

2016-10-31 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 adceebc40 -> ce13463e7


YARN-4396. Log the trace information on FSAppAttempt#assignContainer 
(Contributed by Yiqun Li via Daniel Templeton)

(cherry picked from commit 2528bea67ff80fae597f10e26c5f70d601af9fb1)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce13463e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce13463e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce13463e

Branch: refs/heads/branch-2
Commit: ce13463e7acd6eac3898d00f6f2db6d93e7f6f03
Parents: adceebc
Author: Daniel Templeton 
Authored: Mon Oct 31 13:34:25 2016 -0700
Committer: Daniel Templeton 
Committed: Mon Oct 31 13:36:24 2016 -0700

--
 .../scheduler/fair/FSAppAttempt.java| 57 +++-
 1 file changed, 56 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce13463e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 3555faa..bc988c9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -294,11 +294,27 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   rackLocalityThreshold;
 
   // Relax locality constraints once we've surpassed threshold.
-  if (getSchedulingOpportunities(schedulerKey) > (numNodes * threshold)) {
+  int schedulingOpportunities = getSchedulingOpportunities(schedulerKey);
+  double thresholdNum = numNodes * threshold;
+  if (schedulingOpportunities > thresholdNum) {
 if (allowed.equals(NodeType.NODE_LOCAL)) {
+  if (LOG.isTraceEnabled()) {
+LOG.trace("SchedulingOpportunities: " + schedulingOpportunities
++ ", nodeLocalityThreshold: " + thresholdNum
++ ", change allowedLocality from NODE_LOCAL to RACK_LOCAL"
++ ", priority: " + schedulerKey.getPriority()
++ ", app attempt id: " + this.attemptId);
+  }
   allowedLocalityLevel.put(schedulerKey, NodeType.RACK_LOCAL);
   resetSchedulingOpportunities(schedulerKey);
 } else if (allowed.equals(NodeType.RACK_LOCAL)) {
+  if (LOG.isTraceEnabled()) {
+LOG.trace("SchedulingOpportunities: " + schedulingOpportunities
++ ", rackLocalityThreshold: " + thresholdNum
++ ", change allowedLocality from RACK_LOCAL to OFF_SWITCH"
++ ", priority: " + schedulerKey.getPriority()
++ ", app attempt id: " + this.attemptId);
+  }
   allowedLocalityLevel.put(schedulerKey, NodeType.OFF_SWITCH);
   resetSchedulingOpportunities(schedulerKey);
 }
@@ -365,9 +381,23 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
 
   if (waitTime > thresholdTime) {
 if (allowed.equals(NodeType.NODE_LOCAL)) {
+  if (LOG.isTraceEnabled()) {
+LOG.trace("Waiting time: " + waitTime
++ " ms, nodeLocalityDelay time: " + nodeLocalityDelayMs + " ms"
++ ", change allowedLocality from NODE_LOCAL to RACK_LOCAL"
++ ", priority: " + schedulerKey.getPriority()
++ ", app attempt id: " + this.attemptId);
+  }
   allowedLocalityLevel.put(schedulerKey, NodeType.RACK_LOCAL);
   resetSchedulingOpportunities(schedulerKey, currentTimeMs);
 } else if (allowed.equals(NodeType.RACK_LOCAL)) {
+  if (LOG.isTraceEnabled()) {
+LOG.trace("Waiting time: " + waitTime
++ " ms, nodeLocalityDelay time: " + nodeLocalityDelayMs + " ms"
++ ", change allowedLocality from RACK_LOCAL to OFF_SWITCH"
++ ", priority: " + schedulerKey.getPriority()
++ ", app attempt id: " + this.attemptId);
+  }
   allowedLocalityLevel.put(schedulerKey, NodeType.OFF_SWITCH);

hadoop git commit: YARN-4396. Log the trace information on FSAppAttempt#assignContainer (Contributed by Yiqun Li via Daniel Templeton)

2016-10-31 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk cc2c993a8 -> 2528bea67


YARN-4396. Log the trace information on FSAppAttempt#assignContainer 
(Contributed by Yiqun Li via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2528bea6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2528bea6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2528bea6

Branch: refs/heads/trunk
Commit: 2528bea67ff80fae597f10e26c5f70d601af9fb1
Parents: cc2c993
Author: Daniel Templeton 
Authored: Mon Oct 31 13:34:25 2016 -0700
Committer: Daniel Templeton 
Committed: Mon Oct 31 13:34:25 2016 -0700

--
 .../scheduler/fair/FSAppAttempt.java| 57 +++-
 1 file changed, 56 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2528bea6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index cef4387..11922d9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -294,11 +294,27 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   rackLocalityThreshold;
 
   // Relax locality constraints once we've surpassed threshold.
-  if (getSchedulingOpportunities(schedulerKey) > (numNodes * threshold)) {
+  int schedulingOpportunities = getSchedulingOpportunities(schedulerKey);
+  double thresholdNum = numNodes * threshold;
+  if (schedulingOpportunities > thresholdNum) {
 if (allowed.equals(NodeType.NODE_LOCAL)) {
+  if (LOG.isTraceEnabled()) {
+LOG.trace("SchedulingOpportunities: " + schedulingOpportunities
++ ", nodeLocalityThreshold: " + thresholdNum
++ ", change allowedLocality from NODE_LOCAL to RACK_LOCAL"
++ ", priority: " + schedulerKey.getPriority()
++ ", app attempt id: " + this.attemptId);
+  }
   allowedLocalityLevel.put(schedulerKey, NodeType.RACK_LOCAL);
   resetSchedulingOpportunities(schedulerKey);
 } else if (allowed.equals(NodeType.RACK_LOCAL)) {
+  if (LOG.isTraceEnabled()) {
+LOG.trace("SchedulingOpportunities: " + schedulingOpportunities
++ ", rackLocalityThreshold: " + thresholdNum
++ ", change allowedLocality from RACK_LOCAL to OFF_SWITCH"
++ ", priority: " + schedulerKey.getPriority()
++ ", app attempt id: " + this.attemptId);
+  }
   allowedLocalityLevel.put(schedulerKey, NodeType.OFF_SWITCH);
   resetSchedulingOpportunities(schedulerKey);
 }
@@ -365,9 +381,23 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
 
   if (waitTime > thresholdTime) {
 if (allowed.equals(NodeType.NODE_LOCAL)) {
+  if (LOG.isTraceEnabled()) {
+LOG.trace("Waiting time: " + waitTime
++ " ms, nodeLocalityDelay time: " + nodeLocalityDelayMs + " ms"
++ ", change allowedLocality from NODE_LOCAL to RACK_LOCAL"
++ ", priority: " + schedulerKey.getPriority()
++ ", app attempt id: " + this.attemptId);
+  }
   allowedLocalityLevel.put(schedulerKey, NodeType.RACK_LOCAL);
   resetSchedulingOpportunities(schedulerKey, currentTimeMs);
 } else if (allowed.equals(NodeType.RACK_LOCAL)) {
+  if (LOG.isTraceEnabled()) {
+LOG.trace("Waiting time: " + waitTime
++ " ms, nodeLocalityDelay time: " + nodeLocalityDelayMs + " ms"
++ ", change allowedLocality from RACK_LOCAL to OFF_SWITCH"
++ ", priority: " + schedulerKey.getPriority()
++ ", app attempt id: " + this.attemptId);
+  }
   allowedLocalityLevel.put(schedulerKey, NodeType.OFF_SWITCH);
   resetSchedulingOpportunities(schedulerKey, currentTimeMs);
 }
@@

hadoop git commit: YARN-4907. Make all MockRM#waitForState consistent. (Contributed by Yufei Gu via Daniel Templeton)

2016-10-31 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk a9d68d2e8 -> cc2c993a8


YARN-4907. Make all MockRM#waitForState consistent. (Contributed by Yufei Gu 
via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc2c993a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc2c993a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc2c993a

Branch: refs/heads/trunk
Commit: cc2c993a8af6265b9881550501fd16f783519e03
Parents: a9d68d2
Author: Daniel Templeton 
Authored: Mon Oct 31 13:18:33 2016 -0700
Committer: Daniel Templeton 
Committed: Mon Oct 31 13:20:56 2016 -0700

--
 .../yarn/server/resourcemanager/MockRM.java | 44 +---
 1 file changed, 19 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc2c993a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 25a8288..58bb721 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -272,9 +272,10 @@ public class MockRM extends ResourceManager {
 
   public void waitForContainerToComplete(RMAppAttempt attempt,
   NMContainerStatus completedContainer) throws InterruptedException {
-while (true) {
+int timeWaiting = 0;
+while (timeWaiting < TIMEOUT_MS_FOR_CONTAINER_AND_NODE) {
   List containers = attempt.getJustFinishedContainers();
-  System.out.println("Received completed containers " + containers);
+  LOG.info("Received completed containers " + containers);
   for (ContainerStatus container : containers) {
 if (container.getContainerId().equals(
   completedContainer.getContainerId())) {
@@ -282,6 +283,7 @@ public class MockRM extends ResourceManager {
 }
   }
   Thread.sleep(WAIT_MS_PER_LOOP);
+  timeWaiting += WAIT_MS_PER_LOOP;
 }
   }
 
@@ -289,11 +291,16 @@ public class MockRM extends ResourceManager {
   MockNM nm) throws Exception {
 RMApp app = getRMContext().getRMApps().get(appId);
 Assert.assertNotNull(app);
+int timeWaiting = 0;
 while (app.getAppAttempts().size() != attemptSize) {
-  System.out.println("Application " + appId
+  if (timeWaiting >= TIMEOUT_MS_FOR_ATTEMPT) {
+break;
+  }
+  LOG.info("Application " + appId
   + " is waiting for AM to restart. Current has "
   + app.getAppAttempts().size() + " attempts.");
   Thread.sleep(WAIT_MS_PER_LOOP);
+  timeWaiting += WAIT_MS_PER_LOOP;
 }
 return launchAndRegisterAM(app, this, nm);
   }
@@ -375,7 +382,7 @@ public class MockRM extends ResourceManager {
 nm.nodeHeartbeat(true);
   }
   container = getResourceScheduler().getRMContainer(containerId);
-  System.out.println("Waiting for container " + containerId + " to be "
+  LOG.info("Waiting for container " + containerId + " to be "
   + containerState + ", container is null right now.");
   Thread.sleep(WAIT_MS_PER_LOOP);
   timeWaiting += WAIT_MS_PER_LOOP;
@@ -386,7 +393,7 @@ public class MockRM extends ResourceManager {
 return false;
   }
 
-  System.out.println("Container : " + containerId + " State is : "
+  LOG.info("Container : " + containerId + " State is : "
   + container.getState() + " Waiting for state : " + containerState);
   for (MockNM nm : nms) {
 nm.nodeHeartbeat(true);
@@ -395,7 +402,7 @@ public class MockRM extends ResourceManager {
   timeWaiting += WAIT_MS_PER_LOOP;
 }
 
-System.out.println("Container State is : " + container.getState());
+LOG.info("Container State is : " + container.getState());
 return true;
   }
 
@@ -724,13 +731,13 @@ public class MockRM extends ResourceManager {
 break;
   }
 
-  System.out.println("Node State is : " + node.getState()
+  LOG.info("Node State is : " + node.getState()
   + " Waiting for state : " + finalState);
   Thread.sleep(WAIT_MS_PER_LOOP);
   timeWaiting += WAIT_MS_PER_LOOP;
 }
 
-

hadoop git commit: YARN-4710. Reduce logging application reserved debug info in FSAppAttempt#assignContainer (Contributed by Yiqun Lin via Daniel Templeton)

2016-10-27 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 9449519a2 -> b98fc8249


YARN-4710. Reduce logging application reserved debug info in 
FSAppAttempt#assignContainer (Contributed by Yiqun Lin via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b98fc824
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b98fc824
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b98fc824

Branch: refs/heads/trunk
Commit: b98fc8249f0576e7b4e230ffc3cec5a20eefc543
Parents: 9449519
Author: Daniel Templeton 
Authored: Thu Oct 27 14:35:38 2016 -0700
Committer: Daniel Templeton 
Committed: Thu Oct 27 14:42:19 2016 -0700

--
 .../yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b98fc824/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 3555faa..cef4387 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -771,8 +771,8 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   }
 
   private Resource assignContainer(FSSchedulerNode node, boolean reserved) {
-if (LOG.isDebugEnabled()) {
-  LOG.debug("Node offered to app: " + getName() + " reserved: " + 
reserved);
+if (LOG.isTraceEnabled()) {
+  LOG.trace("Node offered to app: " + getName() + " reserved: " + 
reserved);
 }
 
 Collection keysToTry = (reserved) ?


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-4710. Reduce logging application reserved debug info in FSAppAttempt#assignContainer (Contributed by Yiqun Lin via Daniel Templeton)

2016-10-27 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/official [created] bfa6891e7


YARN-4710. Reduce logging application reserved debug info in 
FSAppAttempt#assignContainer (Contributed by Yiqun Lin via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bfa6891e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bfa6891e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bfa6891e

Branch: refs/heads/official
Commit: bfa6891e7d6bfb1798a19c19c1b0b28cb5f47e27
Parents: 9449519
Author: Daniel Templeton 
Authored: Thu Oct 27 14:51:36 2016 -0700
Committer: Daniel Templeton 
Committed: Thu Oct 27 14:51:36 2016 -0700

--
 .../yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfa6891e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 3555faa..cef4387 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -771,8 +771,8 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   }
 
   private Resource assignContainer(FSSchedulerNode node, boolean reserved) {
-if (LOG.isDebugEnabled()) {
-  LOG.debug("Node offered to app: " + getName() + " reserved: " + 
reserved);
+if (LOG.isTraceEnabled()) {
+  LOG.trace("Node offered to app: " + getName() + " reserved: " + 
reserved);
 }
 
 Collection keysToTry = (reserved) ?


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[2/2] hadoop git commit: [YARN-4752] YARN-5605. Preempt containers (all on one node) to meet the requirement of starved applications (Contributed by Karthik Kambatla via Daniel Templeton)

2016-09-22 Thread templedf 
[YARN-4752] YARN-5605. Preempt containers (all on one node) to meet the 
requirement of starved applications (Contributed by Karthik Kambatla via Daniel 
Templeton)

Change-Id: Iee0962377d019dd64dc69a020725d2eaf360858c


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0692dfe1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0692dfe1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0692dfe1

Branch: refs/heads/YARN-4752
Commit: 0692dfe1874ed3707e15aa9180024976ee0b9112
Parents: 40acace
Author: Daniel Templeton 
Authored: Thu Sep 22 14:08:15 2016 -0700
Committer: Daniel Templeton 
Committed: Thu Sep 22 14:12:04 2016 -0700

--
 .../hadoop/yarn/util/resource/Resources.java|4 +
 .../scheduler/AppSchedulingInfo.java|   17 +
 .../scheduler/fair/FSAppAttempt.java|  111 +-
 .../scheduler/fair/FSContext.java   |   54 +
 .../scheduler/fair/FSLeafQueue.java |  188 ++-
 .../scheduler/fair/FSParentQueue.java   |6 +-
 .../scheduler/fair/FSPreemptionThread.java  |  172 ++
 .../resourcemanager/scheduler/fair/FSQueue.java |   12 +-
 .../scheduler/fair/FSStarvedApps.java   |   75 +
 .../scheduler/fair/FairScheduler.java   |  261 +--
 .../scheduler/fair/FairSchedulerTestBase.java   |2 +-
 .../scheduler/fair/TestFSLeafQueue.java |   17 +-
 .../fair/TestFairSchedulerPreemption.java   | 1483 --
 13 files changed, 595 insertions(+), 1807 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0692dfe1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 760b0ea..462e02a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -143,6 +143,10 @@ public class Resources {
   public static Resource none() {
 return NONE;
   }
+
+  public static boolean isNone(Resource other) {
+return NONE.equals(other);
+  }
   
   public static Resource unbounded() {
 return UNBOUNDED;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0692dfe1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 39820f7..0302ad7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -560,6 +560,23 @@ public class AppSchedulingInfo {
   }
 
   /**
+   * Method to return the next resource request to be serviced.
+   *
+   * In the initial implementation, we just pick any {@link ResourceRequest}
+   * corresponding to the highest priority.
+   *
+   * @return next {@link ResourceRequest} to allocate resources for.
+   */
+  @Unstable
+  public synchronized ResourceRequest getNextResourceRequest() {
+for (ResourceRequest rr:
+resourceRequestMap.get(schedulerKeys.firstKey()).values()) {
+  return rr;
+}
+return null;
+  }
+
+  /**
* Returns if the place (node/rack today) is either blacklisted by the
* application (user) or the system
*

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0692dfe1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git

[1/2] hadoop git commit: [YARN-4752] YARN-5605. Preempt containers (all on one node) to meet the requirement of starved applications (Contributed by Karthik Kambatla via Daniel Templeton)

2016-09-22 Thread templedf 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-4752 40acacee0 -> 0692dfe18


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0692dfe1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
deleted file mode 100644
index 2cbe507..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
+++ /dev/null
@@ -1,1483 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
-import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerRequestKey;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-.TestUtils;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
-import org.apache.hadoop.yarn.util.ControlledClock;
-import org.apache.hadoop.yarn.util.resource.Resources;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
-  private final static String ALLOC_FILE = new File(TEST_DIR,
-  TestFairSchedulerPreemption.class.getName() + ".xml").getAbsolutePath();
-
-  private ControlledClock clock;
-
-  private static class StubbedFairScheduler extends FairScheduler {
-public long lastPreemptMemory = -1;
-
-@Override
-protected void preemptResources(Resource toPreempt) {
-  lastPreemptMemory = toPreempt.getMemorySize();
-}
-
-public void resetLastPreemptResources() {
-  lastPreemptMemory = -1;
-}
-  }
-
-  public Configuration createConfiguration() {
-Configuration conf = super.createConfiguration();
-conf.setClass(YarnConfiguration.RM_SCHEDULER, StubbedFairScheduler.class,
-ResourceScheduler.class);
-

<    1   2   3   4   5   6