[19/50] [abbrv] hadoop git commit: YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG log. (Yufei Gu via rchiang)

2017-03-16 Thread inigoiri 
YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG 
log. (Yufei Gu via rchiang)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4db9cc70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4db9cc70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4db9cc70

Branch: refs/heads/HDFS-10467
Commit: 4db9cc70d0178703fb28f451eb84d97f2bf63af8
Parents: 229c7c9
Author: Ray Chiang 
Authored: Fri Mar 10 16:13:31 2017 -0800
Committer: Ray Chiang 
Committed: Fri Mar 10 16:13:31 2017 -0800

--
 .../src/main/conf/log4j.properties  |  9 +++
 .../scheduler/fair/FSAppAttempt.java| 49 +++---
 .../scheduler/fair/FSLeafQueue.java | 21 ++
 .../scheduler/fair/FSParentQueue.java   | 21 ++
 .../resourcemanager/scheduler/fair/FSQueue.java | 41 ++-
 .../scheduler/fair/FairScheduler.java   | 28 +---
 .../scheduler/fair/TestFairScheduler.java   | 71 
 7 files changed, 206 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db9cc70/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties 
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index b8c84e7..6026763 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -321,3 +321,12 @@ 
log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender
 log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval}
 log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}
 log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
+
+# Fair scheduler requests log on state dump
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSLOGGER
+log4j.appender.FSLOGGER=org.apache.log4j.RollingFileAppender
+log4j.appender.FSLOGGER.File=${hadoop.log.dir}/fairscheduler-statedump.log
+log4j.appender.FSLOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.FSLOGGER.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.FSLOGGER.MaxFileSize=${hadoop.log.maxfilesize}
+log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db9cc70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 3a9c94e..ccfcffb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -835,25 +835,27 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   return capability;
 }
 
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Resource request: " + capability + " exceeds the available"
+  + " resources of the node.");
+}
+
 // The desired container won't fit here, so reserve
 if (isReservable(capability) &&
 reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer,
 type, schedulerKey)) {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the available resources of the node and the request is"
-+ " reserved");
+  updateAMDiagnosticMsg(capability, " exceeds the available resources of "
+  + "the node and the request is reserved)");
+  if (LOG.isDebugEnabled()) {
+LOG.debug(getName() + "'s resource request is reserved.");
   }
   return FairScheduler.CONTAINER_RESERVED;
 } else {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the available resources of the node and the request 
cannot"
-+ " be

[15/33] hadoop git commit: YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG log. (Yufei Gu via rchiang)

2017-03-14 Thread jhung 
YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG 
log. (Yufei Gu via rchiang)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4db9cc70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4db9cc70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4db9cc70

Branch: refs/heads/YARN-5734
Commit: 4db9cc70d0178703fb28f451eb84d97f2bf63af8
Parents: 229c7c9
Author: Ray Chiang 
Authored: Fri Mar 10 16:13:31 2017 -0800
Committer: Ray Chiang 
Committed: Fri Mar 10 16:13:31 2017 -0800

--
 .../src/main/conf/log4j.properties  |  9 +++
 .../scheduler/fair/FSAppAttempt.java| 49 +++---
 .../scheduler/fair/FSLeafQueue.java | 21 ++
 .../scheduler/fair/FSParentQueue.java   | 21 ++
 .../resourcemanager/scheduler/fair/FSQueue.java | 41 ++-
 .../scheduler/fair/FairScheduler.java   | 28 +---
 .../scheduler/fair/TestFairScheduler.java   | 71 
 7 files changed, 206 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db9cc70/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties 
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index b8c84e7..6026763 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -321,3 +321,12 @@ 
log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender
 log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval}
 log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}
 log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
+
+# Fair scheduler requests log on state dump
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSLOGGER
+log4j.appender.FSLOGGER=org.apache.log4j.RollingFileAppender
+log4j.appender.FSLOGGER.File=${hadoop.log.dir}/fairscheduler-statedump.log
+log4j.appender.FSLOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.FSLOGGER.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.FSLOGGER.MaxFileSize=${hadoop.log.maxfilesize}
+log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db9cc70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 3a9c94e..ccfcffb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -835,25 +835,27 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   return capability;
 }
 
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Resource request: " + capability + " exceeds the available"
+  + " resources of the node.");
+}
+
 // The desired container won't fit here, so reserve
 if (isReservable(capability) &&
 reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer,
 type, schedulerKey)) {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the available resources of the node and the request is"
-+ " reserved");
+  updateAMDiagnosticMsg(capability, " exceeds the available resources of "
+  + "the node and the request is reserved)");
+  if (LOG.isDebugEnabled()) {
+LOG.debug(getName() + "'s resource request is reserved.");
   }
   return FairScheduler.CONTAINER_RESERVED;
 } else {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the available resources of the node and the request 
cannot"
-+ " be

hadoop git commit: YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG log. (Yufei Gu via rchiang)

2017-03-14 Thread rchiang 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 0e7879052 -> fa59f4e49


YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG 
log. (Yufei Gu via rchiang)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa59f4e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa59f4e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa59f4e4

Branch: refs/heads/branch-2
Commit: fa59f4e4907d2c37841d59656d79b3162e774310
Parents: 0e78790
Author: Ray Chiang 
Authored: Tue Mar 14 14:45:13 2017 -0700
Committer: Ray Chiang 
Committed: Tue Mar 14 14:45:13 2017 -0700

--
 .../src/main/conf/log4j.properties  |  9 +++
 .../scheduler/fair/FSAppAttempt.java| 51 +++---
 .../scheduler/fair/FSLeafQueue.java | 21 ++
 .../scheduler/fair/FSParentQueue.java   | 21 ++
 .../resourcemanager/scheduler/fair/FSQueue.java | 41 ++-
 .../scheduler/fair/FairScheduler.java   | 28 +---
 .../scheduler/fair/TestFairScheduler.java   | 72 
 7 files changed, 209 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa59f4e4/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties 
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index 95afc61..7c02b20 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -321,3 +321,12 @@ 
log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender
 log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval}
 log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}
 log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
+
+# Fair scheduler requests log on state dump
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSLOGGER
+log4j.appender.FSLOGGER=org.apache.log4j.RollingFileAppender
+log4j.appender.FSLOGGER.File=${hadoop.log.dir}/fairscheduler-statedump.log
+log4j.appender.FSLOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.FSLOGGER.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.FSLOGGER.MaxFileSize=${hadoop.log.maxfilesize}
+log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa59f4e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 60902a2..2e08343 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -834,24 +834,26 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   return capability;
 }
 
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Resource request: " + capability + " exceeds the available"
+  + " resources of the node.");
+}
+
 // The desired container won't fit here, so reserve
 if (isReservable(capability) &&
 reserve(request, node, reservedContainer, type, schedulerKey)) {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the available resources of the node and the request is"
-+ " reserved");
+  updateAMDiagnosticMsg(capability, " exceeds the available resources of "
+  + "the node and the request is reserved)");
+  if (LOG.isDebugEnabled()) {
+LOG.debug(getName() + "'s resource request is reserved.");
   }
   return FairScheduler.CONTAINER_RESERVED;
 } else {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the available resources of the node and the

[47/50] [abbrv] hadoop git commit: YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG log. (Yufei Gu via rchiang)

2017-03-13 Thread stevel 
YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG 
log. (Yufei Gu via rchiang)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4db9cc70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4db9cc70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4db9cc70

Branch: refs/heads/HADOOP-13345
Commit: 4db9cc70d0178703fb28f451eb84d97f2bf63af8
Parents: 229c7c9
Author: Ray Chiang 
Authored: Fri Mar 10 16:13:31 2017 -0800
Committer: Ray Chiang 
Committed: Fri Mar 10 16:13:31 2017 -0800

--
 .../src/main/conf/log4j.properties  |  9 +++
 .../scheduler/fair/FSAppAttempt.java| 49 +++---
 .../scheduler/fair/FSLeafQueue.java | 21 ++
 .../scheduler/fair/FSParentQueue.java   | 21 ++
 .../resourcemanager/scheduler/fair/FSQueue.java | 41 ++-
 .../scheduler/fair/FairScheduler.java   | 28 +---
 .../scheduler/fair/TestFairScheduler.java   | 71 
 7 files changed, 206 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db9cc70/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties 
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index b8c84e7..6026763 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -321,3 +321,12 @@ 
log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender
 log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval}
 log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}
 log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
+
+# Fair scheduler requests log on state dump
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSLOGGER
+log4j.appender.FSLOGGER=org.apache.log4j.RollingFileAppender
+log4j.appender.FSLOGGER.File=${hadoop.log.dir}/fairscheduler-statedump.log
+log4j.appender.FSLOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.FSLOGGER.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.FSLOGGER.MaxFileSize=${hadoop.log.maxfilesize}
+log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db9cc70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 3a9c94e..ccfcffb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -835,25 +835,27 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   return capability;
 }
 
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Resource request: " + capability + " exceeds the available"
+  + " resources of the node.");
+}
+
 // The desired container won't fit here, so reserve
 if (isReservable(capability) &&
 reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer,
 type, schedulerKey)) {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the available resources of the node and the request is"
-+ " reserved");
+  updateAMDiagnosticMsg(capability, " exceeds the available resources of "
+  + "the node and the request is reserved)");
+  if (LOG.isDebugEnabled()) {
+LOG.debug(getName() + "'s resource request is reserved.");
   }
   return FairScheduler.CONTAINER_RESERVED;
 } else {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the available resources of the node and the request 
cannot"
-+ "

[16/50] [abbrv] hadoop git commit: YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG log. (Yufei Gu via rchiang)

2017-03-10 Thread subru 
YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG 
log. (Yufei Gu via rchiang)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4db9cc70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4db9cc70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4db9cc70

Branch: refs/heads/YARN-2915
Commit: 4db9cc70d0178703fb28f451eb84d97f2bf63af8
Parents: 229c7c9
Author: Ray Chiang 
Authored: Fri Mar 10 16:13:31 2017 -0800
Committer: Ray Chiang 
Committed: Fri Mar 10 16:13:31 2017 -0800

--
 .../src/main/conf/log4j.properties  |  9 +++
 .../scheduler/fair/FSAppAttempt.java| 49 +++---
 .../scheduler/fair/FSLeafQueue.java | 21 ++
 .../scheduler/fair/FSParentQueue.java   | 21 ++
 .../resourcemanager/scheduler/fair/FSQueue.java | 41 ++-
 .../scheduler/fair/FairScheduler.java   | 28 +---
 .../scheduler/fair/TestFairScheduler.java   | 71 
 7 files changed, 206 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db9cc70/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties 
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index b8c84e7..6026763 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -321,3 +321,12 @@ 
log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender
 log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval}
 log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}
 log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
+
+# Fair scheduler requests log on state dump
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSLOGGER
+log4j.appender.FSLOGGER=org.apache.log4j.RollingFileAppender
+log4j.appender.FSLOGGER.File=${hadoop.log.dir}/fairscheduler-statedump.log
+log4j.appender.FSLOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.FSLOGGER.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.FSLOGGER.MaxFileSize=${hadoop.log.maxfilesize}
+log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db9cc70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 3a9c94e..ccfcffb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -835,25 +835,27 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   return capability;
 }
 
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Resource request: " + capability + " exceeds the available"
+  + " resources of the node.");
+}
+
 // The desired container won't fit here, so reserve
 if (isReservable(capability) &&
 reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer,
 type, schedulerKey)) {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the available resources of the node and the request is"
-+ " reserved");
+  updateAMDiagnosticMsg(capability, " exceeds the available resources of "
+  + "the node and the request is reserved)");
+  if (LOG.isDebugEnabled()) {
+LOG.debug(getName() + "'s resource request is reserved.");
   }
   return FairScheduler.CONTAINER_RESERVED;
 } else {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the available resources of the node and the request 
cannot"
-+ " be

hadoop git commit: YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG log. (Yufei Gu via rchiang)

2017-03-10 Thread rchiang 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 229c7c9f8 -> 4db9cc70d


YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG 
log. (Yufei Gu via rchiang)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4db9cc70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4db9cc70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4db9cc70

Branch: refs/heads/trunk
Commit: 4db9cc70d0178703fb28f451eb84d97f2bf63af8
Parents: 229c7c9
Author: Ray Chiang 
Authored: Fri Mar 10 16:13:31 2017 -0800
Committer: Ray Chiang 
Committed: Fri Mar 10 16:13:31 2017 -0800

--
 .../src/main/conf/log4j.properties  |  9 +++
 .../scheduler/fair/FSAppAttempt.java| 49 +++---
 .../scheduler/fair/FSLeafQueue.java | 21 ++
 .../scheduler/fair/FSParentQueue.java   | 21 ++
 .../resourcemanager/scheduler/fair/FSQueue.java | 41 ++-
 .../scheduler/fair/FairScheduler.java   | 28 +---
 .../scheduler/fair/TestFairScheduler.java   | 71 
 7 files changed, 206 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db9cc70/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties 
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index b8c84e7..6026763 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -321,3 +321,12 @@ 
log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender
 log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval}
 log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}
 log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
+
+# Fair scheduler requests log on state dump
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSLOGGER
+log4j.appender.FSLOGGER=org.apache.log4j.RollingFileAppender
+log4j.appender.FSLOGGER.File=${hadoop.log.dir}/fairscheduler-statedump.log
+log4j.appender.FSLOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.FSLOGGER.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.FSLOGGER.MaxFileSize=${hadoop.log.maxfilesize}
+log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db9cc70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 3a9c94e..ccfcffb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -835,25 +835,27 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   return capability;
 }
 
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Resource request: " + capability + " exceeds the available"
+  + " resources of the node.");
+}
+
 // The desired container won't fit here, so reserve
 if (isReservable(capability) &&
 reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer,
 type, schedulerKey)) {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the available resources of the node and the request is"
-+ " reserved");
+  updateAMDiagnosticMsg(capability, " exceeds the available resources of "
+  + "the node and the request is reserved)");
+  if (LOG.isDebugEnabled()) {
+LOG.debug(getName() + "'s resource request is reserved.");
   }
   return FairScheduler.CONTAINER_RESERVED;
 } else {
-  if (isWaitingForAMContainer()) {
-updateAMDiagnosticMsg(capability,
-" exceed the