hadoop git commit: YARN-6015. AsyncDispatcher thread name can be set to improved debugging. Contributed by Ajith S.

[naganarasimha_gr]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 020316458 -> a59df1575


YARN-6015. AsyncDispatcher thread name can be set to improved debugging. 
Contributed by Ajith S.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a59df157
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a59df157
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a59df157

Branch: refs/heads/trunk
Commit: a59df15757fac7f917cb96fc8fcfeb7017475e4f
Parents: 0203164
Author: Naganarasimha 
Authored: Sat Jan 7 08:11:58 2017 +0530
Committer: Naganarasimha 
Committed: Sat Jan 7 08:11:58 2017 +0530

--
 .../apache/hadoop/yarn/event/AsyncDispatcher.java   | 16 +++-
 .../yarn/nodelabels/CommonNodeLabelsManager.java|  2 +-
 .../hadoop/yarn/server/nodemanager/NodeManager.java |  2 +-
 .../containermanager/ContainerManagerImpl.java  |  2 +-
 .../timelineservice/NMTimelinePublisher.java|  2 +-
 .../server/resourcemanager/ResourceManager.java |  2 +-
 .../ahs/RMApplicationHistoryWriter.java |  2 +-
 .../metrics/AbstractSystemMetricsPublisher.java |  2 +-
 .../resourcemanager/recovery/RMStateStore.java  |  2 +-
 9 files changed, 23 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59df157/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index a3d306f..913b8ed 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -74,6 +74,11 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   protected final Map eventDispatchers;
   private boolean exitOnDispatchException;
 
+  /**
+   * The thread name for dispatcher.
+   */
+  private String dispatcherThreadName = "AsyncDispatcher event handler";
+
   public AsyncDispatcher() {
 this(new LinkedBlockingQueue());
   }
@@ -84,6 +89,15 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 this.eventDispatchers = new HashMap();
   }
 
+  /**
+   * Set a name for this dispatcher thread.
+   * @param dispatcherName name of the dispatcher thread
+   */
+  public AsyncDispatcher(String dispatcherName) {
+this();
+dispatcherThreadName = dispatcherName;
+  }
+
   Runnable createThread() {
 return new Runnable() {
   @Override
@@ -130,7 +144,7 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 //start all the components
 super.serviceStart();
 eventHandlingThread = new Thread(createThread());
-eventHandlingThread.setName("AsyncDispatcher event handler");
+eventHandlingThread.setName(dispatcherThreadName);
 eventHandlingThread.start();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a59df157/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 1a83632..f3f4ba0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -211,7 +211,7 @@ public class CommonNodeLabelsManager extends 
AbstractService {
   // for UT purpose
   protected void initDispatcher(Configuration conf) {
 // create async handler
-dispatcher = new AsyncDispatcher();
+dispatcher = new AsyncDispatcher("NodeLabelManager dispatcher");
 AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher;
 asyncDispatcher.init(conf);
 asyncDispatcher.setDrainEventsOnStop();

hadoop git commit: YARN-6009. Skip validating app timeout value on recovery. Contributed by Rohith Sharma K S

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 81da7d1d3 -> ba5d48236


YARN-6009. Skip validating app timeout value on recovery. Contributed by Rohith 
Sharma K S

(cherry picked from commit 020316458dfe6059b700f8d93a9791e4cb817b3f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba5d4823
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba5d4823
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba5d4823

Branch: refs/heads/branch-2
Commit: ba5d482366fad3e2187cc4c0f806e2af864fef40
Parents: 81da7d1
Author: Jian He 
Authored: Fri Jan 6 18:19:09 2017 -0800
Committer: Jian He 
Committed: Fri Jan 6 18:27:36 2017 -0800

--
 .../hadoop/yarn/server/resourcemanager/RMAppManager.java| 9 -
 1 file changed, 4 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba5d4823/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index d18a5f6..a808445 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -340,13 +340,16 @@ public class RMAppManager implements 
EventHandler,
   private RMAppImpl createAndPopulateNewRMApp(
   ApplicationSubmissionContext submissionContext, long submitTime,
   String user, boolean isRecovery) throws YarnException {
-// Do queue mapping
 if (!isRecovery) {
+  // Do queue mapping
   if (rmContext.getQueuePlacementManager() != null) {
 // We only do queue mapping when it's a new application
 rmContext.getQueuePlacementManager().placeApplication(
 submissionContext, user);
   }
+  // fail the submission if configured application timeout value is invalid
+  RMServerUtils.validateApplicationTimeouts(
+  submissionContext.getApplicationTimeouts());
 }
 
 ApplicationId applicationId = submissionContext.getApplicationId();
@@ -388,10 +391,6 @@ public class RMAppManager implements 
EventHandler,
   }
 }
 
-// fail the submission if configured application timeout value is invalid
-RMServerUtils.validateApplicationTimeouts(
-submissionContext.getApplicationTimeouts());
-
 // Create RMApp
 RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf,
 submissionContext.getApplicationName(), user,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6009. Skip validating app timeout value on recovery. Contributed by Rohith Sharma K S

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 71a4acf74 -> 020316458


YARN-6009. Skip validating app timeout value on recovery. Contributed by Rohith 
Sharma K S


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02031645
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02031645
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02031645

Branch: refs/heads/trunk
Commit: 020316458dfe6059b700f8d93a9791e4cb817b3f
Parents: 71a4acf
Author: Jian He 
Authored: Fri Jan 6 18:19:09 2017 -0800
Committer: Jian He 
Committed: Fri Jan 6 18:19:09 2017 -0800

--
 .../hadoop/yarn/server/resourcemanager/RMAppManager.java| 9 -
 1 file changed, 4 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/02031645/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index bc21952..9dd544e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -342,13 +342,16 @@ public class RMAppManager implements 
EventHandler,
   private RMAppImpl createAndPopulateNewRMApp(
   ApplicationSubmissionContext submissionContext, long submitTime,
   String user, boolean isRecovery, long startTime) throws YarnException {
-// Do queue mapping
 if (!isRecovery) {
+  // Do queue mapping
   if (rmContext.getQueuePlacementManager() != null) {
 // We only do queue mapping when it's a new application
 rmContext.getQueuePlacementManager().placeApplication(
 submissionContext, user);
   }
+  // fail the submission if configured application timeout value is invalid
+  RMServerUtils.validateApplicationTimeouts(
+  submissionContext.getApplicationTimeouts());
 }
 
 ApplicationId applicationId = submissionContext.getApplicationId();
@@ -390,10 +393,6 @@ public class RMAppManager implements 
EventHandler,
   }
 }
 
-// fail the submission if configured application timeout value is invalid
-RMServerUtils.validateApplicationTimeouts(
-submissionContext.getApplicationTimeouts());
-
 // Create RMApp
 RMAppImpl application =
 new RMAppImpl(applicationId, rmContext, this.conf,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5991. Yarn Distributed Shell does not print throwable t to App Master When failed to start container (Contributed by Jim Frankola via Daniel Templeton)

[templedf]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 2977bc6a1 -> 71a4acf74


YARN-5991. Yarn Distributed Shell does not print throwable t to App Master When 
failed to start container
(Contributed by Jim Frankola via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71a4acf7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71a4acf7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71a4acf7

Branch: refs/heads/trunk
Commit: 71a4acf74bc9ca34f0e57835c9d6e3efbe7c0567
Parents: 2977bc6
Author: Daniel Templeton 
Authored: Fri Jan 6 14:26:08 2017 -0800
Committer: Daniel Templeton 
Committed: Fri Jan 6 14:26:08 2017 -0800

--
 .../yarn/applications/distributedshell/ApplicationMaster.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a4acf7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 17dae6b..5a06ef6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -1001,7 +1001,7 @@ public class ApplicationMaster {
 
 @Override
 public void onStartContainerError(ContainerId containerId, Throwable t) {
-  LOG.error("Failed to start Container " + containerId);
+  LOG.error("Failed to start Container " + containerId, t);
   containers.remove(containerId);
   applicationMaster.numCompletedContainers.incrementAndGet();
   applicationMaster.numFailedContainers.incrementAndGet();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[14/15] hadoop git commit: YARN-5938. Refactoring OpportunisticContainerAllocator to use SchedulerRequestKey instead of Priority and other misc fixes (asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81da7d1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
index d275bfd..e60f70e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
@@ -33,10 +33,12 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerRequestKey;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 
+
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+
 import java.util.List;
 
 public class FifoAppAttempt extends FiCaSchedulerApp {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81da7d1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 657ac03..f0c0942 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -69,7 +69,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerCha
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerRequestKey;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
@@ -709,7 +709,7 @@ public class FifoScheduler extends
 // Inform the application
 RMContainer rmContainer = application.allocate(type, node, 
schedulerKey,
 request, container);
-
+
 // Inform the node
 node.allocateContainer(rmContainer);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81da7d1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
index ffaad58..8f5680c 100644
--- 

[03/15] hadoop git commit: YARN-5377. Fix TestQueuingContainerManager.testKillMultipleOpportunisticContainers. (Konstantinos Karanasos via asuresh)

[asuresh]

YARN-5377. Fix 
TestQueuingContainerManager.testKillMultipleOpportunisticContainers. 
(Konstantinos Karanasos via asuresh)

(cherry picked from commit f38a6d03a11ca6de93a225563ddf55ec99d5063c)
(cherry picked from commit 83ccab8a653a4ef943276651c9604bd434675980)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e60c7cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e60c7cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e60c7cd

Branch: refs/heads/branch-2
Commit: 0e60c7cd33b02cd89f635386c1edd5a8eadb1689
Parents: 7437252
Author: Arun Suresh 
Authored: Mon Nov 7 22:10:03 2016 -0800
Committer: Arun Suresh 
Committed: Fri Jan 6 11:15:09 2017 -0800

--
 .../BaseContainerManagerTest.java   | 38 +---
 .../queuing/TestQueuingContainerManager.java|  6 ++--
 2 files changed, 29 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e60c7cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index cb7815e..609d549 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -284,15 +284,17 @@ public abstract class BaseContainerManagerTest {
 .build());
   }
 
-  public static void waitForContainerState(ContainerManagementProtocol 
containerManager,
-  ContainerId containerID, ContainerState finalState)
+  public static void waitForContainerState(
+  ContainerManagementProtocol containerManager, ContainerId containerID,
+  ContainerState finalState)
   throws InterruptedException, YarnException, IOException {
 waitForContainerState(containerManager, containerID, finalState, 20);
   }
 
-  public static void waitForContainerState(ContainerManagementProtocol 
containerManager,
-  ContainerId containerID, ContainerState finalState, int timeOutMax)
-  throws InterruptedException, YarnException, IOException {
+  public static void waitForContainerState(
+  ContainerManagementProtocol containerManager, ContainerId containerID,
+  ContainerState finalState, int timeOutMax)
+  throws InterruptedException, YarnException, IOException {
 List list = new ArrayList();
 list.add(containerID);
 GetContainerStatusesRequest request =
@@ -314,8 +316,9 @@ public abstract class BaseContainerManagerTest {
   finalState, containerStatus.getState());
   }
 
-  static void waitForApplicationState(ContainerManagerImpl containerManager,
-  ApplicationId appID, ApplicationState finalState)
+  public static void waitForApplicationState(
+  ContainerManagerImpl containerManager, ApplicationId appID,
+  ApplicationState finalState)
   throws InterruptedException {
 // Wait for app-finish
 Application app =
@@ -344,7 +347,16 @@ public abstract class BaseContainerManagerTest {
   public static void waitForNMContainerState(ContainerManagerImpl
   containerManager, ContainerId containerID,
   org.apache.hadoop.yarn.server.nodemanager.containermanager
-  .container.ContainerState finalState, int timeOutMax)
+  .container.ContainerState finalState, int timeOutMax)
+  throws InterruptedException, YarnException, IOException {
+waitForNMContainerState(containerManager, containerID,
+Arrays.asList(finalState), timeOutMax);
+  }
+
+  public static void waitForNMContainerState(ContainerManagerImpl
+  containerManager, ContainerId containerID,
+  List finalStates, int timeOutMax)
   throws InterruptedException, YarnException, IOException {
 Container container = null;
 org.apache.hadoop.yarn.server.nodemanager
@@ -358,15 +370,15 @@ public abstract class BaseContainerManagerTest {
 currentState = container.getContainerState();
   }
   if (currentState != null) {
-LOG.info("Waiting for NM container to get into 

[11/15] hadoop git commit: YARN-2995. Enhance UI to show cluster resource utilization of various container Execution types. (Konstantinos Karanasos via asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/74372521/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index bf23ceb..176f279 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceUtilization;
 import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -702,13 +703,17 @@ public class TestRMWebServicesNodes extends 
JerseyTestBase {
   "aggregatedContainersPhysicalMemoryMB"),
   WebServicesTestUtils.getXmlInt(element,
   "aggregatedContainersVirtualMemoryMB"),
-  WebServicesTestUtils.getXmlFloat(element, "containersCPUUsage"));
+  WebServicesTestUtils.getXmlFloat(element, "containersCPUUsage"),
+  WebServicesTestUtils.getXmlInt(element, 
"numRunningOpportContainers"),
+  WebServicesTestUtils.getXmlLong(element, "usedMemoryOpport"),
+  WebServicesTestUtils.getXmlInt(element, "usedVirtualCoresOpport"),
+  WebServicesTestUtils.getXmlInt(element, "numQueuedContainers"));
 }
   }
 
   public void verifyNodeInfo(JSONObject nodeInfo, RMNode nm)
   throws JSONException, Exception {
-assertEquals("incorrect number of elements", 14, nodeInfo.length());
+assertEquals("incorrect number of elements", 18, nodeInfo.length());
 
 JSONObject resourceInfo = nodeInfo.getJSONObject("resourceUtilization");
 verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
@@ -725,21 +730,29 @@ public class TestRMWebServicesNodes extends 
JerseyTestBase {
 resourceInfo.getDouble("nodeCPUUsage"),
 resourceInfo.getInt("aggregatedContainersPhysicalMemoryMB"),
 resourceInfo.getInt("aggregatedContainersVirtualMemoryMB"),
-resourceInfo.getDouble("containersCPUUsage"));
+resourceInfo.getDouble("containersCPUUsage"),
+nodeInfo.getInt("numRunningOpportContainers"),
+nodeInfo.getLong("usedMemoryOpport"),
+nodeInfo.getInt("usedVirtualCoresOpport"),
+nodeInfo.getInt("numQueuedContainers"));
   }
 
   public void verifyNodeInfoGeneric(RMNode node, String state, String rack,
   String id, String nodeHostName,
   String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
-  int numContainers, long usedMemoryMB, long availMemoryMB, long 
usedVirtualCores, 
-  long availVirtualCores, String version, int nodePhysicalMemoryMB,
-  int nodeVirtualMemoryMB, double nodeCPUUsage,
+  int numContainers, long usedMemoryMB, long availMemoryMB,
+  long usedVirtualCores, long availVirtualCores, String version,
+  int nodePhysicalMemoryMB, int nodeVirtualMemoryMB, double nodeCPUUsage,
   int containersPhysicalMemoryMB, int containersVirtualMemoryMB,
-  double containersCPUUsage)
+  double containersCPUUsage, int numRunningOpportContainers,
+  long usedMemoryOpport, int usedVirtualCoresOpport,
+  int numQueuedContainers)
   throws JSONException, Exception {
 
 ResourceScheduler sched = rm.getResourceScheduler();
 SchedulerNodeReport report = sched.getNodeReport(node.getNodeID());
+OpportunisticContainersStatus opportunisticStatus =
+node.getOpportunisticContainersStatus();
 
 WebServicesTestUtils.checkStringMatch("state", node.getState().toString(),
 state);
@@ -787,6 +800,20 @@ public class TestRMWebServicesNodes extends JerseyTestBase 
{
   assertEquals("availVirtualCores doesn't match: " + availVirtualCores, 
report
   .getAvailableResource().getVirtualCores(), availVirtualCores);
 }
+
+if (opportunisticStatus != null) {
+  assertEquals("numRunningOpportContainers doesn't match: " +
+  

[12/15] hadoop git commit: YARN-2995. Enhance UI to show cluster resource utilization of various container Execution types. (Konstantinos Karanasos via asuresh)

[asuresh]

YARN-2995. Enhance UI to show cluster resource utilization of various container 
Execution types. (Konstantinos Karanasos via asuresh)

(cherry picked from commit 0aafc122d401a2f49d0f9ca556343f9ba871e979)
(cherry picked from commit 25598b6e7118bacd33a167fb496e9cda5e9f9ad0)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74372521
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74372521
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74372521

Branch: refs/heads/branch-2
Commit: 7437252102c937119183a4c28a5e516e90660a57
Parents: 28f3bb3
Author: Arun Suresh 
Authored: Fri Nov 4 07:31:54 2016 -0700
Committer: Arun Suresh 
Committed: Fri Jan 6 11:15:09 2017 -0800

--
 .../v2/app/rm/RMContainerAllocator.java |   6 +
 .../hadoop/yarn/sls/nodemanager/NodeInfo.java   |   4 +-
 .../yarn/sls/scheduler/RMNodeWrapper.java   |   4 +-
 .../yarn/server/api/records/NodeStatus.java |   7 +-
 .../records/OpportunisticContainersStatus.java  | 152 +++
 .../api/records/QueuedContainersStatus.java |  45 --
 .../api/records/impl/pb/NodeStatusPBImpl.java   |  38 ++---
 .../pb/OpportunisticContainersStatusPBImpl.java | 139 +
 .../impl/pb/QueuedContainersStatusPBImpl.java   |  84 --
 .../main/proto/yarn_server_common_protos.proto  |  12 +-
 .../protocolrecords/TestProtocolRecords.java|  17 ++-
 .../nodemanager/NodeStatusUpdaterImpl.java  |  17 ++-
 .../containermanager/ContainerManager.java  |   3 +
 .../containermanager/ContainerManagerImpl.java  |   8 +-
 .../queuing/QueuingContainerManagerImpl.java|  32 
 .../nodemanager/webapp/ContainerPage.java   |   1 +
 .../nodemanager/webapp/dao/ContainerInfo.java   |   7 +
 .../webapp/TestNMWebServicesContainers.java |   2 +-
 ...pportunisticContainerAllocatorAMService.java |   5 +-
 .../server/resourcemanager/rmnode/RMNode.java   |   4 +-
 .../resourcemanager/rmnode/RMNodeImpl.java  |  18 +--
 .../rmnode/RMNodeStatusEvent.java   |   6 +-
 .../scheduler/AbstractYarnScheduler.java|   3 +-
 .../distributed/NodeQueueLoadMonitor.java   |  10 +-
 .../resourcemanager/webapp/NodesPage.java   |  75 ++---
 .../resourcemanager/webapp/dao/NodeInfo.java|  41 -
 .../yarn/server/resourcemanager/MockNodes.java  |  12 +-
 .../distributed/TestNodeQueueLoadMonitor.java   |   8 +-
 .../resourcemanager/webapp/TestNodesPage.java   |  33 
 .../resourcemanager/webapp/TestRMWebApp.java|   1 +
 .../webapp/TestRMWebServicesNodes.java  |  41 -
 31 files changed, 600 insertions(+), 235 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/74372521/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index 9152b1b8..d2299d0 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -233,6 +233,8 @@ public class RMContainerAllocator extends 
RMContainerRequestor
 this.scheduledRequests.setNumOpportunisticMapsPer100(
 conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PER_100,
 MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PER_100));
+LOG.info(this.scheduledRequests.getNumOpportunisticMapsPer100() +
+"% of the mappers will be scheduled using OPPORTUNISTIC containers");
   }
 
   @Override
@@ -1030,6 +1032,10 @@ public class RMContainerAllocator extends 
RMContainerRequestor
   this.numOpportunisticMapsPer100 = numMaps;
 }
 
+int getNumOpportunisticMapsPer100() {
+  return this.numOpportunisticMapsPer100;
+}
+
 @VisibleForTesting
 final LinkedHashMap reduces =
   new LinkedHashMap();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/74372521/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
--
diff --git 

[09/15] hadoop git commit: YARN-4597. Introduce ContainerScheduler and a SCHEDULED state to NodeManager container lifecycle. (asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/368565f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
deleted file mode 100644
index b2d9dcb..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
+++ /dev/null
@@ -1,647 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.queuing;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.security.NMTokenIdentifier;
-import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
-import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
-import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
-import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
-import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
-import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerFinishedEvent;
-import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
-import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
-import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl.ProcessTreeInfo;
-import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Class extending {@link ContainerManagerImpl} and is used when queuing at the
- * NM is enabled.
- */
-public class QueuingContainerManagerImpl extends ContainerManagerImpl {
-
-  private static final Logger LOG = LoggerFactory
-  .getLogger(QueuingContainerManagerImpl.class);
-
-  private ConcurrentMap
-allocatedGuaranteedContainers;
-  private ConcurrentMap
-allocatedOpportunisticContainers;
-
-  private long allocatedMemoryOpportunistic;
-  private int 

[06/15] hadoop git commit: YARN-5918. Handle Opportunistic scheduling allocate request failure when NM is lost. (Bibin A Chundatt via asuresh)

[asuresh]

YARN-5918. Handle Opportunistic scheduling allocate request failure when NM is 
lost. (Bibin A Chundatt via asuresh)

(cherry picked from commit 005850b28feb2f7bb8c2844d11e3f9d21b45d754)
(cherry picked from commit cbff10b4147f98a89b393519b17e16385294af07)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44774eb2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44774eb2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44774eb2

Branch: refs/heads/branch-2
Commit: 44774eb21c64d725e2880860c998a1cdccfbb488
Parents: 368565f
Author: Arun Suresh 
Authored: Wed Nov 23 09:53:31 2016 -0800
Committer: Arun Suresh 
Committed: Fri Jan 6 11:15:09 2017 -0800

--
 ...pportunisticContainerAllocatorAMService.java | 13 ++-
 .../distributed/NodeQueueLoadMonitor.java   |  8 +-
 ...pportunisticContainerAllocatorAMService.java | 97 +++-
 3 files changed, 110 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/44774eb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index 1f83127..ada1a63 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -57,6 +57,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.NodeQueueLoadMonitor;
 
@@ -409,15 +410,19 @@ public class OpportunisticContainerAllocatorAMService
   private List convertToRemoteNodes(List nodeIds) {
 ArrayList retNodes = new ArrayList<>();
 for (NodeId nId : nodeIds) {
-  retNodes.add(convertToRemoteNode(nId));
+  RemoteNode remoteNode = convertToRemoteNode(nId);
+  if (null != remoteNode) {
+retNodes.add(remoteNode);
+  }
 }
 return retNodes;
   }
 
   private RemoteNode convertToRemoteNode(NodeId nodeId) {
-return RemoteNode.newInstance(nodeId,
-((AbstractYarnScheduler)rmContext.getScheduler()).getNode(nodeId)
-.getHttpAddress());
+SchedulerNode node =
+((AbstractYarnScheduler) rmContext.getScheduler()).getNode(nodeId);
+return node != null ? RemoteNode.newInstance(nodeId, node.getHttpAddress())
+: null;
   }
 
   private Resource createMaxContainerResource() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44774eb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
index 232b4ad..dec55ca 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
+++ 

[04/15] hadoop git commit: YARN-5486. Update OpportunisticContainerAllocatorAMService::allocate method to handle OPPORTUNISTIC container requests. (Konstantinos Karanasos via asuresh)

[asuresh]

YARN-5486. Update OpportunisticContainerAllocatorAMService::allocate method to 
handle OPPORTUNISTIC container requests. (Konstantinos Karanasos via asuresh)

(cherry picked from commit 10be45986cdf86a89055065b752959bd6369d54f)
(cherry picked from commit e3baa0988b4dfb592afe426ffeba3c7091b271c5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9ea7534
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9ea7534
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9ea7534

Branch: refs/heads/branch-2
Commit: f9ea753414f79af24fc3ab7f8c46b71cf0fa16a3
Parents: 937f739
Author: Arun Suresh 
Authored: Thu Sep 29 15:11:41 2016 -0700
Committer: Arun Suresh 
Committed: Fri Jan 6 11:15:09 2017 -0800

--
 .../TestOpportunisticContainerAllocation.java   | 398 +++
 .../OpportunisticContainerAllocator.java|  22 +-
 .../OpportunisticContainerContext.java  |  49 ++-
 .../yarn/server/nodemanager/NodeManager.java|   3 +-
 .../amrmproxy/DefaultRequestInterceptor.java|   4 +-
 .../scheduler/DistributedScheduler.java |  59 +--
 ...pportunisticContainerAllocatorAMService.java | 215 ++
 .../server/resourcemanager/ResourceManager.java |  12 +-
 .../scheduler/SchedulerApplicationAttempt.java  |  58 ++-
 .../distributed/NodeQueueLoadMonitor.java   |  45 ++-
 ...pportunisticContainerAllocatorAMService.java |  12 +-
 11 files changed, 708 insertions(+), 169 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9ea7534/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
new file mode 100644
index 000..b9b4b02
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
@@ -0,0 +1,398 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NMToken;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import 

[07/15] hadoop git commit: YARN-4597. Introduce ContainerScheduler and a SCHEDULED state to NodeManager container lifecycle. (asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/368565f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 8ebf364..0aad30b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -77,7 +77,8 @@ import 
org.apache.hadoop.yarn.server.nodemanager.amrmproxy.RequestInterceptor;
 import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
 import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
 import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
-import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.queuing.QueuingContainerManagerImpl;
+
+
 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService;
@@ -723,8 +724,9 @@ public class MiniYARNCluster extends CompositeService {
 ContainerExecutor exec, DeletionService del,
 NodeStatusUpdater nodeStatusUpdater, ApplicationACLsManager 
aclsManager,
 LocalDirsHandlerService dirsHandler) {
-  if 
(getConfig().getBoolean(YarnConfiguration.NM_CONTAINER_QUEUING_ENABLED,
-  YarnConfiguration.NM_CONTAINER_QUEUING_ENABLED_DEFAULT)) {
+  if (getConfig().getInt(
+  YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH, 0)
+  > 0) {
 return new CustomQueueingContainerManagerImpl(context, exec, del,
 nodeStatusUpdater, metrics, dirsHandler);
   } else {
@@ -864,7 +866,7 @@ public class MiniYARNCluster extends CompositeService {
   }
 
   private class CustomQueueingContainerManagerImpl extends
-  QueuingContainerManagerImpl {
+  ContainerManagerImpl {
 
 public CustomQueueingContainerManagerImpl(Context context,
 ContainerExecutor exec, DeletionService del, NodeStatusUpdater
@@ -874,25 +876,6 @@ public class MiniYARNCluster extends CompositeService {
 }
 
 @Override
-protected ContainersMonitor createContainersMonitor(ContainerExecutor
-exec) {
-  return new ContainersMonitorImpl(exec, dispatcher, this.context) {
-
-@Override
-public void increaseContainersAllocation(ProcessTreeInfo pti) { }
-
-@Override
-public void decreaseContainersAllocation(ProcessTreeInfo pti) { }
-
-@Override
-public boolean hasResourcesAvailable(
-ContainersMonitorImpl.ProcessTreeInfo pti) {
-  return true;
-}
-  };
-}
-
-@Override
 protected void createAMRMProxyService(Configuration conf) {
   this.amrmProxyEnabled =
   conf.getBoolean(YarnConfiguration.AMRM_PROXY_ENABLED,
@@ -910,6 +893,32 @@ public class MiniYARNCluster extends CompositeService {
 LOG.info("CustomAMRMProxyService is disabled");
   }
 }
+
+@Override
+protected ContainersMonitor createContainersMonitor(ContainerExecutor
+exec) {
+  return new ContainersMonitorImpl(exec, dispatcher, this.context) {
+@Override
+public float getVmemRatio() {
+  return 2.0f;
+}
+
+@Override
+public long getVmemAllocatedForContainers() {
+  return 16 * 1024L * 1024L * 1024L;
+}
+
+@Override
+public long getPmemAllocatedForContainers() {
+  return 8 * 1024L * 1024L * 1024L;
+}
+
+@Override
+public long getVCoresAllocatedForContainers() {
+  return 10;
+}
+  };
+}
   }
 
   private class ShortCircuitedAMRMProxy extends AMRMProxyService {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[02/15] hadoop git commit: YARN-5823. Update NMTokens in case of requests with only opportunistic containers. (Konstantinos Karanasos via asuresh)

[asuresh]

YARN-5823. Update NMTokens in case of requests with only opportunistic 
containers. (Konstantinos Karanasos via asuresh)

(cherry picked from commit 283fa33febe043bd7b4fa87546be26c9c5a8f8b5)
(cherry picked from commit 1c4cc88a754ac9f557cdc8c859b8aadec19a5067)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/047772f1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/047772f1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/047772f1

Branch: refs/heads/branch-2
Commit: 047772f15f1ba22d41d323639a10fb4b9477a981
Parents: 0e60c7c
Author: Arun Suresh 
Authored: Wed Nov 9 00:11:25 2016 -0800
Committer: Arun Suresh 
Committed: Fri Jan 6 11:15:09 2017 -0800

--
 .../TestOpportunisticContainerAllocation.java   | 71 +++-
 .../OpportunisticContainerAllocator.java| 55 ---
 .../containermanager/ContainerManagerImpl.java  |  2 +-
 .../scheduler/DistributedScheduler.java | 19 --
 .../ApplicationMasterService.java   |  3 +-
 ...pportunisticContainerAllocatorAMService.java | 23 ++-
 6 files changed, 137 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/047772f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
index b9b4b02..ace145d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
@@ -229,6 +229,9 @@ public class TestOpportunisticContainerAllocation {
 
   amClient.registerApplicationMaster("Host", 1, "");
 
+  testOpportunisticAllocation(
+  (AMRMClientImpl) amClient);
+
   testAllocation((AMRMClientImpl)amClient);
 
   amClient
@@ -247,7 +250,6 @@ public class TestOpportunisticContainerAllocation {
   final AMRMClientImpl amClient)
   throws YarnException, IOException {
 // setup container request
-
 assertEquals(0, amClient.ask.size());
 assertEquals(0, amClient.release.size());
 
@@ -388,6 +390,73 @@ public class TestOpportunisticContainerAllocation {
 assertEquals(0, amClient.release.size());
   }
 
+  /**
+   * Tests allocation with requests comprising only opportunistic containers.
+   */
+  private void testOpportunisticAllocation(
+  final AMRMClientImpl amClient)
+  throws YarnException, IOException {
+// setup container request
+assertEquals(0, amClient.ask.size());
+assertEquals(0, amClient.release.size());
+
+amClient.addContainerRequest(
+new AMRMClient.ContainerRequest(capability, null, null, priority, 0,
+true, null,
+ExecutionTypeRequest.newInstance(
+ExecutionType.OPPORTUNISTIC, true)));
+amClient.addContainerRequest(
+new AMRMClient.ContainerRequest(capability, null, null, priority, 0,
+true, null,
+ExecutionTypeRequest.newInstance(
+ExecutionType.OPPORTUNISTIC, true)));
+
+int oppContainersRequestedAny =
+amClient.getTable(0).get(priority, ResourceRequest.ANY,
+ExecutionType.OPPORTUNISTIC, capability).remoteRequest
+.getNumContainers();
+
+assertEquals(2, oppContainersRequestedAny);
+
+assertEquals(1, amClient.ask.size());
+assertEquals(0, amClient.release.size());
+
+// RM should allocate container within 2 calls to allocate()
+int allocatedContainerCount = 0;
+int iterationsLeft = 10;
+Set releases = new TreeSet<>();
+
+amClient.getNMTokenCache().clearCache();
+Assert.assertEquals(0,
+amClient.getNMTokenCache().numberOfTokensInCache());
+HashMap receivedNMTokens = new HashMap<>();
+
+while (allocatedContainerCount < oppContainersRequestedAny
+&& iterationsLeft-- > 0) {
+  AllocateResponse allocResponse = amClient.allocate(0.1f);
+  assertEquals(0, amClient.ask.size());
+  assertEquals(0, amClient.release.size());
+
+  for (Container container : allocResponse.getAllocatedContainers()) {
+allocatedContainerCount++;
+ContainerId rejectContainerId = container.getId();
+

[08/15] hadoop git commit: YARN-4597. Introduce ContainerScheduler and a SCHEDULED state to NodeManager container lifecycle. (asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/368565f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index 2d2c294..eb30c5d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -95,14 +95,16 @@ import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.Conta
 import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
 import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
 import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler;
-import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerScheduler;
+
 import 
org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
 import 
org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
 import 
org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
 import 
org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -552,6 +554,35 @@ public class TestContainerManagerRecovery extends 
BaseContainerManagerTest {
   throw new YarnException("Reject this container");
 }
   }
+  @Override
+  protected ContainerScheduler createContainerScheduler(Context context) {
+return new ContainerScheduler(context, dispatcher, metrics){
+  @Override
+  public ContainersMonitor getContainersMonitor() {
+return new ContainersMonitorImpl(null, null, null) {
+  @Override
+  public float getVmemRatio() {
+return 2.0f;
+  }
+
+  @Override
+  public long getVmemAllocatedForContainers() {
+return 20480;
+  }
+
+  @Override
+  public long getPmemAllocatedForContainers() {
+return 10240;
+  }
+
+  @Override
+  public long getVCoresAllocatedForContainers() {
+return 4;
+  }
+};
+  }
+};
+  }
 };
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/368565f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRegression.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRegression.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRegression.java
deleted file mode 100644
index 71af76f..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRegression.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may 

[01/15] hadoop git commit: YARN-5982. Simplify opportunistic container parameters and metrics. (Konstantinos Karanasos via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 937f73980 -> 81da7d1d3


YARN-5982. Simplify opportunistic container parameters and metrics. 
(Konstantinos Karanasos via asuresh)

(cherry picked from commit b0aace21b1ef3436ba9d516186208fee9a9ceef2)
(cherry picked from commit 3faa0b0d682687099b253feee9b41340edef7111)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d3bb30a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d3bb30a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d3bb30a

Branch: refs/heads/branch-2
Commit: 3d3bb30a2510e19c8c7aacc9fc66283637dc33d8
Parents: 44774eb
Author: Arun Suresh 
Authored: Fri Dec 9 16:41:25 2016 -0800
Committer: Arun Suresh 
Committed: Fri Jan 6 11:15:09 2017 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java | 41 
 .../src/main/resources/yarn-default.xml | 57 -
 .../scheduler/ContainerScheduler.java   | 10 +--
 .../nodemanager/metrics/NodeManagerMetrics.java | 65 ++--
 ...pportunisticContainerAllocatorAMService.java | 60 --
 .../scheduler/SchedulerNode.java|  6 +-
 .../resourcemanager/webapp/NodesPage.java   |  6 +-
 .../resourcemanager/webapp/dao/NodeInfo.java| 10 +--
 .../webapp/TestRMWebServicesNodes.java  | 10 +--
 9 files changed, 68 insertions(+), 197 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d3bb30a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b97c434..7a3ecf2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -309,47 +309,6 @@ public class YarnConfiguration extends Configuration {
   public static final boolean
   OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED_DEFAULT = false;
 
-  /** Minimum memory (in MB) used for allocating an opportunistic container. */
-  public static final String OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB =
-  YARN_PREFIX + "opportunistic-containers.min-memory-mb";
-  public static final int OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB_DEFAULT = 512;
-
-  /** Minimum virtual CPU cores used for allocating an opportunistic container.
-   * */
-  public static final String OPPORTUNISTIC_CONTAINERS_MIN_VCORES =
-  YARN_PREFIX + "opportunistic-containers.min-vcores";
-  public static final int OPPORTUNISTIC_CONTAINERS_MIN_VCORES_DEFAULT = 1;
-
-  /** Maximum memory (in MB) used for allocating an opportunistic container. */
-  public static final String OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB =
-  YARN_PREFIX + "opportunistic-containers.max-memory-mb";
-  public static final int OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB_DEFAULT = 
2048;
-
-  /** Maximum virtual CPU cores used for allocating an opportunistic container.
-   * */
-  public static final String OPPORTUNISTIC_CONTAINERS_MAX_VCORES =
-  YARN_PREFIX + "opportunistic-containers.max-vcores";
-  public static final int OPPORTUNISTIC_CONTAINERS_MAX_VCORES_DEFAULT = 4;
-
-  /** Incremental memory (in MB) used for allocating an opportunistic 
container.
-   * */
-  public static final String OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB =
-  YARN_PREFIX + "opportunistic-containers.incr-memory-mb";
-  public static final int OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB_DEFAULT =
-  512;
-
-  /** Incremental virtual CPU cores used for allocating an opportunistic
-   * container. */
-  public static final String OPPORTUNISTIC_CONTAINERS_INCR_VCORES =
-  YARN_PREFIX + "opportunistic-containers.incr-vcores";
-  public static final int OPPORTUNISTIC_CONTAINERS_INCR_VCORES_DEFAULT = 1;
-
-  /** Container token expiry for opportunistic containers. */
-  public static final String OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS =
-  YARN_PREFIX + "opportunistic-containers.container-token-expiry-ms";
-  public static final int OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT =
-  60;
-
   /** Number of nodes to be used by the Opportunistic Container allocator for
* dispatching containers during container allocation. */
   public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =

[05/15] hadoop git commit: YARN-5799. Fix Opportunistic Allocation to set the correct value of Node Http Address. (asuresh)

[asuresh]

YARN-5799. Fix Opportunistic Allocation to set the correct value of Node Http 
Address. (asuresh)

(cherry picked from commit aa3cab1eb29c56368d15882d7260a994e615e8d8)
(cherry picked from commit ce10e98826884245e733431297d0275177965f17)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28f3bb38
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28f3bb38
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28f3bb38

Branch: refs/heads/branch-2
Commit: 28f3bb38bfb817dd9c16129f1d48ba5328f5b4f9
Parents: f9ea753
Author: Arun Suresh 
Authored: Sat Oct 29 02:03:57 2016 -0700
Committer: Arun Suresh 
Committed: Fri Jan 6 11:15:09 2017 -0800

--
 .../DistributedSchedulingAllocateResponse.java  |   6 +-
 ...RegisterDistributedSchedulingAMResponse.java |   6 +-
 .../server/api/protocolrecords/RemoteNode.java  |  90 +
 ...ributedSchedulingAllocateResponsePBImpl.java |  41 +++---
 ...erDistributedSchedulingAMResponsePBImpl.java |  39 +++---
 .../impl/pb/RemoteNodePBImpl.java   | 135 +++
 .../OpportunisticContainerAllocator.java|  24 ++--
 .../OpportunisticContainerContext.java  |  14 +-
 .../yarn_server_common_service_protos.proto |   9 +-
 .../yarn/server/nodemanager/NodeManager.java|   2 +-
 .../scheduler/DistributedScheduler.java |   3 +-
 .../scheduler/TestDistributedScheduler.java |  20 ++-
 ...pportunisticContainerAllocatorAMService.java |  30 -
 ...pportunisticContainerAllocatorAMService.java |   6 +-
 14 files changed, 343 insertions(+), 82 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/28f3bb38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateResponse.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateResponse.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateResponse.java
index 7a40449..edc0cf8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateResponse.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateResponse.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.api.protocolrecords;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.util.Records;
 
 import java.util.List;
@@ -58,9 +57,10 @@ public abstract class DistributedSchedulingAllocateResponse {
 
   @Public
   @Unstable
-  public abstract void setNodesForScheduling(List nodesForScheduling);
+  public abstract void setNodesForScheduling(
+  List nodesForScheduling);
 
   @Public
   @Unstable
-  public abstract List getNodesForScheduling();
+  public abstract List getNodesForScheduling();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/28f3bb38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterDistributedSchedulingAMResponse.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterDistributedSchedulingAMResponse.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterDistributedSchedulingAMResponse.java
index a0a0e38..f7d8df2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterDistributedSchedulingAMResponse.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterDistributedSchedulingAMResponse.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.api.protocolrecords;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import 

[10/15] hadoop git commit: YARN-4597. Introduce ContainerScheduler and a SCHEDULED state to NodeManager container lifecycle. (asuresh)

[asuresh]

YARN-4597. Introduce ContainerScheduler and a SCHEDULED state to NodeManager 
container lifecycle. (asuresh)

(cherry picked from commit 3219b7b4ac7d12aee343f6ab2980b3357fc618b6)
(cherry picked from commit 49921678858a17ca58ca2ae704ced1ff584e0859)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/368565f0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/368565f0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/368565f0

Branch: refs/heads/branch-2
Commit: 368565f0ce229c573665d38d95526c10fd5e4108
Parents: 047772f
Author: Arun Suresh 
Authored: Tue Nov 15 07:48:55 2016 -0800
Committer: Arun Suresh 
Committed: Fri Jan 6 11:15:09 2017 -0800

--
 .../hadoop/mapred/TestMROpportunisticMaps.java  |  18 +-
 .../yarn/api/records/ContainerExitStatus.java   |   6 +
 .../hadoop/yarn/api/records/ContainerState.java |   4 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |  17 +-
 .../src/main/proto/yarn_protos.proto|   2 +-
 .../api/impl/TestDistributedScheduling.java |   3 +-
 .../yarn/client/api/impl/TestNMClient.java  |   9 +-
 .../TestOpportunisticContainerAllocation.java   |   2 +
 .../src/main/resources/yarn-default.xml |   6 +-
 .../hadoop/yarn/server/nodemanager/Context.java |  18 +-
 .../yarn/server/nodemanager/NodeManager.java|  42 +-
 .../nodemanager/NodeStatusUpdaterImpl.java  |  53 --
 .../containermanager/ContainerManager.java  |   4 +
 .../containermanager/ContainerManagerImpl.java  |  32 +-
 .../containermanager/container/Container.java   |   6 +
 .../container/ContainerImpl.java|  95 +-
 .../container/ContainerState.java   |   2 +-
 .../launcher/ContainerLaunch.java   |  30 +-
 .../launcher/RecoveredContainerLaunch.java  |   4 +-
 .../monitor/ContainersMonitor.java  |  54 +-
 .../monitor/ContainersMonitorImpl.java  |  79 +-
 .../queuing/QueuingContainerManagerImpl.java| 647 --
 .../containermanager/queuing/package-info.java  |  23 -
 ...locationBasedResourceUtilizationTracker.java | 137 +++
 .../scheduler/ContainerScheduler.java   | 419 +
 .../scheduler/ContainerSchedulerEvent.java  |  51 ++
 .../scheduler/ContainerSchedulerEventType.java  |  29 +
 .../scheduler/ResourceUtilizationTracker.java   |  59 ++
 .../scheduler/package-info.java |  22 +
 .../nodemanager/metrics/NodeManagerMetrics.java |  35 +
 .../nodemanager/webapp/ContainerLogsUtils.java  |   5 +-
 .../yarn/server/nodemanager/TestEventFlow.java  |   3 +-
 .../nodemanager/TestNodeManagerResync.java  |   8 +
 .../nodemanager/TestNodeManagerShutdown.java|   5 +-
 .../nodemanager/TestNodeStatusUpdater.java  | 123 ---
 .../amrmproxy/BaseAMRMProxyTest.java|   6 +-
 .../BaseContainerManagerTest.java   |  24 +-
 .../TestContainerManagerRecovery.java   |  35 +-
 .../TestContainerManagerRegression.java |  84 --
 .../container/TestContainer.java|  26 +-
 .../queuing/TestQueuingContainerManager.java| 596 -
 .../TestContainerSchedulerQueuing.java  | 872 +++
 .../nodemanager/webapp/MockContainer.java   |  15 +
 ...pportunisticContainerAllocatorAMService.java |   4 +-
 .../rmcontainer/RMContainerImpl.java|   4 +-
 .../resourcemanager/rmnode/RMNodeImpl.java  |  32 +-
 .../scheduler/AbstractYarnScheduler.java|   2 +
 .../scheduler/SchedulerNode.java|   9 +-
 .../hadoop/yarn/server/MiniYARNCluster.java |  55 +-
 49 files changed, 1996 insertions(+), 1820 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/368565f0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
index 021863b..d975fd0 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java
@@ -68,15 +68,6 @@ public class TestMROpportunisticMaps {
 doTest(4, 1, 1, 2);
   }
 
-  /**
-   * Test will run with 6 Maps and 2 Reducers. All the Maps are OPPORTUNISTIC.
-   * @throws 

[13/15] hadoop git commit: YARN-5646. Add documentation and update config parameter names for scheduling of OPPORTUNISTIC containers. (Konstantinos Karanasos via asuresh)

[asuresh]

YARN-5646. Add documentation and update config parameter names for scheduling 
of OPPORTUNISTIC containers. (Konstantinos Karanasos via asuresh)

(cherry picked from commit 2273a74c1f3895163046cca09ff5e983df301d22)
(cherry picked from commit 9e17ffe599e12c59c48f4355de645c31a4735d04)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b4d3e85
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b4d3e85
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b4d3e85

Branch: refs/heads/branch-2
Commit: 2b4d3e8506a36cb21c347d0bb455be28537ec023
Parents: 3d3bb30
Author: Arun Suresh 
Authored: Fri Dec 16 08:14:34 2016 -0800
Committer: Arun Suresh 
Committed: Fri Jan 6 11:15:10 2017 -0800

--
 .../v2/app/rm/RMContainerAllocator.java |   4 +-
 .../apache/hadoop/mapreduce/MRJobConfig.java|   6 +-
 .../hadoop/mapred/TestMROpportunisticMaps.java  |   2 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |  59 ++---
 .../api/impl/TestDistributedScheduling.java |   1 -
 .../src/main/resources/yarn-default.xml |  28 +--
 .../yarn/server/nodemanager/NodeManager.java|   2 +-
 .../nodemanager/amrmproxy/AMRMProxyService.java |   2 +-
 .../containermanager/ContainerManagerImpl.java  |   4 +-
 .../scheduler/ContainerScheduler.java   |   2 +-
 ...pportunisticContainerAllocatorAMService.java |  16 +-
 .../yarn/server/resourcemanager/MockRM.java |   2 +-
 .../hadoop/yarn/server/MiniYARNCluster.java |   8 +-
 .../site/markdown/OpportunisticContainers.md| 225 +++
 14 files changed, 296 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b4d3e85/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index d2299d0..45c2a0e 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -231,8 +231,8 @@ public class RMContainerAllocator extends 
RMContainerRequestor
 // first attempt to contact RM.
 retrystartTime = System.currentTimeMillis();
 this.scheduledRequests.setNumOpportunisticMapsPer100(
-conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PER_100,
-MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PER_100));
+conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE,
+MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE));
 LOG.info(this.scheduledRequests.getNumOpportunisticMapsPer100() +
 "% of the mappers will be scheduled using OPPORTUNISTIC containers");
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b4d3e85/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index e046c66..140b9ff 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -988,9 +988,9 @@ public interface MRJobConfig {
* requested by the AM will be opportunistic. If the total number of maps
* for the job is less than 'x', then ALL maps will be OPPORTUNISTIC
*/
-  public static final String MR_NUM_OPPORTUNISTIC_MAPS_PER_100 =
-  "mapreduce.job.num-opportunistic-maps-per-100";
-  public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PER_100 = 0;
+  public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE =
+  "mapreduce.job.num-opportunistic-maps-percentage";
+  public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE = 0;
 
   /**
* A comma-separated list 

[15/15] hadoop git commit: YARN-5938. Refactoring OpportunisticContainerAllocator to use SchedulerRequestKey instead of Priority and other misc fixes (asuresh)

[asuresh]

YARN-5938. Refactoring OpportunisticContainerAllocator to use 
SchedulerRequestKey instead of Priority and other misc fixes (asuresh)

(cherry picked from commit ac1e5d4f77e3b9df8dcacb0b1f72eecc27931eb8)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81da7d1d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81da7d1d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81da7d1d

Branch: refs/heads/branch-2
Commit: 81da7d1d3084f8f6cc004193e351fc678435705c
Parents: 2b4d3e8
Author: Arun Suresh 
Authored: Tue Dec 27 11:54:57 2016 -0800
Committer: Arun Suresh 
Committed: Fri Jan 6 11:25:49 2017 -0800

--
 .../api/records/UpdateContainerRequest.java |  11 +
 .../impl/pb/AllocateResponsePBImpl.java |   4 +-
 .../server/api/protocolrecords/RemoteNode.java  |   7 +
 .../OpportunisticContainerAllocator.java|  61 +--
 .../OpportunisticContainerContext.java  |  38 +-
 .../server/scheduler/SchedulerRequestKey.java   | 130 ++
 .../scheduler/DistributedScheduler.java |  11 +-
 .../ApplicationMasterService.java   | 410 ++-
 ...pportunisticContainerAllocatorAMService.java |  45 +-
 .../rmcontainer/RMContainer.java|   3 +-
 .../rmcontainer/RMContainerImpl.java|   2 +-
 .../rmcontainer/RMContainerReservedEvent.java   |   2 +-
 .../scheduler/AppSchedulingInfo.java|   2 +
 .../scheduler/SchedulerApplicationAttempt.java  |   4 +-
 .../scheduler/SchedulerNode.java|   1 +
 .../scheduler/SchedulerRequestKey.java  | 122 --
 .../scheduler/capacity/LeafQueue.java   |   7 +-
 .../allocator/IncreaseContainerAllocator.java   |   2 +-
 .../allocator/RegularContainerAllocator.java|   2 +-
 .../scheduler/common/SchedulerContainer.java|   2 +-
 .../scheduler/common/fica/FiCaSchedulerApp.java |   3 +-
 .../common/fica/FiCaSchedulerNode.java  |   2 +-
 .../distributed/NodeQueueLoadMonitor.java   |  16 +-
 .../scheduler/fair/FSAppAttempt.java|   2 +-
 .../scheduler/fair/FSSchedulerNode.java |   2 +-
 .../scheduler/fifo/FifoAppAttempt.java  |   4 +-
 .../scheduler/fifo/FifoScheduler.java   |   4 +-
 .../LocalitySchedulingPlacementSet.java |  16 +-
 .../placement/SchedulingPlacementSet.java   |   2 +-
 .../server/resourcemanager/Application.java |   2 +-
 .../yarn/server/resourcemanager/Task.java   |   2 +-
 ...alCapacityPreemptionPolicyMockFramework.java |   3 +-
 ...estProportionalCapacityPreemptionPolicy.java |   3 +-
 .../scheduler/TestAppSchedulingInfo.java|   1 +
 .../TestSchedulerApplicationAttempt.java|   2 +
 .../capacity/TestCapacityScheduler.java |   2 +-
 .../scheduler/capacity/TestLeafQueue.java   |   2 +-
 .../scheduler/capacity/TestReservations.java|   2 +-
 .../scheduler/capacity/TestUtils.java   |   2 +-
 .../fair/TestContinuousScheduling.java  |   2 +-
 .../scheduler/fair/TestFSAppAttempt.java|   2 +-
 .../fair/TestFairSchedulerPreemption.java   |   3 +-
 42 files changed, 502 insertions(+), 443 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/81da7d1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
index 200dea3..e4f7a82 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
@@ -159,6 +159,17 @@ public abstract class UpdateContainerRequest extends 
AbstractResourceRequest {
   }
 
   @Override
+  public String toString() {
+return "UpdateReq{" +
+"containerId=" + getContainerId() + ", " +
+"containerVersion=" + getContainerVersion() + ", " +
+"targetExecType=" + getExecutionType() + ", " +
+"targetCapability=" + getCapability() + ", " +
+"updateType=" + getContainerUpdateType() + ", " +
+"}";
+  }
+
+  @Override
   public boolean equals(Object obj) {
 if (this == obj) {
   return true;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81da7d1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java

hadoop git commit: MAPREDUCE-6682. TestMRCJCFileOutputCommitter fails intermittently Contributed by Akira Ajisaka. (cherry picked from commit 8f1c374bec2451568f463ab68f7eb6db734ab14e)

[jlowe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 65e7ae5dc -> 937f73980


MAPREDUCE-6682. TestMRCJCFileOutputCommitter fails intermittently Contributed 
by Akira Ajisaka.
(cherry picked from commit 8f1c374bec2451568f463ab68f7eb6db734ab14e)

Conflicts:


hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/937f7398
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/937f7398
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/937f7398

Branch: refs/heads/branch-2
Commit: 937f7398043c17122af391519d8550ea8111d1db
Parents: 65e7ae5
Author: Jason Lowe 
Authored: Fri Jan 6 18:46:28 2017 +
Committer: Jason Lowe 
Committed: Fri Jan 6 18:46:28 2017 +

--
 .../mapred/TestMRCJCFileOutputCommitter.java| 14 +---
 .../mapred/TestMRTimelineEventHandling.java | 34 +---
 2 files changed, 32 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/937f7398/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java
index 74b6d77..83796e8 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.fs.RawLocalFileSystem;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
 import org.junit.Test;
 
 import java.io.File;
@@ -37,8 +39,7 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 public class TestMRCJCFileOutputCommitter {
-  private static Path outDir = new Path(
- System.getProperty("test.build.data", "/tmp"), "output");
+  private static Path outDir = new 
Path(GenericTestUtils.getTempPath("output"));
 
   // A random task attempt id for testing.
   private static String attempt = "attempt_200707121733_0001_m_00_0";
@@ -112,12 +113,11 @@ public class TestMRCJCFileOutputCommitter {
 expectedOutput.append(key2).append('\t').append(val2).append("\n");
 String output = UtilsForTests.slurp(expectedFile);
 assertEquals(output, expectedOutput.toString());
-
-FileUtil.fullyDelete(new File(outDir.toString()));
   }
 
   @Test
   public void testAbort() throws IOException {
+FileUtil.fullyDelete(new File(outDir.toString()));
 JobConf job = new JobConf();
 setConfForFileOutputCommitter(job);
 JobContext jContext = new JobContextImpl(job, taskID.getJobID());
@@ -152,7 +152,6 @@ public class TestMRCJCFileOutputCommitter {
 assertFalse("job temp dir "+expectedFile+" still exists", 
expectedFile.exists());
 assertEquals("Output directory not empty", 0, new File(outDir.toString())
 .listFiles().length);
-FileUtil.fullyDelete(new File(outDir.toString()));
   }
 
   public static class FakeFileSystem extends RawLocalFileSystem {
@@ -223,4 +222,9 @@ public class TestMRCJCFileOutputCommitter {
 assertTrue(th.getMessage().contains("fake delete failed"));
 assertTrue("job temp dir does not exists", jobTmpDir.exists());
   }
+
+  @After
+  public void teardown() {
+FileUtil.fullyDelete(new File(outDir.toString()));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/937f7398/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index 40ed9ad..140591c 100644
--- 

hadoop git commit: MAPREDUCE-6682. TestMRCJCFileOutputCommitter fails intermittently Contributed by Akira Ajisaka. (cherry picked from commit 8f1c374bec2451568f463ab68f7eb6db734ab14e)

[jlowe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 15e89634a -> 8ffe5eb7b


MAPREDUCE-6682. TestMRCJCFileOutputCommitter fails intermittently Contributed 
by Akira Ajisaka.
(cherry picked from commit 8f1c374bec2451568f463ab68f7eb6db734ab14e)

Conflicts:


hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ffe5eb7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ffe5eb7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ffe5eb7

Branch: refs/heads/branch-2.8
Commit: 8ffe5eb7bd6815fdbb3beb7822919f1ff3b2cba9
Parents: 15e8963
Author: Jason Lowe 
Authored: Fri Jan 6 18:55:48 2017 +
Committer: Jason Lowe 
Committed: Fri Jan 6 18:55:48 2017 +

--
 .../mapred/TestMRCJCFileOutputCommitter.java| 14 +---
 .../mapred/TestMRTimelineEventHandling.java | 34 +---
 2 files changed, 32 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ffe5eb7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java
index 74b6d77..83796e8 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileOutputCommitter.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.fs.RawLocalFileSystem;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
 import org.junit.Test;
 
 import java.io.File;
@@ -37,8 +39,7 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 public class TestMRCJCFileOutputCommitter {
-  private static Path outDir = new Path(
- System.getProperty("test.build.data", "/tmp"), "output");
+  private static Path outDir = new 
Path(GenericTestUtils.getTempPath("output"));
 
   // A random task attempt id for testing.
   private static String attempt = "attempt_200707121733_0001_m_00_0";
@@ -112,12 +113,11 @@ public class TestMRCJCFileOutputCommitter {
 expectedOutput.append(key2).append('\t').append(val2).append("\n");
 String output = UtilsForTests.slurp(expectedFile);
 assertEquals(output, expectedOutput.toString());
-
-FileUtil.fullyDelete(new File(outDir.toString()));
   }
 
   @Test
   public void testAbort() throws IOException {
+FileUtil.fullyDelete(new File(outDir.toString()));
 JobConf job = new JobConf();
 setConfForFileOutputCommitter(job);
 JobContext jContext = new JobContextImpl(job, taskID.getJobID());
@@ -152,7 +152,6 @@ public class TestMRCJCFileOutputCommitter {
 assertFalse("job temp dir "+expectedFile+" still exists", 
expectedFile.exists());
 assertEquals("Output directory not empty", 0, new File(outDir.toString())
 .listFiles().length);
-FileUtil.fullyDelete(new File(outDir.toString()));
   }
 
   public static class FakeFileSystem extends RawLocalFileSystem {
@@ -223,4 +222,9 @@ public class TestMRCJCFileOutputCommitter {
 assertTrue(th.getMessage().contains("fake delete failed"));
 assertTrue("job temp dir does not exists", jobTmpDir.exists());
   }
+
+  @After
+  public void teardown() {
+FileUtil.fullyDelete(new File(outDir.toString()));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ffe5eb7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index 40ed9ad..140591c 100644
--- 

hadoop git commit: HADOOP-13931. S3AGuard: Use BatchWriteItem in DynamoDBMetadataStore#put(). Contributed by Mingliang Liu.

[cnauroth]

Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-13345 a412b1020 -> a1b47db40


HADOOP-13931. S3AGuard: Use BatchWriteItem in DynamoDBMetadataStore#put(). 
Contributed by Mingliang Liu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1b47db4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1b47db4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1b47db4

Branch: refs/heads/HADOOP-13345
Commit: a1b47db405832087c080a4c5743f2efaa620d566
Parents: a412b10
Author: Chris Nauroth 
Authored: Fri Jan 6 10:30:47 2017 -0800
Committer: Chris Nauroth 
Committed: Fri Jan 6 10:30:47 2017 -0800

--
 .../fs/s3a/s3guard/DynamoDBMetadataStore.java   | 51 
 .../hadoop/fs/s3a/s3guard/MetadataStore.java|  5 +-
 2 files changed, 34 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1b47db4/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index 89ce3c4..1c19625 100644
--- 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -413,9 +413,27 @@ public class DynamoDBMetadataStore implements 
MetadataStore {
 
   @Override
   public void put(PathMetadata meta) throws IOException {
-checkPathMetadata(meta);
+// For a deeply nested path, this method will automatically create the full
+// ancestry and save respective item in DynamoDB table.
+// So after put operation, we maintain the invariant that if a path exists,
+// all its ancestors will also exist in the table.
+// For performance purpose, we generate the full paths to put and use batch
+// write item request to save the items.
 LOG.debug("Saving to table {} in region {}: {}", tableName, region, meta);
-innerPut(meta);
+processBatchWriteRequest(null, pathMetadataToItem(fullPathsToPut(meta)));
+  }
+
+  /**
+   * Helper method to get full path of ancestors that are nonexistent in table.
+   */
+  private Collection fullPathsToPut(PathMetadata meta)
+  throws IOException {
+checkPathMetadata(meta);
+final Collection metasToPut = new ArrayList<>();
+// root path is not persisted
+if (!meta.getFileStatus().getPath().isRoot()) {
+  metasToPut.add(meta);
+}
 
 // put all its ancestors if not present; as an optimization we return at 
its
 // first existent ancestor
@@ -427,34 +445,29 @@ public class DynamoDBMetadataStore implements 
MetadataStore {
   final Item item = table.getItem(spec);
   if (item == null) {
 final S3AFileStatus status = new S3AFileStatus(false, path, username);
-innerPut(new PathMetadata(status));
+metasToPut.add(new PathMetadata(status));
 path = path.getParent();
   } else {
 break;
   }
 }
-  }
-
-  private void innerPut(PathMetadata meta) throws IOException {
-final Path path = meta.getFileStatus().getPath();
-if (path.isRoot()) {
-  LOG.debug("Root path / is not persisted");
-  return;
-}
-
-try {
-  table.putItem(pathMetadataToItem(meta));
-} catch (AmazonClientException e) {
-  throw translateException("put", path, e);
-}
+return metasToPut;
   }
 
   @Override
   public void put(DirListingMetadata meta) throws IOException {
 LOG.debug("Saving to table {} in region {}: {}", tableName, region, meta);
 
-for (PathMetadata pathMetadata : meta.getListing()) {
-  put(pathMetadata);
+// directory path
+final Collection metasToPut = fullPathsToPut(
+new PathMetadata(new S3AFileStatus(false, meta.getPath(), username)));
+// all children of the directory
+metasToPut.addAll(meta.getListing());
+
+try {
+  processBatchWriteRequest(null, pathMetadataToItem(metasToPut));
+} catch (AmazonClientException e) {
+  throw translateException("put", (String) null, e);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1b47db4/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
index 6d3c440..5c611c2 100644
--- 

[1/2] hadoop git commit: YARN-6040. Introduce api independent PendingAsk to replace usage of ResourceRequest within Scheduler classes. (Wangda Tan via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 4a659ff40 -> 2977bc6a1


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
index 3cf5fa2..3e0620e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
@@ -21,6 +21,8 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
 import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
 
 import java.util.Collection;
@@ -70,22 +72,38 @@ public interface SchedulingPlacementSet {
   Map getResourceRequests();
 
   /**
-   * Get ResourceRequest by given schedulerKey and resourceName
+   * Get pending ask for given resourceName. If there's no such pendingAsk,
+   * returns {@link PendingAsk#ZERO}
+   *
+   * @param resourceName resourceName
+   * @return PendingAsk
+   */
+  PendingAsk getPendingAsk(String resourceName);
+
+  /**
+   * Get #pending-allocations for given resourceName. If there's no such
+   * pendingAsk, returns 0
+   *
* @param resourceName resourceName
-   * @return ResourceRequest
+   * @return #pending-allocations
*/
-  ResourceRequest getResourceRequest(String resourceName);
+  int getOutstandingAsksCount(String resourceName);
 
   /**
* Notify container allocated.
* @param schedulerKey SchedulerRequestKey for this ResourceRequest
* @param type Type of the allocation
* @param node Which node this container allocated on
-   * @param request Which resource request to allocate
* @return list of ResourceRequests deducted
*/
   List allocate(SchedulerRequestKey schedulerKey,
-  NodeType type, SchedulerNode node, ResourceRequest request);
+  NodeType type, SchedulerNode node);
+
+  /**
+   * Returns list of accepted resourceNames.
+   * @return Iterator of accepted resourceNames
+   */
+  Iterator getAcceptedResouceNames();
 
   /**
* We can still have pending requirement for a given NodeType and node
@@ -94,4 +112,47 @@ public interface SchedulingPlacementSet {
* @return true if we has pending requirement
*/
   boolean canAllocate(NodeType type, SchedulerNode node);
+
+  /**
+   * Can delay to give locality?
+   * TODO (wangda): This should be moved out of SchedulingPlacementSet
+   * and should belong to specific delay scheduling policy impl.
+   *
+   * @param resourceName resourceName
+   * @return can/cannot
+   */
+  boolean canDelayTo(String resourceName);
+
+  /**
+   * Does this {@link SchedulingPlacementSet} accept resources on 
nodePartition?
+   *
+   * @param nodePartition nodePartition
+   * @param schedulingMode schedulingMode
+   * @return accepted/not
+   */
+  boolean acceptNodePartition(String nodePartition,
+  SchedulingMode schedulingMode);
+
+  /**
+   * It is possible that one request can accept multiple node partition,
+   * So this method returns primary node partition for pending resource /
+   * headroom calculation.
+   *
+   * @return primary requested node partition
+   */
+  String getPrimaryRequestedNodePartition();
+
+  /**
+   * @return number of unique location asks with #pending greater than 0,
+   * (like /rack1, host1, etc.).
+   *
+   * TODO (wangda): This should be moved out of SchedulingPlacementSet
+   * and should belong to specific delay scheduling policy impl.
+   */
+  int getUniqueLocationAsks();
+
+  /**
+   * Print human-readable requests to LOG debug.
+   */
+  void showRequests();
 }

[2/2] hadoop git commit: YARN-6040. Introduce api independent PendingAsk to replace usage of ResourceRequest within Scheduler classes. (Wangda Tan via asuresh)

[asuresh]

YARN-6040. Introduce api independent PendingAsk to replace usage of 
ResourceRequest within Scheduler classes. (Wangda Tan via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2977bc6a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2977bc6a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2977bc6a

Branch: refs/heads/trunk
Commit: 2977bc6a141041ef7579efc416e93fc55e0c2a1a
Parents: 4a659ff
Author: Arun Suresh 
Authored: Fri Jan 6 09:59:57 2017 -0800
Committer: Arun Suresh 
Committed: Fri Jan 6 09:59:57 2017 -0800

--
 .../scheduler/AppSchedulingInfo.java| 163 +++--
 .../scheduler/SchedulerApplicationAttempt.java  |  44 ++--
 .../allocator/AbstractContainerAllocator.java   |   4 +
 .../allocator/RegularContainerAllocator.java| 132 +-
 .../scheduler/common/PendingAsk.java|  57 +
 .../scheduler/common/fica/FiCaSchedulerApp.java |  55 +++--
 .../scheduler/fair/FSAppAttempt.java| 160 +++--
 .../scheduler/fair/FSPreemptionThread.java  |  11 +-
 .../scheduler/fifo/FifoAppAttempt.java  |  10 +-
 .../scheduler/fifo/FifoScheduler.java   | 121 +-
 .../LocalitySchedulingPlacementSet.java | 129 --
 .../placement/SchedulingPlacementSet.java   |  71 +-
 .../scheduler/TestAbstractYarnScheduler.java|  15 +-
 .../scheduler/TestAppSchedulingInfo.java|   4 +-
 .../TestSchedulerApplicationAttempt.java|   3 +-
 .../capacity/TestCapacityScheduler.java | 238 +--
 .../scheduler/capacity/TestLeafQueue.java   |  67 +++---
 .../TestNodeLabelContainerAllocation.java   |  65 +++--
 .../scheduler/capacity/TestReservations.java|  22 +-
 19 files changed, 803 insertions(+), 568 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2977bc6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index d901d90..e2ff082 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -34,16 +34,18 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.LocalitySchedulingPlacementSet;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.ResourceRequestUpdateResult;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
 
 import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -582,16 +584,10 @@ public class AppSchedulingInfo {
 return schedulerKeys.keySet();
   }
 
-  @SuppressWarnings("unchecked")
-  public Map getResourceRequests(
-  SchedulerRequestKey schedulerKey) {
-SchedulingPlacementSet ps = schedulerKeyToPlacementSets.get(schedulerKey);
-if (null != ps) {
-  return ps.getResourceRequests();
-}
-return Collections.emptyMap();
-  }
-
+  /**
+   * Used by REST API to fetch ResourceRequest
+   * @return All pending ResourceRequests.
+   */
   public List getAllResourceRequests() {
 List ret = new ArrayList<>();
 try {
@@ -605,53 +601,51 @@ public class AppSchedulingInfo {
 return ret;
   }
 
-  public ResourceRequest 

hadoop git commit: YARN-5906. Update AppSchedulingInfo to use SchedulingPlacementSet. Contributed by Wangda Tan.

[sunilg]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 6f35700f0 -> 65e7ae5dc


YARN-5906. Update AppSchedulingInfo to use SchedulingPlacementSet. Contributed 
by Wangda Tan.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/65e7ae5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/65e7ae5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/65e7ae5d

Branch: refs/heads/branch-2
Commit: 65e7ae5dcf5b5141446b28cfc6a0f828f4eb833a
Parents: 6f35700
Author: Sunil G 
Authored: Fri Jan 6 21:30:52 2017 +0530
Committer: Sunil G 
Committed: Fri Jan 6 21:30:52 2017 +0530

--
 .../scheduler/AppSchedulingInfo.java| 427 +--
 .../LocalitySchedulingPlacementSet.java | 311 ++
 .../placement/SchedulingPlacementSet.java   |  22 +-
 .../TestApplicationLimitsByPartition.java   |   6 +
 4 files changed, 445 insertions(+), 321 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/65e7ae5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 80811b1..b9deb6c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -18,22 +18,6 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.commons.collections.IteratorUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -48,14 +32,28 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.LocalitySchedulingPlacementSet;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.ResourceRequestUpdateResult;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
 /**
  * This class keeps track of all the consumption of an application. This also
  * keeps track of current running/completed containers for the application.
@@ -87,8 +85,8 @@ public class AppSchedulingInfo {
 
   private final ConcurrentSkipListMap
   schedulerKeys = new ConcurrentSkipListMap<>();
-  final Map>
-  resourceRequestMap = new ConcurrentHashMap<>();
+  final 

hadoop git commit: HDFS-11149. Support for parallel checking of FsVolumes.

[arp]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 fd3b1ca26 -> 6f35700f0


HDFS-11149. Support for parallel checking of FsVolumes.

Change-Id: I6e7ada81562f380a260f8f561d0fa6a4cb102560


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f35700f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f35700f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f35700f

Branch: refs/heads/branch-2
Commit: 6f35700f0427348a60a65abcea20816d13cb3ecf
Parents: fd3b1ca
Author: Arpit Agarwal 
Authored: Tue Nov 29 20:31:02 2016 -0800
Committer: Arpit Agarwal 
Committed: Fri Jan 6 07:55:19 2017 -0800

--
 .../datanode/checker/DatasetVolumeChecker.java  | 442 +++
 .../server/datanode/fsdataset/FsDatasetSpi.java |   7 +
 .../server/datanode/fsdataset/FsVolumeSpi.java  |  12 +-
 .../datanode/fsdataset/impl/FsVolumeImpl.java   |  21 +-
 .../src/main/resources/hdfs-default.xml |  10 +-
 .../server/datanode/SimulatedFSDataset.java |   7 +
 .../server/datanode/TestDirectoryScanner.java   |   7 +
 .../checker/TestDatasetVolumeChecker.java   | 259 +++
 .../TestDatasetVolumeCheckerFailures.java   | 190 
 .../datanode/extdataset/ExternalVolumeImpl.java |   7 +
 10 files changed, 951 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f35700f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java
new file mode 100644
index 000..e49ec7d
--- /dev/null
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java
@@ -0,0 +1,442 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.checker;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import 
org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.VolumeCheckContext;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+import org.apache.hadoop.util.Timer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.nio.channels.ClosedChannelException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY;
+import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DISK_CHECK_TIMEOUT_DEFAULT;
+import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DISK_CHECK_TIMEOUT_KEY;
+import static 
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT;
+import static 

[Hadoop Wiki] Update of "ConnectionRefused" by SteveLoughran

[Apache Wiki]

Dear Wiki user,

You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change 
notification.

The "ConnectionRefused" page has been changed by SteveLoughran:
https://wiki.apache.org/hadoop/ConnectionRefused?action=diff=16=17

Comment:
fix name

   1. If you are using a Hadoop-based product from a third party, -please use 
the support channels provided by the vendor.
   1. Please do not file bug reports related to your problem, as they will be 
closed as [[http://wiki.apache.org/hadoop/InvalidJiraIssues|Invalid]]
  
- See also 
[[http://serverfault.com/questions/725262/what-causes-the-connection-refused-message|Stack
 Overflow]]
+ See also 
[[http://serverfault.com/questions/725262/what-causes-the-connection-refused-message|Server
 Overflow]]
  
  None of these are Hadoop problems, they are hadoop, host, network and 
firewall configuration issues. As it is your cluster, 
[[YourNetworkYourProblem|only you can find out and track down the problem.]]
  

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[Hadoop Wiki] Update of "ConnectionRefused" by SteveLoughran

[Apache Wiki]

Dear Wiki user,

You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change 
notification.

The "ConnectionRefused" page has been changed by SteveLoughran:
https://wiki.apache.org/hadoop/ConnectionRefused?action=diff=15=16

Comment:
ref to stack overflow

   1. If you are using a Hadoop-based product from a third party, -please use 
the support channels provided by the vendor.
   1. Please do not file bug reports related to your problem, as they will be 
closed as [[http://wiki.apache.org/hadoop/InvalidJiraIssues|Invalid]]
  
- None of these are Hadoop problems, they are host, network and firewall 
configuration issues. As it is your cluster, [[YourNetworkYourProblem|only you 
can find out and track down the problem.]]
+ See also 
[[http://serverfault.com/questions/725262/what-causes-the-connection-refused-message|Stack
 Overflow]]
  
+ None of these are Hadoop problems, they are hadoop, host, network and 
firewall configuration issues. As it is your cluster, 
[[YourNetworkYourProblem|only you can find out and track down the problem.]]
+ 

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org