hadoop git commit: YARN-8827. Plumb aggregated application resource utilization from the NM to RM. (asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-1011 bb5991423 -> 608f00998


YARN-8827. Plumb aggregated application resource utilization from the NM to RM. 
(asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/608f0099
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/608f0099
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/608f0099

Branch: refs/heads/YARN-1011
Commit: 608f009980b779857b3660a3bd6c70ee44738b8f
Parents: bb59914
Author: Arun Suresh 
Authored: Tue Oct 9 21:09:50 2018 -0700
Committer: Arun Suresh 
Committed: Tue Oct 9 21:09:50 2018 -0700

--
 .../hadoop/yarn/sls/nodemanager/NodeInfo.java   |   6 +
 .../yarn/sls/scheduler/RMNodeWrapper.java   |   6 +
 .../yarn/api/records/ResourceUtilization.java   |  24 ++
 .../yarn/server/api/records/NodeStatus.java |  19 +
 .../api/records/impl/pb/NodeStatusPBImpl.java   |  59 +++
 .../main/proto/yarn_server_common_protos.proto  |   6 +
 .../nodemanager/NodeStatusUpdaterImpl.java  |   9 +
 .../monitor/ContainersMonitor.java  |  33 ++
 .../monitor/ContainersMonitorImpl.java  |  53 ++-
 .../server/resourcemanager/ResourceManager.java |   8 +
 .../ResourceUtilizationAggregator.java  | 178 +
 .../server/resourcemanager/rmnode/RMNode.java   |   7 +
 .../resourcemanager/rmnode/RMNodeImpl.java  |  26 +-
 .../rmnode/RMNodeStatusEvent.java   |   7 +-
 .../yarn/server/resourcemanager/MockNM.java |  26 +-
 .../yarn/server/resourcemanager/MockNodes.java  |  16 +
 .../TestResourceUtilizationAggregator.java  | 357 +++
 17 files changed, 820 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/608f0099/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 350f4a3..716a1d8 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -199,6 +199,12 @@ public class NodeInfo {
 }
 
 @Override
+public Map
+getAggregatedAppUtilizations() {
+  return null;
+}
+
+@Override
 public ResourceUtilization getNodeUtilization() {
   return null;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/608f0099/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index bb6fb9d..0e2a84e 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -225,6 +225,12 @@ public class RMNodeWrapper implements RMNode {
   }
 
   @Override
+  public Map
+  getAggregatedAppUtilizations() {
+return node.getAggregatedAppUtilizations();
+  }
+
+  @Override
   public Resource getPhysicalResource() {
 return null;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/608f0099/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
index 2ae4872..c340093 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
@@ -53,6 +53,16 @@ public abstract class ResourceUtilization implements
   }
 
   /**
+   * Helper function to return a zero-ed out Utilization.
+   * @return New Resource Utilization.
+   */
+  @Public
+  @Unstable
+  public static ResourceUtilization newZero() {
+return newInstance(0, 0, 0.0f);
+  }
+
+  /**
* Get used virtual memory.
*
* @return virtual memory in MB
@@ -157,6 +167,20 @@ public abstract class ResourceUtilization implements
   }
 
   /**
+   * 

hadoop git commit: YARN-8808. Use aggregate container utilization instead of node utilization to determine resources available for oversubscription. (Haibo Chen via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-1011 8d217ee3c -> efd852449


YARN-8808. Use aggregate container utilization instead of node utilization to 
determine resources available for oversubscription. (Haibo Chen via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efd85244
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efd85244
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efd85244

Branch: refs/heads/YARN-1011
Commit: efd852449f8db899553fa351ece06cbb53823fc0
Parents: 8d217ee
Author: Arun Suresh 
Authored: Wed Oct 3 11:22:16 2018 -0700
Committer: Arun Suresh 
Committed: Wed Oct 3 11:22:16 2018 -0700

--
 .../scheduler/SchedulerNode.java| 10 -
 .../yarn/server/resourcemanager/MockNodes.java  |  8 +---
 .../scheduler/fair/TestFairScheduler.java   | 42 ++--
 3 files changed, 32 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/efd85244/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 9e38d52..8d47f34 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -731,8 +731,16 @@ public abstract class SchedulerNode {
   return Resources.none();
 }
 
+ResourceUtilization aggregateContainersUtilization =
+getAggregatedContainersUtilization();
+if (aggregateContainersUtilization == null) {
+  // be conservative if the aggregate container utilization is unknown
+  return Resources.none();
+
+}
+
 ResourceUtilization projectedNodeUtilization = ResourceUtilization.
-newInstance(getNodeUtilization());
+newInstance(aggregateContainersUtilization);
 // account for resources allocated in this heartbeat
 projectedNodeUtilization.addTo(
 (int) (resourceAllocatedPendingLaunch.getMemorySize()), 0,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/efd85244/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index bff6125..d841ff0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -330,17 +330,13 @@ public class MockNodes {
   return this.physicalResource;
 }
 
-public void updateResourceUtilization(ResourceUtilization utilization) {
-  this.nodeUtilization = utilization;
-}
-
-public void updateContainersAndNodeUtilization(
+public void updateContainersInfoAndUtilization(
 UpdatedContainerInfo updatedContainerInfo,
 ResourceUtilization resourceUtilization) {
   if (updatedContainerInfo != null) {
 containerUpdates = Collections.singletonList(updatedContainerInfo);
   }
-  this.nodeUtilization = resourceUtilization;
+  this.containersUtilization = resourceUtilization;
 }
   };
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/efd85244/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/

hadoop git commit: YARN-7974. Allow updating application tracking url after registration. (Jonathan Hung via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 c617dba49 -> 4d69741a6


YARN-7974. Allow updating application tracking url after registration. 
(Jonathan Hung via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d69741a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d69741a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d69741a

Branch: refs/heads/branch-2
Commit: 4d69741a61b8d29774a1e5f5075406808fc0594a
Parents: c617dba
Author: Arun Suresh 
Authored: Wed Sep 26 00:08:10 2018 -0700
Committer: Arun Suresh 
Committed: Wed Sep 26 00:08:10 2018 -0700

--
 .../api/protocolrecords/AllocateRequest.java| 47 +++-
 .../src/main/proto/yarn_service_protos.proto|  1 +
 .../hadoop/yarn/client/api/AMRMClient.java  | 11 +++
 .../yarn/client/api/async/AMRMClientAsync.java  | 11 +++
 .../api/async/impl/AMRMClientAsyncImpl.java |  5 ++
 .../yarn/client/api/impl/AMRMClientImpl.java| 11 +++
 .../yarn/client/api/impl/TestAMRMClient.java| 77 
 .../impl/pb/AllocateRequestPBImpl.java  | 27 ++-
 .../resourcemanager/DefaultAMSProcessor.java|  2 +-
 .../rmapp/attempt/RMAppAttemptImpl.java | 22 ++
 .../event/RMAppAttemptStatusupdateEvent.java| 11 +++
 .../TestApplicationMasterService.java   | 34 +
 .../server/resourcemanager/TestRMRestart.java   | 45 
 13 files changed, 300 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
index da68d8c..8f6a204 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
@@ -100,7 +100,21 @@ public abstract class AllocateRequest {
 allocateRequest.setIncreaseRequests(increaseRequests);
 return allocateRequest;
   }
-  
+
+  @Public
+  @Unstable
+  public static AllocateRequest newInstance(int responseID, float appProgress,
+  List resourceAsk,
+  List containersToBeReleased,
+  ResourceBlacklistRequest resourceBlacklistRequest,
+  String trackingUrl) {
+return AllocateRequest.newBuilder().responseId(responseID)
+.progress(appProgress).askList(resourceAsk)
+.releaseList(containersToBeReleased)
+.resourceBlacklistRequest(resourceBlacklistRequest)
+.trackingUrl(trackingUrl).build();
+  }
+
   @Public
   @Unstable
   public static AllocateRequest newInstance(int responseID, float appProgress,
@@ -256,6 +270,22 @@ public abstract class AllocateRequest {
   public abstract void setUpdateRequests(
   List updateRequests);
 
+  /**
+   * Get the tracking url update for this heartbeat.
+   * @return tracking url to update this application with
+   */
+  @Public
+  @Unstable
+  public abstract String getTrackingUrl();
+
+  /**
+   * Set the new tracking url for this application.
+   * @param trackingUrl the new tracking url
+   */
+  @Public
+  @Unstable
+  public abstract void setTrackingUrl(String trackingUrl);
+
   @Public
   @Unstable
   public static AllocateRequestBuilder newBuilder() {
@@ -358,6 +388,19 @@ public abstract class AllocateRequest {
 }
 
 /**
+ * Set the trackingUrl of the request.
+ * @see AllocateRequest#setTrackingUrl(String)
+ * @param trackingUrl new tracking url
+ * @return {@link AllocateRequestBuilder}
+ */
+@Public
+@Unstable
+public AllocateRequestBuilder trackingUrl(String trackingUrl) {
+  allocateRequest.setTrackingUrl(trackingUrl);
+  return this;
+}
+
+/**
  * Return generated {@link AllocateRequest} object.
  * @return {@link AllocateRequest}
  */
@@ -367,4 +410,4 @@ public abstract class AllocateRequest {
   return allocateRequest;
 }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/pr

hadoop git commit: YARN-8809. Refactor AbstractYarnScheduler and CapacityScheduler OPPORTUNISTIC container completion codepaths. (Haibo Chen via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-1011 36ec27ed3 -> 4d858dd92


YARN-8809. Refactor AbstractYarnScheduler and CapacityScheduler OPPORTUNISTIC 
container completion codepaths. (Haibo Chen via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d858dd9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d858dd9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d858dd9

Branch: refs/heads/YARN-1011
Commit: 4d858dd92d0cf388330c41e984fde5fd0c5aa95a
Parents: 36ec27e
Author: Arun Suresh 
Authored: Fri Sep 21 12:02:00 2018 -0700
Committer: Arun Suresh 
Committed: Fri Sep 21 12:02:00 2018 -0700

--
 .../scheduler/AbstractYarnScheduler.java| 32 
 .../scheduler/capacity/CapacityScheduler.java   | 25 ++-
 .../scheduler/fair/FairScheduler.java   | 11 ---
 .../scheduler/fifo/FifoScheduler.java   |  2 +-
 .../scheduler/fair/TestFairScheduler.java   | 25 +++
 5 files changed, 70 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d858dd9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index a05ee26..d382263 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -677,25 +677,12 @@ public abstract class AbstractYarnScheduler
 }
 
 if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
-  completedContainerInternal(rmContainer, containerStatus, event);
+  completeGuaranteedContainerInternal(rmContainer, containerStatus, event);
   completeOustandingUpdatesWhichAreReserved(
   rmContainer, containerStatus, event);
 } else {
-  ContainerId containerId = rmContainer.getContainerId();
-  // Inform the container
-  rmContainer.handle(
-  new RMContainerFinishedEvent(containerId, containerStatus, event));
-  SchedulerApplicationAttempt schedulerAttempt =
-  getCurrentAttemptForContainer(containerId);
-  if (schedulerAttempt != null) {
-schedulerAttempt.removeRMContainer(containerId);
-  }
-  if (LOG.isDebugEnabled()) {
-LOG.debug("Completed container: " + rmContainer.getContainerId() +
-" in state: " + rmContainer.getState() + " event:" + event);
-  }
-  getSchedulerNode(rmContainer.getNodeId()).releaseContainer(
-  rmContainer.getContainerId(), false);
+  completeOpportunisticContainerInternal(rmContainer, containerStatus,
+  event);
 }
 
 // If the container is getting killed in ACQUIRED state, the requester (AM
@@ -705,6 +692,12 @@ public abstract class AbstractYarnScheduler
 recoverResourceRequestForContainer(rmContainer);
   }
 
+  protected void completeOpportunisticContainerInternal(
+  RMContainer rmContainer, ContainerStatus containerStatus,
+  RMContainerEventType event) {
+completeGuaranteedContainerInternal(rmContainer, containerStatus, event);
+  }
+
   // Optimization:
   // Check if there are in-flight container updates and complete the
   // associated temp containers. These are removed when the app completes,
@@ -722,7 +715,7 @@ public abstract class AbstractYarnScheduler
 .getReservedSchedulerKey().getContainerToUpdate();
 if (containerToUpdate != null &&
 containerToUpdate.equals(containerStatus.getContainerId())) {
-  completedContainerInternal(resContainer,
+  completeGuaranteedContainerInternal(resContainer,
   ContainerStatus.newInstance(resContainer.getContainerId(),
   containerStatus.getState(), containerStatus
   .getDiagnostics(),
@@ -732,8 +725,9 @@ public abstract class AbstractYarnScheduler
 }
   }
 
-  // clean up a completed container
-  protected abstract void completedContainerInternal(RMContainer rmContainer,
+  // clean up a co

hadoop git commit: MAPREDUCE-7101. Add config parameter to allow JHS to alway scan user dir irrespective of modTime. (Thomas Marquardt via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b3f4c2766 -> 358fe4c61


MAPREDUCE-7101. Add config parameter to allow JHS to alway scan user dir 
irrespective of modTime. (Thomas Marquardt via asuresh)

(cherry picked from commit 5670e89b2ec69ab71e32dcd5acbd3a57ca6abea5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/358fe4c6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/358fe4c6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/358fe4c6

Branch: refs/heads/branch-2
Commit: 358fe4c61a563ec70dbf96164abd28f28677ef9a
Parents: b3f4c27
Author: Arun Suresh 
Authored: Tue Jun 12 15:36:52 2018 -0700
Committer: Arun Suresh 
Committed: Tue Jun 12 16:25:01 2018 -0700

--
 .../hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java   | 9 +++--
 .../src/main/resources/mapred-default.xml   | 9 +
 .../apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java   | 8 +++-
 3 files changed, 23 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/358fe4c6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index 644727f..9c3e668 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -61,8 +61,13 @@ public class JHAdminConfig {
 MR_HISTORY_PREFIX + "cleaner.interval-ms";
   public static final long DEFAULT_MR_HISTORY_CLEANER_INTERVAL_MS = 
 1 * 24 * 60 * 60 * 1000l; //1 day
-  
-  
+
+  /** Always scan user dir, irrespective of dir modification time.*/
+  public static final String MR_HISTORY_ALWAYS_SCAN_USER_DIR =
+  MR_HISTORY_PREFIX + "always-scan-user-dir";
+  public static final boolean DEFAULT_MR_HISTORY_ALWAYS_SCAN_USER_DIR =
+  false;
+
   /** The number of threads to handle client API requests.*/
   public static final String MR_HISTORY_CLIENT_THREAD_COUNT = 
 MR_HISTORY_PREFIX + "client.thread-count";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/358fe4c6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index a33960c..8b133ed 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1719,6 +1719,15 @@
 
 
 
+  mapreduce.jobhistory.always-scan-user-dir
+  false
+  Some Cloud FileSystems do not currently update the
+  modification time of directories. To support these filesystems, this
+  configuration value should be set to 'true'.
+  
+
+
+
   mapreduce.jobhistory.done-dir
   ${yarn.app.mapreduce.am.staging-dir}/history/done
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/358fe4c6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 6fd1c0e..9ba00ad 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -322,7 +322,13 @@ public class HistoryFileManager extends AbstractServi

hadoop git commit: MAPREDUCE-7101. Add config parameter to allow JHS to alway scan user dir irrespective of modTime. (Thomas Marquardt via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/trunk aeaf9fec6 -> 5670e89b2


MAPREDUCE-7101. Add config parameter to allow JHS to alway scan user dir 
irrespective of modTime. (Thomas Marquardt via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5670e89b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5670e89b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5670e89b

Branch: refs/heads/trunk
Commit: 5670e89b2ec69ab71e32dcd5acbd3a57ca6abea5
Parents: aeaf9fe
Author: Arun Suresh 
Authored: Tue Jun 12 15:36:52 2018 -0700
Committer: Arun Suresh 
Committed: Tue Jun 12 15:36:52 2018 -0700

--
 .../hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java   | 9 +++--
 .../src/main/resources/mapred-default.xml   | 9 +
 .../apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java   | 8 +++-
 3 files changed, 23 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5670e89b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index 1cadf84..9e964e1 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -61,8 +61,13 @@ public class JHAdminConfig {
 MR_HISTORY_PREFIX + "cleaner.interval-ms";
   public static final long DEFAULT_MR_HISTORY_CLEANER_INTERVAL_MS = 
 1 * 24 * 60 * 60 * 1000l; //1 day
-  
-  
+
+  /** Always scan user dir, irrespective of dir modification time.*/
+  public static final String MR_HISTORY_ALWAYS_SCAN_USER_DIR =
+  MR_HISTORY_PREFIX + "always-scan-user-dir";
+  public static final boolean DEFAULT_MR_HISTORY_ALWAYS_SCAN_USER_DIR =
+  false;
+
   /** The number of threads to handle client API requests.*/
   public static final String MR_HISTORY_CLIENT_THREAD_COUNT = 
 MR_HISTORY_PREFIX + "client.thread-count";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5670e89b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index dcb312c..9f33d65 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1775,6 +1775,15 @@
 
 
 
+  mapreduce.jobhistory.always-scan-user-dir
+  false
+  Some Cloud FileSystems do not currently update the
+  modification time of directories. To support these filesystems, this
+  configuration value should be set to 'true'.
+  
+
+
+
   mapreduce.jobhistory.done-dir
   ${yarn.app.mapreduce.am.staging-dir}/history/done
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5670e89b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index a07ca26..7fe99a2 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -324,7 +324,13 @@ public class HistoryFileManager extends AbstractService {
   // so we need to have additional check.
   // Note: modTi

hadoop git commit: YARN-7900. [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor. (Botong Huang via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 734cbdd6b -> 113e2d680


YARN-7900. [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor. 
(Botong Huang via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/113e2d68
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/113e2d68
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/113e2d68

Branch: refs/heads/branch-2
Commit: 113e2d6801e06f90dcd2c1110bcebab9be06a60b
Parents: 734cbdd
Author: Arun Suresh <asur...@apache.org>
Authored: Mon May 21 11:26:32 2018 -0700
Committer: Arun Suresh <asur...@apache.org>
Committed: Mon May 21 11:26:32 2018 -0700

--
 .../hadoop/yarn/client/AMRMClientUtils.java | 190 +++
 .../hadoop/yarn/server/AMRMClientRelayer.java   | 336 +++
 .../failover/FederationProxyProviderUtil.java   |   3 +-
 .../apache/hadoop/yarn/server/package-info.java |  18 +
 .../server/scheduler/ResourceRequestSet.java| 206 
 .../server/scheduler/ResourceRequestSetKey.java | 133 
 .../server/scheduler/SchedulerRequestKey.java   |   4 +-
 .../yarn/server/uam/UnmanagedAMPoolManager.java |   2 +-
 .../server/uam/UnmanagedApplicationManager.java |   2 +-
 .../yarn/server/utils/AMRMClientUtils.java  | 191 ---
 .../yarn/server/MockResourceManagerFacade.java  |   2 +-
 .../yarn/server/TestAMRMClientRelayer.java  | 275 +++
 .../amrmproxy/FederationInterceptor.java|   2 +-
 .../ApplicationMasterService.java   |   2 +-
 .../TestApplicationMasterLauncher.java  |   2 +-
 15 files changed, 1166 insertions(+), 202 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/113e2d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java
new file mode 100644
index 000..7f624859
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import 
org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
+import 
org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
+import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import 
org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
+import 
org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class for

[1/2] hadoop git commit: YARN-7900. [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor. (Botong Huang via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/trunk f749517cc -> 3159bffce


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3159bffc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
index e518b90..38181e2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.SerializedException;
 import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.client.AMRMClientUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
 import 
org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
@@ -78,7 +79,6 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
-import org.apache.hadoop.yarn.server.utils.AMRMClientUtils;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[2/2] hadoop git commit: YARN-7900. [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor. (Botong Huang via asuresh)

[asuresh]

YARN-7900. [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor. 
(Botong Huang via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3159bffc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3159bffc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3159bffc

Branch: refs/heads/trunk
Commit: 3159bffce23abf35754da2d7d51de7d8c2631ae3
Parents: f749517
Author: Arun Suresh <asur...@apache.org>
Authored: Thu May 17 20:00:52 2018 -0700
Committer: Arun Suresh <asur...@apache.org>
Committed: Thu May 17 20:00:52 2018 -0700

--
 .../yarn/client/api/impl/AMRMClientImpl.java| 151 ++--
 .../hadoop/yarn/client/AMRMClientUtils.java | 262 +
 .../hadoop/yarn/server/AMRMClientRelayer.java   | 364 +++
 .../failover/FederationProxyProviderUtil.java   |   2 +-
 .../apache/hadoop/yarn/server/package-info.java |  18 +
 .../server/scheduler/ResourceRequestSet.java| 206 +++
 .../server/scheduler/ResourceRequestSetKey.java | 133 +++
 .../server/scheduler/SchedulerRequestKey.java   |   4 +-
 .../yarn/server/uam/UnmanagedAMPoolManager.java |   2 +-
 .../server/uam/UnmanagedApplicationManager.java |   2 +-
 .../yarn/server/utils/AMRMClientUtils.java  | 191 --
 .../yarn/server/MockResourceManagerFacade.java  |   2 +-
 .../yarn/server/TestAMRMClientRelayer.java  | 275 ++
 .../amrmproxy/FederationInterceptor.java|   2 +-
 .../ApplicationMasterService.java   |   2 +-
 .../TestApplicationMasterLauncher.java  |   2 +-
 16 files changed, 1299 insertions(+), 319 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3159bffc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index ef849b2..36c3cf1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -31,11 +31,9 @@ import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Queue;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.AbstractMap.SimpleEntry;
-import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -68,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
 import org.apache.hadoop.yarn.api.records.UpdatedContainer;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.client.AMRMClientUtils;
 import org.apache.hadoop.yarn.client.ClientRMProxy;
 import org.apache.hadoop.yarn.client.api.AMRMClient;
 import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
@@ -113,13 +112,9 @@ public class AMRMClientImpl 
extends AMRMClient {
   protected final Set blacklistRemovals = new HashSet();
   private Map<Set, PlacementConstraint> placementConstraints =
   new HashMap<>();
-  private Queue<Collection> batchedSchedulingRequests =
-  new LinkedList<>();
-  private Map<Set, List> outstandingSchedRequests =
-  new ConcurrentHashMap<>();
 
   protected Map<String, Resource> resourceProfilesMap;
-  
+
   static class ResourceRequestInfo {
 ResourceRequest remoteRequest;
 LinkedHashSet containerRequests;
@@ -168,6 +163,10 @@ public class AMRMClientImpl 
extends AMRMClient {
   SimpleEntry<Container, UpdateContainerRequest>> pendingChange =
   new HashMap<>();
 
+  private List schedulingRequests = new ArrayList<>();
+  private Map<Set, List> outstandingSchedRequests =
+  new HashMap<>();
+
   public AMRMClientImpl() {
 super(AMRMClientImpl.class.getName());
   }
@@ -252,18 +251,18 @@ public class AMRMClientImpl 
extends AMRMClient {
   this.resourceProfilesMap = response.getResourceProfiles();
   List prevContainers =
   response.getContainersFromPreviousAttempts();
-  removeFromOutstandingSchedulingRequests(prevContainers);
-  recreateSchedulingRequestBatch();
+  AMRMClientUtils.removeFromOutstandingSchedulingRequests

hadoop git commit: HADOOP-15446. WASB: PageBlobInputStream.skip breaks HBASE replication. Contributed by Thomas Marquardt

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.9 019ba5998 -> 1f32345cd


HADOOP-15446. WASB: PageBlobInputStream.skip breaks HBASE replication.
Contributed by Thomas Marquardt

(cherry picked from commit 5b11b9fd413470e134ecdc7c50468f8c7b39fa50)
(cherry picked from commit 67468651b19ae9de04d20decb79c23e2541df584)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f32345c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f32345c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f32345c

Branch: refs/heads/branch-2.9
Commit: 1f32345cd0d59a14a264933509d143aecb82db5e
Parents: 019ba59
Author: Steve Loughran 
Authored: Mon May 7 11:54:08 2018 +0100
Committer: Arun Suresh 
Committed: Thu May 10 10:46:14 2018 -0700

--
 .../hadoop/fs/azure/PageBlobInputStream.java| 123 +++--
 .../fs/azure/ITestPageBlobInputStream.java  | 527 +++
 2 files changed, 605 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f32345c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
--
diff --git 
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
 
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
index 097201b..40bf6f4 100644
--- 
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
+++ 
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
@@ -25,12 +25,14 @@ import static 
org.apache.hadoop.fs.azure.PageBlobFormatHelpers.toShort;
 import static org.apache.hadoop.fs.azure.PageBlobFormatHelpers.withMD5Checking;
 
 import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSExceptionMessages;
 import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper;
 
 import com.microsoft.azure.storage.OperationContext;
@@ -58,7 +60,9 @@ final class PageBlobInputStream extends InputStream {
   // The buffer holding the current data we last read from the server.
   private byte[] currentBuffer;
   // The current byte offset we're at in the buffer.
-  private int currentOffsetInBuffer;
+  private int currentBufferOffset;
+  // The current buffer length
+  private int currentBufferLength;
   // Maximum number of pages to get per any one request.
   private static final int MAX_PAGES_PER_DOWNLOAD =
   4 * 1024 * 1024 / PAGE_SIZE;
@@ -174,7 +178,7 @@ final class PageBlobInputStream extends InputStream {
 
   private boolean dataAvailableInBuffer() {
 return currentBuffer != null 
-&& currentOffsetInBuffer < currentBuffer.length;
+&& currentBufferOffset < currentBufferLength;
   }
 
   /**
@@ -194,6 +198,8 @@ final class PageBlobInputStream extends InputStream {
   return true;
 }
 currentBuffer = null;
+currentBufferOffset = 0;
+currentBufferLength = 0;
 if (numberOfPagesRemaining == 0) {
   // No more data to read.
   return false;
@@ -209,43 +215,48 @@ final class PageBlobInputStream extends InputStream {
   ByteArrayOutputStream baos = new ByteArrayOutputStream(bufferSize);
   blob.downloadRange(currentOffsetInBlob, bufferSize, baos,
   withMD5Checking(), opContext);
-  currentBuffer = baos.toByteArray();
+  validateDataIntegrity(baos.toByteArray());
 } catch (StorageException e) {
   throw new IOException(e);
 }
 numberOfPagesRemaining -= pagesToRead;
 currentOffsetInBlob += bufferSize;
-currentOffsetInBuffer = PAGE_HEADER_SIZE;
-
-// Since we just downloaded a new buffer, validate its consistency.
-validateCurrentBufferConsistency();
 
 return true;
   }
 
-  private void validateCurrentBufferConsistency()
+  private void validateDataIntegrity(byte[] buffer)
   throws IOException {
-if (currentBuffer.length % PAGE_SIZE != 0) {
+
+if (buffer.length % PAGE_SIZE != 0) {
   throw new AssertionError("Unexpected buffer size: " 
-  + currentBuffer.length);
+  + buffer.length);
 }
-int numberOfPages = currentBuffer.length / PAGE_SIZE;
+
+int bufferLength = 0;
+int numberOfPages = buffer.length / PAGE_SIZE;
+long totalPagesAfterCurrent = numberOfPagesRemaining;
+
 for (int page = 0; page < numberOfPages; page++) {
-  short currentPageSize = getPageSize(blob, currentBuffer,
-  page * PAGE_SIZE);
-  // Calculate the number of pages that exist after this one

hadoop git commit: HADOOP-15446. WASB: PageBlobInputStream.skip breaks HBASE replication. Contributed by Thomas Marquardt

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 ed592a210 -> 67468651b


HADOOP-15446. WASB: PageBlobInputStream.skip breaks HBASE replication.
Contributed by Thomas Marquardt

(cherry picked from commit 5b11b9fd413470e134ecdc7c50468f8c7b39fa50)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67468651
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67468651
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67468651

Branch: refs/heads/branch-2
Commit: 67468651b19ae9de04d20decb79c23e2541df584
Parents: ed592a2
Author: Steve Loughran 
Authored: Mon May 7 11:54:08 2018 +0100
Committer: Arun Suresh 
Committed: Thu May 10 10:33:03 2018 -0700

--
 .../hadoop/fs/azure/PageBlobInputStream.java| 123 +++--
 .../fs/azure/ITestPageBlobInputStream.java  | 527 +++
 2 files changed, 605 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/67468651/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
--
diff --git 
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
 
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
index aaac490..40bf6f4 100644
--- 
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
+++ 
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
@@ -25,12 +25,14 @@ import static 
org.apache.hadoop.fs.azure.PageBlobFormatHelpers.toShort;
 import static org.apache.hadoop.fs.azure.PageBlobFormatHelpers.withMD5Checking;
 
 import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSExceptionMessages;
 import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper;
 
 import com.microsoft.azure.storage.OperationContext;
@@ -58,7 +60,9 @@ final class PageBlobInputStream extends InputStream {
   // The buffer holding the current data we last read from the server.
   private byte[] currentBuffer;
   // The current byte offset we're at in the buffer.
-  private int currentOffsetInBuffer;
+  private int currentBufferOffset;
+  // The current buffer length
+  private int currentBufferLength;
   // Maximum number of pages to get per any one request.
   private static final int MAX_PAGES_PER_DOWNLOAD =
   4 * 1024 * 1024 / PAGE_SIZE;
@@ -174,7 +178,7 @@ final class PageBlobInputStream extends InputStream {
 
   private boolean dataAvailableInBuffer() {
 return currentBuffer != null 
-&& currentOffsetInBuffer < currentBuffer.length;
+&& currentBufferOffset < currentBufferLength;
   }
 
   /**
@@ -194,6 +198,8 @@ final class PageBlobInputStream extends InputStream {
   return true;
 }
 currentBuffer = null;
+currentBufferOffset = 0;
+currentBufferLength = 0;
 if (numberOfPagesRemaining == 0) {
   // No more data to read.
   return false;
@@ -209,43 +215,48 @@ final class PageBlobInputStream extends InputStream {
   ByteArrayOutputStream baos = new ByteArrayOutputStream(bufferSize);
   blob.downloadRange(currentOffsetInBlob, bufferSize, baos,
   withMD5Checking(), opContext);
-  currentBuffer = baos.toByteArray();
+  validateDataIntegrity(baos.toByteArray());
 } catch (StorageException e) {
   throw new IOException(e);
 }
 numberOfPagesRemaining -= pagesToRead;
 currentOffsetInBlob += bufferSize;
-currentOffsetInBuffer = PAGE_HEADER_SIZE;
-
-// Since we just downloaded a new buffer, validate its consistency.
-validateCurrentBufferConsistency();
 
 return true;
   }
 
-  private void validateCurrentBufferConsistency()
+  private void validateDataIntegrity(byte[] buffer)
   throws IOException {
-if (currentBuffer.length % PAGE_SIZE != 0) {
+
+if (buffer.length % PAGE_SIZE != 0) {
   throw new AssertionError("Unexpected buffer size: " 
-  + currentBuffer.length);
+  + buffer.length);
 }
-int numberOfPages = currentBuffer.length / PAGE_SIZE;
+
+int bufferLength = 0;
+int numberOfPages = buffer.length / PAGE_SIZE;
+long totalPagesAfterCurrent = numberOfPagesRemaining;
+
 for (int page = 0; page < numberOfPages; page++) {
-  short currentPageSize = getPageSize(blob, currentBuffer,
-  page * PAGE_SIZE);
-  // Calculate the number of pages that exist after this one
-  // in the blob.
-  long totalPagesAfterCurrent =
-  

hadoop git commit: YARN-7972. Support inter-app placement constraints for allocation tags by application ID. (Weiwei Yang via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 2e1e049bd -> 1054b48c2


YARN-7972. Support inter-app placement constraints for allocation tags by 
application ID. (Weiwei Yang via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1054b48c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1054b48c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1054b48c

Branch: refs/heads/trunk
Commit: 1054b48c27f3158110bd0512afecded36eecb8ad
Parents: 2e1e049
Author: Arun Suresh <asur...@apache.org>
Authored: Mon Mar 5 11:24:17 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Mon Mar 5 11:24:17 2018 -0800

--
 .../api/records/AllocationTagNamespace.java | 336 +++
 .../api/records/AllocationTagNamespaceType.java |  74 
 .../hadoop/yarn/api/records/AllocationTags.java |  50 +++
 .../hadoop/yarn/api/records/Evaluable.java  |  38 +++
 .../yarn/api/records/TargetApplications.java|  53 +++
 .../yarn/api/resource/PlacementConstraints.java |  26 +-
 .../InvalidAllocationTagException.java  |  34 ++
 .../constraint/AllocationTagsManager.java   |   9 +
 .../InvalidAllocationTagsQueryException.java|   4 +
 .../constraint/PlacementConstraintsUtil.java|  71 +++-
 .../SingleConstraintAppPlacementAllocator.java  |  31 +-
 .../constraint/TestAllocationTagsNamespace.java | 147 
 .../TestPlacementConstraintsUtil.java   | 151 +
 13 files changed, 1001 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1054b48c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespace.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespace.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespace.java
new file mode 100644
index 000..25f8761
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespace.java
@@ -0,0 +1,336 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.yarn.exceptions.InvalidAllocationTagException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import static 
org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.SELF;
+import static 
org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.NOT_SELF;
+import static 
org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.APP_LABEL;
+import static 
org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.APP_ID;
+import static 
org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.ALL;
+import static 
org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.fromString;
+
+/**
+ * Class to describe the namespace of an allocation tag.
+ * Each namespace can be evaluated against a set of applications.
+ * After evaluation, the namespace should have an implicit set of
+ * applications which defines its scope.
+ */
+public abstract class AllocationTagNamespace implements
+Evaluable {
+
+  public final static String NAMESPACE_DELIMITER = "/";
+
+  private AllocationTagNamespaceType nsType;
+  // Namespace scope value will be delay binding by eval method.
+  private Set nsScope;
+
+  public AllocationTagNamespace(AllocationTagNamespaceType
+  allocationTagNamespaceType) {
+this.nsType = allocationTagNamespaceType;
+  }
+
+  protected void setScopeIfNotNull(Set appIds) {
+if (appIds != null) {
+  this.nsScope = appIds;
+}
+  }
+
+  /**
+   * Get the type of the namespace.
+   * @return namespace type

hadoop git commit: YARN-7918. Fix TestAMRMClientPlacementConstraints. (Gergely Novák via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 f878ddb06 -> 950566230


YARN-7918. Fix TestAMRMClientPlacementConstraints. (Gergely Novák via asuresh)

(cherry picked from commit a1e56a62863d8d494af309ec5f476c4b7e4d5ef9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95056623
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95056623
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95056623

Branch: refs/heads/branch-3.1
Commit: 950566230c5fb2a7e5c5b1eb5328ca155e6d0f45
Parents: f878ddb
Author: Arun Suresh <asur...@apache.org>
Authored: Sat Feb 17 03:24:55 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Sat Feb 17 03:26:13 2018 -0800

--
 .../constraint/processor/PlacementConstraintProcessor.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/95056623/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
index f089a19..cf944a6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
@@ -17,6 +17,7 @@
  */
 package 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor;
 
+import com.google.common.collect.Lists;
 import org.apache.hadoop.yarn.ams.ApplicationMasterServiceContext;
 import org.apache.hadoop.yarn.ams.ApplicationMasterServiceProcessor;
 import org.apache.hadoop.yarn.ams.ApplicationMasterServiceUtils;
@@ -329,7 +330,7 @@ public class PlacementConstraintProcessor extends 
AbstractPlacementProcessor {
 if (!isAdded) {
   BatchedRequests br = new BatchedRequests(iteratorType,
   schedulerResponse.getApplicationId(),
-  Collections.singleton(schedulerResponse.getSchedulingRequest()),
+  Lists.newArrayList(schedulerResponse.getSchedulingRequest()),
   placementAttempt + 1);
   reqsToRetry.add(br);
   br.addToBlacklist(


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7918. Fix TestAMRMClientPlacementConstraints. (Gergely Novák via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 31db977cb -> a1e56a628


YARN-7918. Fix TestAMRMClientPlacementConstraints. (Gergely Novák via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1e56a62
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1e56a62
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1e56a62

Branch: refs/heads/trunk
Commit: a1e56a62863d8d494af309ec5f476c4b7e4d5ef9
Parents: 31db977
Author: Arun Suresh <asur...@apache.org>
Authored: Sat Feb 17 03:24:55 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Sat Feb 17 03:24:55 2018 -0800

--
 .../constraint/processor/PlacementConstraintProcessor.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1e56a62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
index f089a19..cf944a6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
@@ -17,6 +17,7 @@
  */
 package 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor;
 
+import com.google.common.collect.Lists;
 import org.apache.hadoop.yarn.ams.ApplicationMasterServiceContext;
 import org.apache.hadoop.yarn.ams.ApplicationMasterServiceProcessor;
 import org.apache.hadoop.yarn.ams.ApplicationMasterServiceUtils;
@@ -329,7 +330,7 @@ public class PlacementConstraintProcessor extends 
AbstractPlacementProcessor {
 if (!isAdded) {
   BatchedRequests br = new BatchedRequests(iteratorType,
   schedulerResponse.getApplicationId(),
-  Collections.singleton(schedulerResponse.getSchedulingRequest()),
+  Lists.newArrayList(schedulerResponse.getSchedulingRequest()),
   placementAttempt + 1);
   reqsToRetry.add(br);
   br.addToBlacklist(


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7839. Modify PlacementAlgorithm to Check node capacity before placing request on node. (Panagiotis Garefalakis via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 460d77bd6 -> 6e5ba9366


YARN-7839. Modify PlacementAlgorithm to Check node capacity before placing 
request on node. (Panagiotis Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e5ba936
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e5ba936
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e5ba936

Branch: refs/heads/trunk
Commit: 6e5ba9366fc05719906ff2789b1a0fd26001182b
Parents: 460d77b
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Feb 2 10:28:22 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Fri Feb 2 10:28:22 2018 -0800

--
 .../scheduler/capacity/CapacityScheduler.java   |  4 -
 .../algorithm/DefaultPlacementAlgorithm.java| 61 ++
 .../api/ConstraintPlacementAlgorithmOutput.java |  5 +-
 .../SchedulingRequestWithPlacementAttempt.java  | 52 
 .../constraint/processor/BatchedRequests.java   |  2 +-
 .../processor/PlacementDispatcher.java  | 12 +--
 .../processor/PlacementProcessor.java   | 28 +--
 .../constraint/TestPlacementProcessor.java  | 87 +++-
 8 files changed, 215 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e5ba936/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index cb01351..d3aa5cb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2609,10 +2609,6 @@ public class CapacityScheduler extends
 " but only 1 will be attempted !!");
   }
   if (!appAttempt.isStopped()) {
-Resource resource =
-schedulingRequest.getResourceSizing().getResources();
-schedulingRequest.getResourceSizing().setResources(
-getNormalizedResource(resource));
 ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>
 resourceCommitRequest = createResourceCommitRequest(
 appAttempt, schedulingRequest, schedulerNode);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e5ba936/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
index 4e6473f..710e6c0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
@@ -18,10 +18,15 @@
 package 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceSizing;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.server.reso

[hadoop] Git Push Summary

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-7812 [deleted] e6d2d26a1

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7822. Constraint satisfaction checker support for composite OR and AND constraints. (Weiwei Yang via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 8d1e2c640 -> d48134478


YARN-7822. Constraint satisfaction checker support for composite OR and AND 
constraints. (Weiwei Yang via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4813447
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4813447
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4813447

Branch: refs/heads/trunk
Commit: d4813447831770446399f2d6501860141551ff33
Parents: 8d1e2c6
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Jan 30 10:15:33 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:51:08 2018 -0800

--
 .../TestPlacementConstraintTransformations.java |   2 +-
 .../constraint/PlacementConstraintsUtil.java|  53 +++-
 .../TestPlacementConstraintsUtil.java   | 278 ---
 .../constraint/TestPlacementProcessor.java  | 159 ++-
 4 files changed, 444 insertions(+), 48 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4813447/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
index 62da092..aa92d7a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
@@ -156,7 +156,7 @@ public class TestPlacementConstraintTransformations {
 SingleConstraintTransformer singleTransformer =
 new SingleConstraintTransformer(specConstraint);
 PlacementConstraint simConstraint = singleTransformer.transform();
-Assert.assertTrue(constraintExpr instanceof Or);
+Assert.assertTrue(simConstraint.getConstraintExpr() instanceof Or);
 Or simOrExpr = (Or) specConstraint.getConstraintExpr();
 for (AbstractConstraint child : simOrExpr.getChildren()) {
   Assert.assertTrue(child instanceof SingleConstraint);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4813447/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
index 199dd62..6396e57 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType;
@@ -149,6 +151,48 @@ public final class PlacementConstraintsUtil {
 return true;
   }
 
+  /**
+   * Returns true if all child constraints are satisfied.
+   * @param appId application id
+   * @param constraint Or constraint
+   * @param node node
+   * @param atm allocation tags manager
+   * @return true if all child constraints are satisfied, false otherwise
+   * @throws InvalidAllocationTagsQueryException
+   */
+  private

[09/32] hadoop git commit: YARN-7522. Introduce AllocationTagsManager to associate allocation tags to nodes. (Wangda Tan via asuresh)

[asuresh]

YARN-7522. Introduce AllocationTagsManager to associate allocation tags to 
nodes. (Wangda Tan via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/801c0988
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/801c0988
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/801c0988

Branch: refs/heads/trunk
Commit: 801c0988b5ad1eff1e896a2635c2937721c96b04
Parents: 69de9a1
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Dec 8 00:24:00 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../resourcemanager/RMActiveServiceContext.java |  15 +
 .../yarn/server/resourcemanager/RMContext.java  |   5 +
 .../server/resourcemanager/RMContextImpl.java   |  12 +
 .../server/resourcemanager/ResourceManager.java |   9 +
 .../constraint/AllocationTagsManager.java   | 431 +++
 .../constraint/AllocationTagsNamespaces.java|  31 ++
 .../InvalidAllocationTagsQueryException.java|  35 ++
 .../rmcontainer/RMContainer.java|   8 +
 .../rmcontainer/RMContainerImpl.java|  21 +
 .../constraint/TestAllocationTagsManager.java   | 328 ++
 .../rmcontainer/TestRMContainerImpl.java| 124 ++
 .../scheduler/capacity/TestUtils.java   |   9 +
 .../scheduler/fifo/TestFifoScheduler.java   |   5 +
 13 files changed, 1033 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/801c0988/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 9dc5945..6ee3a4c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.event.Dispatcher;
 import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater;
 import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import 
org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
@@ -107,6 +108,7 @@ public class RMActiveServiceContext {
 
   private RMAppLifetimeMonitor rmAppLifetimeMonitor;
   private QueueLimitCalculator queueLimitCalculator;
+  private AllocationTagsManager allocationTagsManager;
 
   public RMActiveServiceContext() {
 queuePlacementManager = new PlacementManager();
@@ -398,6 +400,19 @@ public class RMActiveServiceContext {
 
   @Private
   @Unstable
+  public AllocationTagsManager getAllocationTagsManager() {
+return allocationTagsManager;
+  }
+
+  @Private
+  @Unstable
+  public void setAllocationTagsManager(
+  AllocationTagsManager allocationTagsManager) {
+this.allocationTagsManager = allocationTagsManager;
+  }
+
+  @Private
+  @Unstable
   public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() {
 return rmDelegatedNodeLabelsUpdater;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/801c0988/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index ec94030..62899d9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext

[30/32] hadoop git commit: YARN-7779. Display allocation tags in RM web UI and expose same through REST API. Contributed by Weiwei Yang.

[asuresh]

YARN-7779. Display allocation tags in RM web UI and expose same through REST 
API. Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b81cb05
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b81cb05
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b81cb05

Branch: refs/heads/trunk
Commit: 9b81cb0537e5b731581e6a375bf0a59abf61c359
Parents: adbe87a
Author: Sunil G 
Authored: Tue Jan 23 17:09:58 2018 +0530
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/sls/nodemanager/NodeInfo.java   |  6 ++
 .../yarn/sls/scheduler/RMNodeWrapper.java   |  6 ++
 .../server/resourcemanager/rmnode/RMNode.java   |  7 ++
 .../resourcemanager/rmnode/RMNodeImpl.java  |  6 ++
 .../constraint/AllocationTagsManager.java   | 11 +++
 .../resourcemanager/webapp/NodesPage.java   |  3 +
 .../webapp/dao/AllocationTagInfo.java   | 56 ++
 .../webapp/dao/AllocationTagsInfo.java  | 59 +++
 .../resourcemanager/webapp/dao/NodeInfo.java| 15 
 .../yarn/server/resourcemanager/MockNodes.java  |  6 ++
 .../resourcemanager/webapp/TestNodesPage.java   |  4 +-
 .../webapp/TestRMWebServicesNodes.java  | 77 +++-
 12 files changed, 253 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 1016ce1..0c99139 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.sls.nodemanager;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -213,6 +214,11 @@ public class NodeInfo {
 }
 
 @Override
+public Map getAllocationTagsWithCount() {
+  return null;
+}
+
+@Override
 public Resource getPhysicalResource() {
   return null;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index fdad826..92f9b0f 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 @Private
@@ -203,6 +204,11 @@ public class RMNodeWrapper implements RMNode {
   }
 
   @Override
+  public Map getAllocationTagsWithCount() {
+return node.getAllocationTagsWithCount();
+  }
+
+  @Override
   public Resource getPhysicalResource() {
 return null;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index a5615ef..872f2a6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
 
 
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 

[23/32] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
index 73b4f9e..24c5a5e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -20,6 +20,8 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
 import java.util.Iterator;
 import java.util.Set;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -30,9 +32,12 @@ import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm;
 
+import static 
org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE_PARTITION;
+
 /**
  * This class contains various static methods used by the Placement Algorithms
  * to simplify constrained placement.
@@ -41,16 +46,20 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algori
 @Public
 @Unstable
 public final class PlacementConstraintsUtil {
+  private static final Log LOG =
+  LogFactory.getLog(PlacementConstraintsUtil.class);
 
   // Suppresses default constructor, ensuring non-instantiability.
   private PlacementConstraintsUtil() {
   }
 
   /**
-   * Returns true if **single** application constraint with associated
+   * Returns true if **single** placement constraint with associated
* allocationTags and scope is satisfied by a specific scheduler Node.
*
-   * @param appId the application id
+   * @param targetApplicationId the application id, which could be override by
+   *   target application id specified inside 
allocation
+   *   tags.
* @param sc the placement constraint
* @param te the target expression
* @param node the scheduler node
@@ -59,32 +68,123 @@ public final class PlacementConstraintsUtil {
* @throws InvalidAllocationTagsQueryException
*/
   private static boolean canSatisfySingleConstraintExpression(
-  ApplicationId appId, SingleConstraint sc, TargetExpression te,
-  SchedulerNode node, AllocationTagsManager tm)
+  ApplicationId targetApplicationId, SingleConstraint sc,
+  TargetExpression te, SchedulerNode node, AllocationTagsManager tm)
   throws InvalidAllocationTagsQueryException {
 long minScopeCardinality = 0;
 long maxScopeCardinality = 0;
+
+// Optimizations to only check cardinality if necessary.
+int desiredMinCardinality = sc.getMinCardinality();
+int desiredMaxCardinality = sc.getMaxCardinality();
+boolean checkMinCardinality = desiredMinCardinality > 0;
+boolean checkMaxCardinality = desiredMaxCardinality < Integer.MAX_VALUE;
+
 if (sc.getScope().equals(PlacementConstraints.NODE)) {
-  minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId,
-  te.getTargetValues(), Long::max);
-  maxScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId,
-  te.getTargetValues(), Long::min);
+  if (checkMinCardinality) {
+minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(),
+targetApplicationId, te.getTargetValues(), Long::max);
+  }
+  if (checkMaxCardinality) {
+maxScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(),
+targetApplicationId, te.getTargetValues(), 

[28/32] hadoop git commit: YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and NMContainerStatus to handle all recovery cases. (asuresh)

[asuresh]

YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and 
NMContainerStatus to handle all recovery cases. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5c1fc88
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5c1fc88
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5c1fc88

Branch: refs/heads/trunk
Commit: a5c1fc881e21ebf43da7ead5f3852808fce25492
Parents: 4eda58c
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Jan 12 14:37:06 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/api/records/Container.java  | 15 +
 .../src/main/proto/yarn_protos.proto|  1 +
 .../api/records/impl/pb/ContainerPBImpl.java| 31 +
 .../yarn/security/ContainerTokenIdentifier.java | 69 +++-
 .../src/main/proto/yarn_security_token.proto|  1 +
 .../api/protocolrecords/NMContainerStatus.java  | 14 
 .../impl/pb/NMContainerStatusPBImpl.java| 33 ++
 .../yarn_server_common_service_protos.proto |  1 +
 .../containermanager/ContainerManagerImpl.java  |  3 +-
 .../container/ContainerImpl.java| 19 +++---
 .../rmcontainer/RMContainerImpl.java| 10 ++-
 .../scheduler/SchedulerApplicationAttempt.java  |  3 +-
 .../security/RMContainerTokenSecretManager.java | 21 ++
 .../capacity/TestContainerAllocation.java   |  5 +-
 14 files changed, 194 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
index 4fdc803..b9ca3f9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
@@ -27,6 +27,9 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
 import org.apache.hadoop.yarn.util.Records;
 
+import java.util.Collections;
+import java.util.Set;
+
 /**
  * {@code Container} represents an allocated resource in the cluster.
  * 
@@ -256,4 +259,16 @@ public abstract class Container implements 
Comparable {
   public void setVersion(int version) {
 throw new UnsupportedOperationException();
   }
+
+  @Private
+  @Unstable
+  public Set getAllocationTags() {
+return Collections.EMPTY_SET;
+  }
+
+  @Private
+  @Unstable
+  public void setAllocationTags(Set allocationTags) {
+
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 5cb1177..25c8569 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -152,6 +152,7 @@ message ContainerProto {
   optional ExecutionTypeProto execution_type = 7 [default = GUARANTEED];
   optional int64 allocation_request_id = 8 [default = -1];
   optional int32 version = 9 [default = 0];
+  repeated string allocation_tags = 10;
 }
 
 message ContainerReportProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
index be84938..47be2f0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
@@ -36,6 +36,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.PriorityPr

[18/32] hadoop git commit: YARN-7681. Double-check placement constraints in scheduling phase before actual allocation is made. (Weiwei Yang via asuresh)

[asuresh]

YARN-7681. Double-check placement constraints in scheduling phase before actual 
allocation is made. (Weiwei Yang via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4eda58c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4eda58c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4eda58c1

Branch: refs/heads/trunk
Commit: 4eda58c13641c14c4b248843a2589781cbcd343f
Parents: bdba01f
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Jan 10 09:04:30 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../scheduler/capacity/CapacityScheduler.java   | 23 
 1 file changed, 23 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eda58c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index e682d0f..d2713c8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -124,6 +124,8 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCo
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintsUtil;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
@@ -2574,6 +2576,27 @@ public class CapacityScheduler extends
 ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>
 resourceCommitRequest = createResourceCommitRequest(
 appAttempt, schedulingRequest, schedulerNode);
+
+// Validate placement constraint is satisfied before
+// committing the request.
+try {
+  if (!PlacementConstraintsUtil.canSatisfyConstraints(
+  appAttempt.getApplicationId(),
+  schedulingRequest.getAllocationTags(),
+  schedulerNode,
+  rmContext.getPlacementConstraintManager(),
+  rmContext.getAllocationTagsManager())) {
+LOG.debug("Failed to allocate container for application "
++ appAttempt.getApplicationId() + " on node "
++ schedulerNode.getNodeName()
++ " because this allocation violates the"
++ " placement constraint.");
+return false;
+  }
+} catch (InvalidAllocationTagsQueryException e) {
+  LOG.warn("Unable to allocate container", e);
+  return false;
+}
 return tryCommit(getClusterResource(), resourceCommitRequest, false);
   }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[21/32] hadoop git commit: YARN-7745. Allow DistributedShell to take a placement specification for containers it wants to launch. (Arun Suresh via wangda)

[asuresh]

YARN-7745. Allow DistributedShell to take a placement specification for 
containers it wants to launch. (Arun Suresh via wangda)

Change-Id: Ided146d662e944a8a4692e5d6885f23fd9bbcad5


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e60f5129
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e60f5129
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e60f5129

Branch: refs/heads/trunk
Commit: e60f51299dba360d13aa39f9ab714fdfc666b532
Parents: 38af237
Author: Wangda Tan 
Authored: Thu Jan 18 14:22:45 2018 -0800
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../distributedshell/ApplicationMaster.java | 124 +++--
 .../applications/distributedshell/Client.java   |  14 ++
 .../distributedshell/PlacementSpec.java | 137 +++
 3 files changed, 263 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e60f5129/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 270ef1b..9ba2138 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -42,6 +42,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
@@ -87,8 +88,11 @@ import 
org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.ProfileCapability;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.UpdatedContainer;
 import org.apache.hadoop.yarn.api.records.ExecutionType;
@@ -99,6 +103,7 @@ import 
org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
 import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.client.api.TimelineV2Client;
@@ -274,6 +279,10 @@ public class ApplicationMaster {
   @VisibleForTesting
   protected AtomicInteger numRequestedContainers = new AtomicInteger();
 
+  protected AtomicInteger numIgnore = new AtomicInteger();
+
+  protected AtomicInteger totalRetries = new AtomicInteger(10);
+
   // Shell command to be executed
   private String shellCommand = "";
   // Args to be passed to the shell command
@@ -289,6 +298,9 @@ public class ApplicationMaster {
   // File length needed for local resource
   private long shellScriptPathLen = 0;
 
+  // Placement Specifications
+  private Map placementSpecs = null;
+
   // Container retry options
   private ContainerRetryPolicy containerRetryPolicy =
   ContainerRetryPolicy.NEVER_RETRY;
@@ -334,6 +346,7 @@ public class ApplicationMaster {
   private final String windows_command = "cmd /c";
 
   private int yarnShellIdCounter = 1;
+  private final AtomicLong allocIdCounter = new AtomicLong(1);
 
   @VisibleForTesting
   protected final Set launchedContainers =
@@ -457,6 +470,7 @@ public class ApplicationMaster {
 "If container could retry, it specifies max 

[17/32] hadoop git commit: YARN-7682. Expose canSatisfyConstraints utility function to validate a placement against a constraint. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-7682. Expose canSatisfyConstraints utility function to validate a 
placement against a constraint. (Panagiotis Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bdba01f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bdba01f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bdba01f7

Branch: refs/heads/trunk
Commit: bdba01f73b58d2228e808c6f61377f101b6bac1c
Parents: a52d11f
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Jan 3 08:00:50 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../constraint/PlacementConstraintsUtil.java| 132 +
 .../algorithm/DefaultPlacementAlgorithm.java|  55 +---
 .../TestPlacementConstraintsUtil.java   | 287 +++
 .../constraint/TestPlacementProcessor.java  | 204 +++--
 4 files changed, 601 insertions(+), 77 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdba01f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
new file mode 100644
index 000..956a3c9
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType;
+import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import 
org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm;
+
+/**
+ * This class contains various static methods used by the Placement Algorithms
+ * to simplify constrained placement.
+ * (see also {@link DefaultPlacementAlgorithm}).
+ */
+@Public
+@Unstable
+public final class PlacementConstraintsUtil {
+
+  // Suppresses default constructor, ensuring non-instantiability.
+  private PlacementConstraintsUtil() {
+  }
+
+  /**
+   * Returns true if **single** application constraint with associated
+   * allocationTags and scope is satisfied by a specific scheduler Node.
+   *
+   * @param appId the application id
+   * @param sc the placement constraint
+   * @param te the target expression
+   * @param node the scheduler node
+   * @param tm the allocation tags store
+   * @return true if single application constraint is satisfied by node
+   * @throws InvalidAllocationTagsQueryException
+   */
+  private static boolean canS

[19/32] hadoop git commit: YARN-7709. Remove SELF from TargetExpression type. (Konstantinos Karanasos via asuresh)

[asuresh]

YARN-7709. Remove SELF from TargetExpression type. (Konstantinos Karanasos via 
asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8779a357
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8779a357
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8779a357

Branch: refs/heads/trunk
Commit: 8779a35742085fadddccc21342b55d4f17fae5c2
Parents: 29d9e4d
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 18 04:29:57 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../yarn/api/resource/PlacementConstraint.java  | 32 ++
 .../yarn/api/resource/PlacementConstraints.java | 35 +---
 .../api/resource/TestPlacementConstraints.java  |  3 +-
 .../PlacementConstraintTransformations.java | 19 +++
 .../TestPlacementConstraintTransformations.java | 35 +---
 .../constraint/PlacementConstraintsUtil.java| 10 --
 6 files changed, 64 insertions(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8779a357/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
index b6e851a..4d998ac 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
@@ -242,7 +242,7 @@ public class PlacementConstraint {
  * Enum specifying the type of the target expression.
  */
 public enum TargetType {
-  NODE_ATTRIBUTE, ALLOCATION_TAG, SELF
+  NODE_ATTRIBUTE, ALLOCATION_TAG
 }
 
 private TargetType targetType;
@@ -418,23 +418,25 @@ public class PlacementConstraint {
   }
 
   /**
-   * Class that represents a cardinality constraint. Such a constraint the
-   * number of allocations within a given scope to some minimum and maximum
-   * values.
+   * Class that represents a cardinality constraint. Such a constraint allows
+   * the number of allocations with a specific set of tags and within a given
+   * scope to be between some minimum and maximum values.
*
* It is a specialized version of the {@link SingleConstraint}, where the
-   * target is self (i.e., the allocation to which the constraint is attached).
+   * target is a set of allocation tags.
*/
   public static class CardinalityConstraint extends AbstractConstraint {
 private String scope;
 private int minCardinality;
 private int maxCardinality;
+private Set allocationTags;
 
 public CardinalityConstraint(String scope, int minCardinality,
-int maxCardinality) {
+int maxCardinality, Set allocationTags) {
   this.scope = scope;
   this.minCardinality = minCardinality;
   this.maxCardinality = maxCardinality;
+  this.allocationTags = allocationTags;
 }
 
 /**
@@ -464,11 +466,21 @@ public class PlacementConstraint {
   return maxCardinality;
 }
 
+/**
+ * Get the allocation tags of the constraint.
+ *
+ * @return the allocation tags of the constraint
+ */
+public Set getAllocationTags() {
+  return allocationTags;
+}
+
 @Override
 public  T accept(Visitor visitor) {
   return visitor.visit(this);
 }
 
+
 @Override
 public boolean equals(Object o) {
   if (this == o) {
@@ -486,7 +498,11 @@ public class PlacementConstraint {
   if (maxCardinality != that.maxCardinality) {
 return false;
   }
-  return scope != null ? scope.equals(that.scope) : that.scope == null;
+  if (scope != null ? !scope.equals(that.scope) : that.scope != null) {
+return false;
+  }
+  return allocationTags != null ? 
allocationTags.equals(that.allocationTags)
+  : that.allocationTags == null;
 }
 
 @Override
@@ -494,6 +510,8 @@ public class PlacementConstraint {
   int result = scope != null ? scope.hashCode() : 0;
   result = 31 * result + minCardinality;
   result = 31 * result + maxCardinality;
+  result = 31 * result
+  + (allocationTags != null ? allocationTags.hashCode() : 0);
   return result;
 }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8779a357/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementCons

[16/32] hadoop git commit: YARN-7613. Implement Basic algorithm for constraint based placement. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-7613. Implement Basic algorithm for constraint based placement. 
(Panagiotis Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a52d11fb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a52d11fb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a52d11fb

Branch: refs/heads/trunk
Commit: a52d11fb8c103f14e42692600a058ba3b56e2ecf
Parents: f9af15d
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Dec 27 22:59:22 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   4 +
 .../src/main/resources/yarn-default.xml |   8 +-
 .../rmcontainer/RMContainerImpl.java|  10 +-
 .../constraint/AllocationTagsManager.java   | 121 ++---
 .../algorithm/DefaultPlacementAlgorithm.java| 172 +++
 .../iterators/PopularTagsIterator.java  |  71 
 .../algorithm/iterators/SerialIterator.java |  53 ++
 .../algorithm/iterators/package-info.java   |  29 
 .../constraint/algorithm/package-info.java  |  29 
 .../constraint/processor/BatchedRequests.java   |  45 -
 .../processor/PlacementProcessor.java   |  32 ++--
 .../processor/SamplePlacementAlgorithm.java | 144 
 .../constraint/TestAllocationTagsManager.java   | 156 -
 .../TestBatchedRequestsIterators.java   |  82 +
 .../constraint/TestPlacementProcessor.java  |   4 +-
 15 files changed, 721 insertions(+), 239 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8fb3c2e..367b1ae 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -536,6 +536,10 @@ public class YarnConfiguration extends Configuration {
   public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS =
   RM_PREFIX + "placement-constraints.algorithm.class";
 
+  /** Used for BasicPlacementAlgorithm - default SERIAL. **/
+  public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_ITERATOR =
+  RM_PREFIX + "placement-constraints.algorithm.iterator";
+
   public static final String RM_PLACEMENT_CONSTRAINTS_ENABLED =
   RM_PREFIX + "placement-constraints.enabled";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 6d52ace..509a040 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -145,7 +145,13 @@
   
 Constraint Placement Algorithm to be used.
 yarn.resourcemanager.placement-constraints.algorithm.class
-
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor.SamplePlacementAlgorithm
+
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm
+  
+
+  
+Placement Algorithm Requests Iterator to be 
used.
+yarn.resourcemanager.placement-constraints.algorithm.iterator
+SERIAL
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index c873509..2c4ef7b 100644
--- 
a/hadoop-yarn-project/

[15/32] hadoop git commit: YARN-7612. Add Processor Framework for Rich Placement Constraints. (asuresh)

[asuresh]

YARN-7612. Add Processor Framework for Rich Placement Constraints. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9af15d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9af15d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9af15d6

Branch: refs/heads/trunk
Commit: f9af15d659f59fd0cf564fe1ecc8e06c6429ba68
Parents: 1efb2b6
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Dec 22 15:51:20 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  26 ++
 .../src/main/resources/yarn-default.xml |  30 ++
 .../ApplicationMasterService.java   |  15 +
 .../rmcontainer/RMContainerImpl.java|   7 +-
 .../scheduler/capacity/CapacityScheduler.java   |   2 +
 .../constraint/processor/BatchedRequests.java   | 105 +
 .../processor/NodeCandidateSelector.java|  38 ++
 .../processor/PlacementDispatcher.java  | 145 +++
 .../processor/PlacementProcessor.java   | 343 
 .../processor/SamplePlacementAlgorithm.java | 144 +++
 .../constraint/processor/package-info.java  |  29 ++
 .../yarn/server/resourcemanager/MockAM.java |  26 ++
 .../yarn/server/resourcemanager/MockRM.java |  14 +
 .../constraint/TestPlacementProcessor.java  | 394 +++
 14 files changed, 1316 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9af15d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index bbbfc52..8fb3c2e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -531,6 +531,32 @@ public class YarnConfiguration extends Configuration {
   /** The class to use as the resource scheduler.*/
   public static final String RM_SCHEDULER = 
 RM_PREFIX + "scheduler.class";
+
+  /** Placement Algorithm. */
+  public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS =
+  RM_PREFIX + "placement-constraints.algorithm.class";
+
+  public static final String RM_PLACEMENT_CONSTRAINTS_ENABLED =
+  RM_PREFIX + "placement-constraints.enabled";
+
+  public static final boolean DEFAULT_RM_PLACEMENT_CONSTRAINTS_ENABLED = true;
+
+  public static final String RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS =
+  RM_PREFIX + "placement-constraints.retry-attempts";
+
+  public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS = 3;
+
+  public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE =
+  RM_PREFIX + "placement-constraints.algorithm.pool-size";
+
+  public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE 
=
+  1;
+
+  public static final String RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE =
+  RM_PREFIX + "placement-constraints.scheduler.pool-size";
+
+  public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE 
=
+  1;
  
   public static final String DEFAULT_RM_SCHEDULER = 
   
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9af15d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0bb4fca..6d52ace 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -131,6 +131,36 @@
   
 
   
+Enable Constraint Placement.
+yarn.resourcemanager.placement-constraints.enabled
+false
+  
+
+  
+Number of times to retry placing of rejected 
SchedulingRequests
+yarn.resourcemanager.placement-constraints.retry-attempts
+3
+  
+
+  
+Constraint Placement Algorithm to be used.
+yarn.resourcemanager.placement-constraints.algorithm.class
+
org.apache.hadoop.yarn.server.resourcemanager

[20/32] hadoop git commit: YARN-7807. Assume intra-app anti-affinity as default for scheduling request inside AppPlacementAllocator. (Wangda Tan via asuresh)

[asuresh]

YARN-7807. Assume intra-app anti-affinity as default for scheduling request 
inside AppPlacementAllocator. (Wangda Tan via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/644afe5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/644afe5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/644afe5f

Branch: refs/heads/trunk
Commit: 644afe5fd800ac4f2b873a99f9b3868c3a8c5c40
Parents: a4c539f
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Jan 24 12:55:01 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../placement/SingleConstraintAppPlacementAllocator.java| 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/644afe5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
index 9e7d71c..b02cb00 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
@@ -334,14 +334,15 @@ public class SingleConstraintAppPlacementAllocator
 targetAllocationTags = new HashSet<>(
 targetExpression.getTargetValues());
 
-if (targetExpression.getTargetKey() == null || !targetExpression
+if (targetExpression.getTargetKey() != null && !targetExpression
 .getTargetKey().equals(APPLICATION_LABEL_INTRA_APPLICATION)) {
   throwExceptionWithMetaInfo(
   "As of now, the only accepted target key for targetKey of "
   + "allocation_tag target expression is: ["
   + APPLICATION_LABEL_INTRA_APPLICATION
   + "]. Please make changes to placement constraints "
-  + "accordingly.");
+  + "accordingly. If this is null, it will be set to "
+  + APPLICATION_LABEL_INTRA_APPLICATION + " by default.");
 }
   }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[32/32] hadoop git commit: Merge branch 'YARN-6592' into trunk

[asuresh]

Merge branch 'YARN-6592' into trunk


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d1e2c64
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d1e2c64
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d1e2c64

Branch: refs/heads/trunk
Commit: 8d1e2c6409a44f4515a1549ae82c7e2597e96467
Parents: 7288b08 8df7666
Author: Arun Suresh 
Authored: Wed Jan 31 01:42:42 2018 -0800
Committer: Arun Suresh 
Committed: Wed Jan 31 01:42:42 2018 -0800

--
 .../v2/app/rm/TestRMContainerAllocator.java |  15 +-
 .../hadoop/yarn/sls/nodemanager/NodeInfo.java   |   6 +
 .../yarn/sls/scheduler/RMNodeWrapper.java   |   6 +
 .../sls/scheduler/SLSCapacityScheduler.java |  15 +-
 .../yarn/sls/scheduler/SLSFairScheduler.java|  12 +-
 .../dev-support/findbugs-exclude.xml|   8 +
 .../yarn/ams/ApplicationMasterServiceUtils.java |  16 +
 .../api/protocolrecords/AllocateRequest.java|  42 ++
 .../api/protocolrecords/AllocateResponse.java   |  23 +
 .../RegisterApplicationMasterRequest.java   |  42 +-
 .../hadoop/yarn/api/records/Container.java  |  15 +
 .../api/records/RejectedSchedulingRequest.java  |  70 ++
 .../yarn/api/records/RejectionReason.java   |  44 ++
 .../hadoop/yarn/api/records/ResourceSizing.java |  91 +++
 .../yarn/api/records/SchedulingRequest.java | 206 +
 .../yarn/api/resource/PlacementConstraint.java  | 744 +++
 .../yarn/api/resource/PlacementConstraints.java | 320 
 .../hadoop/yarn/api/resource/package-info.java  |  23 +
 .../hadoop/yarn/conf/YarnConfiguration.java |  31 +
 ...SchedulerInvalidResoureRequestException.java |  47 ++
 .../src/main/proto/yarn_protos.proto|  86 +++
 .../src/main/proto/yarn_service_protos.proto|   3 +
 .../api/resource/TestPlacementConstraints.java  | 107 +++
 .../distributedshell/ApplicationMaster.java | 124 +++-
 .../applications/distributedshell/Client.java   |  14 +
 .../distributedshell/PlacementSpec.java | 137 
 .../hadoop/yarn/client/api/AMRMClient.java  |  38 +-
 .../yarn/client/api/async/AMRMClientAsync.java  |  48 ++
 .../api/async/impl/AMRMClientAsyncImpl.java |  49 +-
 .../yarn/client/api/impl/AMRMClientImpl.java| 142 +++-
 .../client/api/impl/BaseAMRMClientTest.java | 212 ++
 .../yarn/client/api/impl/TestAMRMClient.java| 156 +---
 .../api/impl/TestAMRMClientOnRMRestart.java |   9 +-
 .../TestAMRMClientPlacementConstraints.java | 204 +
 .../PlacementConstraintFromProtoConverter.java  | 116 +++
 .../pb/PlacementConstraintToProtoConverter.java | 174 +
 .../apache/hadoop/yarn/api/pb/package-info.java |  23 +
 .../impl/pb/AllocateRequestPBImpl.java  |  84 +++
 .../impl/pb/AllocateResponsePBImpl.java |  85 +++
 .../RegisterApplicationMasterRequestPBImpl.java | 106 ++-
 .../api/records/impl/pb/ContainerPBImpl.java|  31 +
 .../yarn/api/records/impl/pb/ProtoUtils.java|  43 ++
 .../pb/RejectedSchedulingRequestPBImpl.java | 148 
 .../records/impl/pb/ResourceSizingPBImpl.java   | 128 
 .../impl/pb/SchedulingRequestPBImpl.java| 296 
 .../PlacementConstraintTransformations.java | 200 +
 .../hadoop/yarn/api/resource/package-info.java  |  23 +
 .../yarn/security/ContainerTokenIdentifier.java |  69 +-
 .../src/main/proto/yarn_security_token.proto|   1 +
 .../src/main/resources/yarn-default.xml |  36 +
 .../hadoop/yarn/api/BasePBImplRecordsTest.java  |  11 +
 .../hadoop/yarn/api/TestPBImplRecords.java  |  21 +
 .../TestPlacementConstraintPBConversion.java| 195 +
 .../TestPlacementConstraintTransformations.java | 166 +
 .../api/protocolrecords/NMContainerStatus.java  |  14 +
 .../impl/pb/NMContainerStatusPBImpl.java|  33 +
 .../server/scheduler/SchedulerRequestKey.java   |  11 +
 .../yarn_server_common_service_protos.proto |   1 +
 .../containermanager/ContainerManagerImpl.java  |   3 +-
 .../container/ContainerImpl.java|  19 +-
 .../ApplicationMasterService.java   |  15 +
 .../resourcemanager/DefaultAMSProcessor.java|  13 +-
 .../resourcemanager/RMActiveServiceContext.java |  30 +
 .../yarn/server/resourcemanager/RMContext.java  |  11 +
 .../server/resourcemanager/RMContextImpl.java   |  25 +
 .../server/resourcemanager/ResourceManager.java |  22 +
 .../rmapp/attempt/RMAppAttemptImpl.java |   5 +-
 .../rmcontainer/RMContainer.java|   8 +
 .../rmcontainer/RMContainerImpl.java|  37 +-
 .../server/resourcemanager/rmnode/RMNode.java   |   7 +
 .../resourcemanager/rmnode/RMNodeImpl.java  |   6 +
 .../scheduler/AbstractYarnScheduler.java|  21 +-
 .../scheduler/AppSchedulingInfo.java| 214 --
 .../ApplicationPlacementAllocatorFactory.java   |  68 ++
 

[22/32] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index a3b88c0..01d5e6c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -170,7 +170,7 @@ public class TestCapacitySchedulerAutoQueueCreation
   1 * GB, 1, true, priority, recordFactory);
 
   cs.allocate(appAttemptId, Collections.singletonList(r1),
-  Collections.emptyList(), 
Collections.singletonList(host),
+  null, Collections.emptyList(), 
Collections.singletonList(host),
   null, NULL_UPDATE_REQUESTS);
 
   //And this will result in container assignment for app1

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java
new file mode 100644
index 000..b6ac4b6
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+public class 

[10/32] hadoop git commit: YARN-7670. Modifications to the ResourceScheduler API to support SchedulingRequests. (asuresh)

[asuresh]

YARN-7670. Modifications to the ResourceScheduler API to support 
SchedulingRequests. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88d8d3f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88d8d3f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88d8d3f4

Branch: refs/heads/trunk
Commit: 88d8d3f40b2923fab23a933bce1cd2e9c320ae84
Parents: 801c098
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Dec 19 08:59:23 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../scheduler/AbstractYarnScheduler.java| 18 +
 .../scheduler/ResourceScheduler.java| 13 
 .../scheduler/capacity/CapacityScheduler.java   | 78 ++--
 .../common/ResourceAllocationCommitter.java | 12 ++-
 .../scheduler/common/fica/FiCaSchedulerApp.java | 30 +---
 .../TestCapacitySchedulerAsyncScheduling.java   | 10 +--
 6 files changed, 138 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/88d8d3f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 4b76327..213d784 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceOption;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerError;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -295,6 +296,10 @@ public abstract class AbstractYarnScheduler
 return nodeTracker.getNodes(nodeFilter);
   }
 
+  public List getNodes(final NodeFilter filter) {
+return nodeTracker.getNodes(filter);
+  }
+
   public boolean shouldContainersBeAutoUpdated() {
 return this.autoUpdateContainers;
   }
@@ -1443,4 +1448,17 @@ public abstract class AbstractYarnScheduler
   throw new IOException(e);
 }
   }
+
+  /**
+   * Default implementation. Always returns false.
+   * @param appAttempt ApplicationAttempt.
+   * @param schedulingRequest SchedulingRequest.
+   * @param schedulerNode SchedulerNode.
+   * @return Success or not.
+   */
+  @Override
+  public boolean attemptAllocationOnNode(SchedulerApplicationAttempt 
appAttempt,
+  SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) {
+return false;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/88d8d3f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
index d96d625..5a56ac7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
@@ -25,6 +25,7 @@ import 
org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.re

[13/32] hadoop git commit: YARN-7653. Node group support for AllocationTagsManager. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-7653. Node group support for AllocationTagsManager. (Panagiotis 
Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37f1a7b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37f1a7b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37f1a7b6

Branch: refs/heads/trunk
Commit: 37f1a7b64fcc93191367330cd59d4d71d7b29ac7
Parents: 06eb63e
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Dec 22 07:24:37 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../server/resourcemanager/ResourceManager.java |   2 +-
 .../constraint/AllocationTagsManager.java   | 282 ++-
 .../rmcontainer/TestRMContainerImpl.java|   2 +-
 .../constraint/TestAllocationTagsManager.java   | 269 --
 4 files changed, 392 insertions(+), 163 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/37f1a7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index a1d3dfc..1d838f0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -496,7 +496,7 @@ public class ResourceManager extends CompositeService 
implements Recoverable {
   }
 
   protected AllocationTagsManager createAllocationTagsManager() {
-return new AllocationTagsManager();
+return new AllocationTagsManager(this.rmContext);
   }
   
   protected DelegationTokenRenewer createDelegationTokenRenewer() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/37f1a7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
index c278606..7b0b959 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.log4j.Logger;
 
 import java.util.HashMap;
@@ -38,9 +39,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.LongBinaryOperator;
 
 /**
- * Support storing maps between container-tags/applications and
- * nodes. This will be required by affinity/anti-affinity implementation and
- * cardinality.
+ * In-memory mapping between applications/container-tags and nodes/racks.
+ * Required by constrained affinity/anti-affinity and cardinality placement.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
@@ -51,48 +51,54 @@ public class AllocationTagsManager {
 
   private ReentrantReadWriteLock.ReadLock readLock;
   private ReentrantReadWriteLock.WriteLock writeLock;
+  private final RMContext rmContext;
 
-  // Application's tags to node
-  private Map<ApplicationId, NodeToCountedTags> perAppMappings =
+  // Application's tags to Node
+  private Map<ApplicationId, NodeToCountedTags> perAppNodeMappings =
+  new HashMap<>();
+  // Application's tags to Rac

[25/32] hadoop git commit: YARN-7774. Miscellaneous fixes to the PlacementProcessor. (asuresh)

[asuresh]

YARN-7774. Miscellaneous fixes to the PlacementProcessor. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28fe7f33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28fe7f33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28fe7f33

Branch: refs/heads/trunk
Commit: 28fe7f331837b36e78fa34ed990993677dddeaee
Parents: e60f512
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 18 11:01:36 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../scheduler/SchedulerNode.java| 16 +++-
 .../scheduler/capacity/CapacityScheduler.java   |  4 +
 .../constraint/PlacementConstraintsUtil.java|  5 +-
 .../constraint/algorithm/CircularIterator.java  | 86 
 .../algorithm/DefaultPlacementAlgorithm.java| 50 ++--
 .../constraint/processor/BatchedRequests.java   |  8 ++
 .../SingleConstraintAppPlacementAllocator.java  |  2 +-
 .../yarn/server/resourcemanager/MockAM.java |  4 +-
 .../constraint/TestPlacementProcessor.java  | 24 +++---
 .../algorithm/TestCircularIterator.java | 84 +++
 ...stSingleConstraintAppPlacementAllocator.java | 28 +++
 11 files changed, 271 insertions(+), 40 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/28fe7f33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 89f748d..96a8e34 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -469,6 +469,20 @@ public abstract class SchedulerNode {
 this.lastHeartbeatMonotonicTime = Time.monotonicNow();
   }
 
+  @Override
+  public boolean equals(Object o) {
+if (this == o) return true;
+if (!(o instanceof SchedulerNode)) return false;
+
+SchedulerNode that = (SchedulerNode) o;
+
+return getNodeID().equals(that.getNodeID());
+  }
+
+  @Override
+  public int hashCode() {
+return getNodeID().hashCode();
+  }
 
   private static class ContainerInfo {
 private final RMContainer container;
@@ -479,4 +493,4 @@ public abstract class SchedulerNode {
   this.launchedOnNode = launchedOnNode;
 }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/28fe7f33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index c713036..429f9f3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2610,6 +2610,10 @@ public class CapacityScheduler extends
 " but only 1 will be attempted !!");
   }
   if (!appAttempt.isStopped()) {
+Resource resource =
+schedulingRequest.getResourceSizing().getResources();
+schedulingRequest.getResourceSizing().setResources(
+getNormalizedResource(resource));
 ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>
 resourceCommitRequest = createResourceCommitRequest(
 appAttempt, schedulingRequest, schedulerNode);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/

[04/32] hadoop git commit: YARN-6619. AMRMClient Changes to use the PlacementConstraint and SchcedulingRequest objects. (Arun Suresh via wangda)

[asuresh]

YARN-6619. AMRMClient Changes to use the PlacementConstraint and 
SchcedulingRequest objects. (Arun Suresh via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29d9e4d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29d9e4d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29d9e4d5

Branch: refs/heads/trunk
Commit: 29d9e4d5814900d5c59d77fe05d32186d4ad9385
Parents: a5c1fc8
Author: Wangda Tan 
Authored: Wed Jan 17 11:36:26 2018 -0800
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/client/api/AMRMClient.java  |  38 +++-
 .../yarn/client/api/async/AMRMClientAsync.java  |  48 +
 .../api/async/impl/AMRMClientAsyncImpl.java |  49 -
 .../yarn/client/api/impl/AMRMClientImpl.java| 142 -
 .../client/api/impl/BaseAMRMClientTest.java | 212 +++
 .../yarn/client/api/impl/TestAMRMClient.java| 156 +-
 .../TestAMRMClientPlacementConstraints.java | 204 ++
 .../rmcontainer/RMContainerImpl.java|   3 +
 .../scheduler/AbstractYarnScheduler.java|   1 +
 .../scheduler/SchedulerApplicationAttempt.java  |   1 +
 .../constraint/PlacementConstraintsUtil.java|   4 +-
 11 files changed, 700 insertions(+), 158 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d9e4d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index d3d1974..914a146 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.client.api;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Supplier;
 import java.util.List;
 
@@ -39,7 +41,9 @@ import 
org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.ProfileCapability;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
 import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.resource.Resources;
@@ -554,6 +558,18 @@ public abstract class AMRMClient extends
   }
 
   /**
+   * Add a Collection of SchedulingRequests. The AMRMClient will ensure that
+   * all requests in the same batch are sent in the same allocate call.
+   * @param schedulingRequests Collection of Scheduling Requests.
+   */
+  @Public
+  @InterfaceStability.Unstable
+  public void addSchedulingRequests(
+  Collection schedulingRequests) {
+
+  }
+
+  /**
* Register the application master. This must be called before any 
* other interaction
* @param appHostName Name of the host on which master is running
@@ -568,7 +584,27 @@ public abstract class AMRMClient extends
  int appHostPort,
  String appTrackingUrl) 
throws YarnException, IOException;
-  
+
+  /**
+   * Register the application master. This must be called before any
+   * other interaction
+   * @param appHostName Name of the host on which master is running
+   * @param appHostPort Port master is listening on
+   * @param appTrackingUrl URL at which the master info can be seen
+   * @param placementConstraints Placement Constraints mappings.
+   * @return RegisterApplicationMasterResponse
+   * @throws YarnException
+   * @throws IOException
+   */
+  @Public
+  @InterfaceStability.Unstable
+  public RegisterApplicationMasterResponse registerApplicationMaster(
+  String appHostName, int appHostPort, String appTrackingUrl,
+  Map placementConstraints)
+  throws YarnException, IOException {
+throw new YarnException("Not supported");
+  }
+
   /**
* Request additional containers and receive new container allocations.
* Requests made via addContainerRequest are sent to the

[12/32] hadoop git commit: YARN-7669. API and interface modifications for placement constraint processor. (asuresh)

[asuresh]

YARN-7669. API and interface modifications for placement constraint processor. 
(asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06eb63e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06eb63e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06eb63e6

Branch: refs/heads/trunk
Commit: 06eb63e64b05e2e8bb8a76c15360ab0495f11317
Parents: 88d8d3f
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Dec 19 22:47:46 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../yarn/ams/ApplicationMasterServiceUtils.java |  16 +
 .../api/protocolrecords/AllocateResponse.java   |  23 +
 .../api/records/RejectedSchedulingRequest.java  |  70 +++
 .../yarn/api/records/RejectionReason.java   |  44 ++
 .../src/main/proto/yarn_protos.proto|  10 +
 .../src/main/proto/yarn_service_protos.proto|   1 +
 .../impl/pb/AllocateResponsePBImpl.java |  85 
 .../yarn/api/records/impl/pb/ProtoUtils.java|  16 +
 .../pb/RejectedSchedulingRequestPBImpl.java | 148 +++
 .../records/impl/pb/ResourceSizingPBImpl.java   |   8 +
 .../impl/pb/SchedulingRequestPBImpl.java|  11 +
 .../hadoop/yarn/api/TestPBImplRecords.java  |   2 +
 .../resourcemanager/RMActiveServiceContext.java |   2 +-
 .../yarn/server/resourcemanager/RMContext.java  |   2 +-
 .../server/resourcemanager/RMContextImpl.java   |   2 +-
 .../server/resourcemanager/ResourceManager.java |   2 +-
 .../constraint/AllocationTagsManager.java   | 431 ---
 .../constraint/AllocationTagsNamespaces.java|  31 --
 .../InvalidAllocationTagsQueryException.java|  35 --
 .../constraint/AllocationTagsManager.java   | 431 +++
 .../constraint/AllocationTagsNamespaces.java|  31 ++
 .../InvalidAllocationTagsQueryException.java|  35 ++
 .../api/ConstraintPlacementAlgorithm.java   |  43 ++
 .../api/ConstraintPlacementAlgorithmInput.java  |  32 ++
 .../api/ConstraintPlacementAlgorithmOutput.java |  58 +++
 ...traintPlacementAlgorithmOutputCollector.java |  32 ++
 .../constraint/api/PlacedSchedulingRequest.java |  79 
 .../constraint/api/SchedulingResponse.java  |  70 +++
 .../scheduler/constraint/api/package-info.java  |  28 ++
 .../constraint/TestAllocationTagsManager.java   | 328 --
 .../rmcontainer/TestRMContainerImpl.java|   2 +-
 .../scheduler/capacity/TestUtils.java   |   2 +-
 .../constraint/TestAllocationTagsManager.java   | 328 ++
 .../scheduler/fifo/TestFifoScheduler.java   |   2 +-
 34 files changed, 1608 insertions(+), 832 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
index 476da8b..8bdfaf3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.ams;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerError;
 import org.apache.hadoop.yarn.api.records.UpdatedContainer;
 
@@ -86,4 +87,19 @@ public final class ApplicationMasterServiceUtils {
 }
 allocateResponse.setAllocatedContainers(allocatedContainers);
   }
+
+  /**
+   * Add rejected Scheduling Requests to {@link AllocateResponse}.
+   * @param allocateResponse Allocate Response.
+   * @param rejectedRequests Rejected SchedulingRequests.
+   */
+  public static void addToRejectedSchedulingRequests(
+  AllocateResponse allocateResponse,
+  List rejectedRequests) {
+if (allocateResponse.getRejectedSchedulingRequests() != null
+&& !allocateResponse.getRejectedSchedulingRequests().isEmpty()) {
+  
rejectedRequests.addAll(allocateResponse.getRejectedSchedulingRequests());
+}
+allocateResponse.setRejectedSchedulingRequests(rejectedRequests);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hado

[14/32] hadoop git commit: YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos Karanasos via asuresh)

[asuresh]

YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos 
Karanasos via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1efb2b6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1efb2b6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1efb2b6f

Branch: refs/heads/trunk
Commit: 1efb2b6f250022f41fe5911c1bb3028ec15c5447
Parents: 37f1a7b
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Dec 22 13:26:30 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../resourcemanager/RMActiveServiceContext.java |  15 +
 .../yarn/server/resourcemanager/RMContext.java  |   6 +
 .../server/resourcemanager/RMContextImpl.java   |  13 +
 .../server/resourcemanager/ResourceManager.java |  13 +
 .../MemoryPlacementConstraintManager.java   | 282 +++
 .../constraint/PlacementConstraintManager.java  | 151 ++
 .../PlacementConstraintManagerService.java  |  93 ++
 .../scheduler/constraint/package-info.java  |  29 ++
 .../TestPlacementConstraintManagerService.java  | 182 
 9 files changed, 784 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1efb2b6f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 4d0c230..06a1d00 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -43,6 +43,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator;
 import 
org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
 import 
org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
@@ -109,6 +110,7 @@ public class RMActiveServiceContext {
   private RMAppLifetimeMonitor rmAppLifetimeMonitor;
   private QueueLimitCalculator queueLimitCalculator;
   private AllocationTagsManager allocationTagsManager;
+  private PlacementConstraintManager placementConstraintManager;
 
   public RMActiveServiceContext() {
 queuePlacementManager = new PlacementManager();
@@ -413,6 +415,19 @@ public class RMActiveServiceContext {
 
   @Private
   @Unstable
+  public PlacementConstraintManager getPlacementConstraintManager() {
+return placementConstraintManager;
+  }
+
+  @Private
+  @Unstable
+  public void setPlacementConstraintManager(
+  PlacementConstraintManager placementConstraintManager) {
+this.placementConstraintManager = placementConstraintManager;
+  }
+
+  @Private
+  @Unstable
   public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() {
 return rmDelegatedNodeLabelsUpdater;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1efb2b6f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 00da108..eb91a31 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-s

[24/32] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)

[asuresh]

YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda 
Tan via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38af2379
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38af2379
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38af2379

Branch: refs/heads/trunk
Commit: 38af23796971193fa529c3d08ffde8fcd6e607b6
Parents: 8779a35
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 18 14:10:30 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../v2/app/rm/TestRMContainerAllocator.java |  15 +-
 .../sls/scheduler/SLSCapacityScheduler.java |  15 +-
 .../yarn/sls/scheduler/SLSFairScheduler.java|  12 +-
 .../dev-support/findbugs-exclude.xml|   8 +
 .../yarn/api/resource/PlacementConstraints.java |  43 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |   2 +-
 ...SchedulerInvalidResoureRequestException.java |  47 ++
 .../api/impl/TestAMRMClientOnRMRestart.java |   9 +-
 .../impl/pb/AllocateRequestPBImpl.java  |   1 +
 .../server/scheduler/SchedulerRequestKey.java   |  11 +
 .../resourcemanager/DefaultAMSProcessor.java|  13 +-
 .../rmapp/attempt/RMAppAttemptImpl.java |   5 +-
 .../scheduler/AbstractYarnScheduler.java|   3 +-
 .../scheduler/AppSchedulingInfo.java| 205 +--
 .../ApplicationPlacementAllocatorFactory.java   |  68 +++
 .../scheduler/ApplicationPlacementFactory.java  |  63 ---
 .../scheduler/ContainerUpdateContext.java   |   4 +-
 .../scheduler/SchedulerApplicationAttempt.java  |  20 +-
 .../scheduler/YarnScheduler.java|  15 +-
 .../scheduler/capacity/CapacityScheduler.java   |  54 +-
 .../CapacitySchedulerConfiguration.java |   5 +
 .../allocator/RegularContainerAllocator.java|   3 +-
 .../scheduler/common/ContainerRequest.java  |  12 +
 .../scheduler/common/PendingAsk.java|   6 +
 .../scheduler/common/fica/FiCaSchedulerApp.java |   6 +
 .../constraint/AllocationTagsManager.java   |  71 +--
 .../constraint/AllocationTagsNamespaces.java|  31 --
 .../constraint/PlacementConstraintsUtil.java| 165 --
 .../algorithm/DefaultPlacementAlgorithm.java|   2 +-
 .../processor/PlacementProcessor.java   |   8 +-
 .../scheduler/fair/FairScheduler.java   |  12 +-
 .../scheduler/fifo/FifoScheduler.java   |   7 +-
 .../placement/AppPlacementAllocator.java|  66 ++-
 .../LocalityAppPlacementAllocator.java  |  35 +-
 .../SingleConstraintAppPlacementAllocator.java  | 531 +++
 .../server/resourcemanager/Application.java |   9 +-
 .../yarn/server/resourcemanager/MockAM.java |  51 ++
 .../attempt/TestRMAppAttemptTransitions.java|  10 +-
 .../rmcontainer/TestRMContainerImpl.java|   6 +-
 .../scheduler/TestAppSchedulingInfo.java|   4 +-
 .../capacity/CapacitySchedulerTestBase.java |  79 +++
 .../capacity/TestCapacityScheduler.java |  90 +---
 .../TestCapacitySchedulerAsyncScheduling.java   |   2 +-
 .../TestCapacitySchedulerAutoQueueCreation.java |   2 +-
 ...apacitySchedulerSchedulingRequestUpdate.java | 260 +
 .../capacity/TestIncreaseAllocationExpirer.java |   2 +-
 ...estSchedulingRequestContainerAllocation.java | 277 ++
 ...hedulingRequestContainerAllocationAsync.java | 139 +
 .../scheduler/capacity/TestUtils.java   |   2 +
 .../constraint/TestAllocationTagsManager.java   |  30 +-
 .../TestPlacementConstraintsUtil.java   |  36 +-
 .../scheduler/fair/FairSchedulerTestBase.java   |   6 +-
 .../fair/TestContinuousScheduling.java  |  10 +-
 .../scheduler/fair/TestFairScheduler.java   |  30 +-
 .../scheduler/fifo/TestFifoScheduler.java   |  28 +-
 ...stSingleConstraintAppPlacementAllocator.java | 403 ++
 56 files changed, 2557 insertions(+), 492 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 85e4181..7875917 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ 
b/hadoop-mapreduce-project/hado

[29/32] hadoop git commit: YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh)

[asuresh]

YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8c5f5b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8c5f5b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8c5f5b2

Branch: refs/heads/trunk
Commit: f8c5f5b23732a1e35f012c1a6850bed09c8a5180
Parents: c23980c
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 25 19:09:21 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../scheduler/AppSchedulingInfo.java| 10 -
 .../scheduler/common/fica/FiCaSchedulerApp.java |  7 +++
 .../constraint/TestPlacementProcessor.java  | 45 
 3 files changed, 60 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c5f5b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 0389895..1efdd8b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -694,6 +694,12 @@ public class AppSchedulingInfo {
   metrics.runAppAttempt(applicationId, user);
 }
 
+updateMetrics(applicationId, type, node, containerAllocated, user, queue);
+  }
+
+  public static void updateMetrics(ApplicationId applicationId, NodeType type,
+  SchedulerNode node, Container containerAllocated, String user,
+  Queue queue) {
 if (LOG.isDebugEnabled()) {
   LOG.debug("allocate: applicationId=" + applicationId + " container="
   + containerAllocated.getId() + " host=" + containerAllocated
@@ -702,10 +708,10 @@ public class AppSchedulingInfo {
   + type);
 }
 if(node != null) {
-  metrics.allocateResources(node.getPartition(), user, 1,
+  queue.getMetrics().allocateResources(node.getPartition(), user, 1,
   containerAllocated.getResource(), true);
 }
-metrics.incrNodeTypeAggregations(user, type);
+queue.getMetrics().incrNodeTypeAggregations(user, type);
   }
 
   // Get AppPlacementAllocator by specified schedulerKey

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c5f5b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 7eb1e31..f3da0a3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -56,6 +56,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRese
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
@@ -548,6 +549,1

[02/32] hadoop git commit: YARN-6597. Add RMContainer recovery test to verify tag population in the AllocationTagsManager. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-6597. Add RMContainer recovery test to verify tag population in the 
AllocationTagsManager. (Panagiotis Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/add993e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/add993e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/add993e2

Branch: refs/heads/trunk
Commit: add993e26a3c96f77dfd42086f186a139966019e
Parents: f8c5f5b
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 25 23:01:43 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../rmcontainer/RMContainerImpl.java|  8 +++
 .../rmcontainer/TestRMContainerImpl.java| 25 ++--
 2 files changed, 26 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/add993e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index a504221..541621b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -20,7 +20,6 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
 
 import java.util.Collections;
 import java.util.EnumSet;
-import java.util.List;
 import java.util.Set;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -40,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -533,7 +531,7 @@ public class RMContainerImpl implements RMContainer {
 RMContainerEvent event) {
   NMContainerStatus report =
   ((RMContainerRecoverEvent) event).getContainerReport();
-  // Set the allocation tags from the
+  // Set the allocation tags from the NMContainerStatus
   container.setAllocationTags(report.getAllocationTags());
   // Notify AllocationTagsManager
   container.rmContext.getAllocationTagsManager().addContainer(
@@ -689,7 +687,7 @@ public class RMContainerImpl implements RMContainer {
 // Something wrong happened, kill the container
 LOG.warn("Something wrong happened, container size reported by NM"
 + " is not expected, ContainerID=" + container.getContainerId()
-+ " rm-size-resource:" + rmContainerResource + " nm-size-reosurce:"
++ " rm-size-resource:" + rmContainerResource + " nm-size-resource:"
 + nmContainerResource);
 container.eventHandler.handle(new RMNodeCleanContainerEvent(
 container.nodeId, container.getContainerId()));
@@ -702,7 +700,7 @@ public class RMContainerImpl implements RMContainer {
 
 @Override
 public void transition(RMContainerImpl container, RMContainerEvent event) {
-  // Notify placementManager
+  // Notify AllocationTagsManager
   container.rmContext.getAllocationTagsManager().removeContainer(
   container.getNodeId(), container.getContainerId(),
   container.getAllocationTags());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/add993e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
 
b/hadoop-yarn-p

[27/32] hadoop git commit: YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun Suresh via kkaranasos)

[asuresh]

YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun 
Suresh via kkaranasos)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adbe87ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adbe87ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adbe87ab

Branch: refs/heads/trunk
Commit: adbe87abf8b2814e0e2988d09ef8a8569190c80e
Parents: 8bf7c44
Author: Konstantinos Karanasos 
Authored: Mon Jan 22 23:51:02 2018 -0800
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../constraint/AllocationTagsManager.java   | 110 +++-
 .../algorithm/DefaultPlacementAlgorithm.java|   8 +-
 .../algorithm/LocalAllocationTagsManager.java   | 167 +++
 .../constraint/TestAllocationTagsManager.java   |  82 -
 .../TestLocalAllocationTagsManager.java | 139 +++
 5 files changed, 336 insertions(+), 170 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/adbe87ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
index 962e548..7ad5e8c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
@@ -24,17 +24,14 @@ import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.log4j.Logger;
 
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -61,9 +58,6 @@ public class AllocationTagsManager {
   // Application's tags to Rack
   private Map perAppRackMappings =
   new HashMap<>();
-  // Application's Temporary containers mapping
-  private Map>>
-  appTempMappings = new HashMap<>();
 
   // Global tags to node mapping (used to fast return aggregated tags
   // cardinality across apps)
@@ -76,7 +70,7 @@ public class AllocationTagsManager {
* Currently used both for NodeId to Tag, Count and Rack to Tag, Count
*/
   @VisibleForTesting
-  static class TypeToCountedTags {
+  public static class TypeToCountedTags {
 // Map>
 private Map> typeToTagsWithCount = new HashMap<>();
 
@@ -214,7 +208,7 @@ public class AllocationTagsManager {
   }
 
   @VisibleForTesting
-  Map getPerAppNodeMappings() {
+  public Map getPerAppNodeMappings() {
 return perAppNodeMappings;
   }
 
@@ -233,12 +227,6 @@ public class AllocationTagsManager {
 return globalRackMapping;
   }
 
-  @VisibleForTesting
-  public Map> getAppTempMappings(
-  ApplicationId applicationId) {
-return appTempMappings.get(applicationId);
-  }
-
   public AllocationTagsManager(RMContext context) {
 ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
 readLock = lock.readLock();
@@ -246,39 +234,6 @@ public class AllocationTagsManager {
 rmContext = context;
   }
 
-  //
-
-  /**
-   * Method adds a temporary fake-container tag to Node mapping.
-   * Used by the constrained placement algorithm to keep track of containers
-   * that are currently placed on nodes but are not yet 

[26/32] hadoop git commit: YARN-7763. Allow Constraints specified in the SchedulingRequest to override application level constraints. (Weiwei Yang via asuresh)

[asuresh]

YARN-7763. Allow Constraints specified in the SchedulingRequest to override 
application level constraints. (Weiwei Yang via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bf7c444
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bf7c444
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bf7c444

Branch: refs/heads/trunk
Commit: 8bf7c444368f48f63f8011cf155f551c6b51ee21
Parents: 28fe7f3
Author: Arun Suresh <asur...@apache.org>
Authored: Sun Jan 21 19:11:17 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../scheduler/capacity/CapacityScheduler.java   |  4 +-
 .../constraint/PlacementConstraintsUtil.java| 98 +++-
 .../algorithm/DefaultPlacementAlgorithm.java|  4 +-
 .../SingleConstraintAppPlacementAllocator.java  | 10 +-
 .../TestPlacementConstraintsUtil.java   | 94 ---
 5 files changed, 123 insertions(+), 87 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf7c444/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 429f9f3..a096e2f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2621,9 +2621,9 @@ public class CapacityScheduler extends
 // Validate placement constraint is satisfied before
 // committing the request.
 try {
-  if (!PlacementConstraintsUtil.canSatisfySingleConstraint(
+  if (!PlacementConstraintsUtil.canSatisfyConstraints(
   appAttempt.getApplicationId(),
-  schedulingRequest.getAllocationTags(), schedulerNode,
+  schedulingRequest, schedulerNode,
   rmContext.getPlacementConstraintManager(),
   rmContext.getAllocationTagsManager())) {
 LOG.debug("Failed to allocate container for application "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf7c444/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
index ff5cb67..c07c16f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
@@ -54,7 +55,7 @@ public final class PlacementConstraintsUtil {
   }
 
   /**
-   * Returns true if **single** placement constraint with associated
+   * Returns true if single placement constraint with associated
* allocationTags and scope is satisfied by a s

[31/32] hadoop git commit: YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh)

[asuresh]

YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c23980c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c23980c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c23980c4

Branch: refs/heads/trunk
Commit: c23980c4f2cf4c751a99fd310e60149cb32ea7c7
Parents: 644afe5
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Jan 24 14:18:32 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/api/protocolrecords/AllocateRequest.java | 4 ++--
 .../apache/hadoop/yarn/api/records/SchedulingRequest.java| 8 
 .../apache/hadoop/yarn/api/resource/PlacementConstraint.java | 3 +++
 .../hadoop/yarn/api/resource/PlacementConstraints.java   | 3 ---
 .../yarn/api/records/impl/pb/ResourceSizingPBImpl.java   | 3 +++
 .../yarn/api/records/impl/pb/SchedulingRequestPBImpl.java| 3 +++
 .../resourcemanager/scheduler/AbstractYarnScheduler.java | 1 -
 .../server/resourcemanager/scheduler/AppSchedulingInfo.java  | 3 +--
 .../yarn/server/resourcemanager/scheduler/SchedulerNode.java | 8 ++--
 .../scheduler/capacity/CapacityScheduler.java| 3 +--
 .../scheduler/constraint/AllocationTagsManager.java  | 8 +---
 .../scheduler/constraint/PlacementConstraintsUtil.java   | 4 ++--
 .../scheduler/placement/AppPlacementAllocator.java   | 4 ++--
 .../placement/SingleConstraintAppPlacementAllocator.java | 1 -
 .../scheduler/capacity/TestCapacityScheduler.java| 1 -
 .../TestCapacitySchedulerSchedulingRequestUpdate.java| 4 +++-
 .../capacity/TestSchedulingRequestContainerAllocation.java   | 8 
 .../TestSchedulingRequestContainerAllocationAsync.java   | 1 -
 18 files changed, 35 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c23980c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
index d8d2347..876957e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
@@ -229,8 +229,8 @@ public abstract class AllocateRequest {
   /**
* Set the list of Scheduling requests to inform the
* ResourceManager about the application's resource requirements
-   * (potentially including allocation tags & placement constraints).
-   * @param schedulingRequests list of SchedulingRequest to update
+   * (potentially including allocation tags and placement constraints).
+   * @param schedulingRequests list of {@link SchedulingRequest} to update
*  the ResourceManager about the application's resource
*  requirements.
*/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c23980c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
index e32dd24..4bb2b84 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
@@ -77,7 +77,7 @@ public abstract class SchedulingRequest {
 
 /**
  * Set the allocationRequestId of the request.
- * 
+ *
  * @see SchedulingRequest#setAllocationRequestId(long)
  * @param allocationRequestId allocationRequestId of the
  *  request
@@ -107,7 +107,7 @@ public abstract class SchedulingRequest {
 
 /**
  * Set the executionType of the request.
- * 
+ *
  * @see SchedulingRequest#setExecutionType(ExecutionTypeRequest)
  * @param executionType executionType of the request
  * @return {@link SchedulingRequest.SchedulingRequestBuilder}
@@ -119,7 +119,7 @@ public abstract class SchedulingRequest {
   schedulingRequest.

[07/32] hadoop git commit: YARN-6595. [API] Add Placement Constraints at the application level. (Arun Suresh via kkaranasos)

[asuresh]

YARN-6595. [API] Add Placement Constraints at the application level. (Arun 
Suresh via kkaranasos)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db928556
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db928556
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db928556

Branch: refs/heads/trunk
Commit: db928556c81e5950b3fe374fa5b99ab26791ef3a
Parents: b57e8bc
Author: Konstantinos Karanasos 
Authored: Mon Nov 13 15:25:24 2017 -0800
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../RegisterApplicationMasterRequest.java   |  42 -
 .../yarn/api/resource/PlacementConstraint.java  | 156 +++
 .../src/main/proto/yarn_protos.proto|   6 +
 .../src/main/proto/yarn_service_protos.proto|   1 +
 .../RegisterApplicationMasterRequestPBImpl.java | 106 -
 .../hadoop/yarn/api/BasePBImplRecordsTest.java  |  11 ++
 6 files changed, 313 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/db928556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
index 395e190..f2d537a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
@@ -18,11 +18,16 @@
 
 package org.apache.hadoop.yarn.api.protocolrecords;
 
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
 import org.apache.hadoop.yarn.util.Records;
-
 /**
  * The request sent by the {@code ApplicationMaster} to {@code ResourceManager}
  * on registration.
@@ -132,4 +137,39 @@ public abstract class RegisterApplicationMasterRequest {
   @Public
   @Stable
   public abstract void setTrackingUrl(String trackingUrl);
+
+  /**
+   * Return all Placement Constraints specified at the Application level. The
+   * mapping is from a set of allocation tags to a
+   * PlacementConstraint associated with the tags, i.e., each
+   * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest} that has 
those
+   * tags will be placed taking into account the corresponding constraint.
+   *
+   * @return A map of Placement Constraints.
+   */
+  @Public
+  @Unstable
+  public Map getPlacementConstraints() {
+return new HashMap<>();
+  }
+
+  /**
+   * Set Placement Constraints applicable to the
+   * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}s
+   * of this application.
+   * The mapping is from a set of allocation tags to a
+   * PlacementConstraint associated with the tags.
+   * For example:
+   *  Map 
+   *   hb_regionserver - node_anti_affinity,
+   *   hb_regionserver, hb_master - rack_affinity,
+   *   ...
+   *  
+   * @param placementConstraints Placement Constraint Mapping.
+   */
+  @Public
+  @Unstable
+  public void setPlacementConstraints(
+  Map placementConstraints) {
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db928556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
index f0e3982..b6e851a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
@@ -54,6 +54,26 @@ public class PlacementConstraint {
 return constraintExpr;
   }
 
+  @Override
+  

[06/32] hadoop git commit: YARN-6594. [API] Introduce SchedulingRequest object. (Konstantinos Karanasos via wangda)

[asuresh]

YARN-6594. [API] Introduce SchedulingRequest object. (Konstantinos Karanasos 
via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b57e8bc3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b57e8bc3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b57e8bc3

Branch: refs/heads/trunk
Commit: b57e8bc3002a95d2f2f328554d792151cdc1120d
Parents: 33a796d
Author: Wangda Tan 
Authored: Mon Oct 30 16:54:02 2017 -0700
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/api/records/ResourceSizing.java |  64 +
 .../yarn/api/records/SchedulingRequest.java | 205 ++
 .../src/main/proto/yarn_protos.proto|  14 +
 .../records/impl/pb/ResourceSizingPBImpl.java   | 117 
 .../impl/pb/SchedulingRequestPBImpl.java| 266 +++
 5 files changed, 666 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57e8bc3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
new file mode 100644
index 000..d82be11
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@code ResourceSizing} contains information for the size of a
+ * {@link SchedulingRequest}, such as the number of requested allocations and
+ * the resources for each allocation.
+ */
+@Public
+@Unstable
+public abstract class ResourceSizing {
+
+  @Public
+  @Unstable
+  public static ResourceSizing newInstance(Resource resources) {
+return ResourceSizing.newInstance(1, resources);
+  }
+
+  @Public
+  @Unstable
+  public static ResourceSizing newInstance(int numAllocations, Resource 
resources) {
+ResourceSizing resourceSizing = Records.newRecord(ResourceSizing.class);
+resourceSizing.setNumAllocations(numAllocations);
+resourceSizing.setResources(resources);
+return resourceSizing;
+  }
+
+  @Public
+  @Unstable
+  public abstract int getNumAllocations();
+
+  @Public
+  @Unstable
+  public abstract void setNumAllocations(int numAllocations);
+
+  @Public
+  @Unstable
+  public abstract Resource getResources();
+
+  @Public
+  @Unstable
+  public abstract void setResources(Resource resources);
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57e8bc3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
new file mode 100644
index 000..47a0697
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in 

[05/32] hadoop git commit: YARN-6593. [API] Introduce Placement Constraint object. (Konstantinos Karanasos via wangda)

[asuresh]

YARN-6593. [API] Introduce Placement Constraint object. (Konstantinos Karanasos 
via wangda)

Change-Id: Id00edb7185fdf01cce6e40f920cac3585f8cbe9c


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33a796d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33a796d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33a796d9

Branch: refs/heads/trunk
Commit: 33a796d9b778bf7350e87a4e36ca30c925cf7036
Parents: 1453a04
Author: Wangda Tan 
Authored: Thu Aug 3 14:03:55 2017 -0700
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../yarn/api/resource/PlacementConstraint.java  | 567 +++
 .../yarn/api/resource/PlacementConstraints.java | 286 ++
 .../hadoop/yarn/api/resource/package-info.java  |  23 +
 .../src/main/proto/yarn_protos.proto|  55 ++
 .../api/resource/TestPlacementConstraints.java  | 106 
 .../PlacementConstraintFromProtoConverter.java  | 116 
 .../pb/PlacementConstraintToProtoConverter.java | 174 ++
 .../apache/hadoop/yarn/api/pb/package-info.java |  23 +
 .../yarn/api/records/impl/pb/ProtoUtils.java|  27 +
 .../PlacementConstraintTransformations.java | 209 +++
 .../hadoop/yarn/api/resource/package-info.java  |  23 +
 .../TestPlacementConstraintPBConversion.java| 195 +++
 .../TestPlacementConstraintTransformations.java | 183 ++
 13 files changed, 1987 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/33a796d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
new file mode 100644
index 000..f0e3982
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
@@ -0,0 +1,567 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * {@code PlacementConstraint} represents a placement constraint for a resource
+ * allocation.
+ */
+@Public
+@Unstable
+public class PlacementConstraint {
+
+  /**
+   * The constraint expression tree.
+   */
+  private AbstractConstraint constraintExpr;
+
+  public PlacementConstraint(AbstractConstraint constraintExpr) {
+this.constraintExpr = constraintExpr;
+  }
+
+  /**
+   * Get the constraint expression of the placement constraint.
+   *
+   * @return the constraint expression
+   */
+  public AbstractConstraint getConstraintExpr() {
+return constraintExpr;
+  }
+
+  /**
+   * Interface used to enable the elements of the constraint tree to be 
visited.
+   */
+  @Private
+  public interface Visitable {
+/**
+ * Visitor pattern.
+ *
+ * @param visitor visitor to be used
+ * @param  defines the type that the visitor will use and the return 
type
+ *  of the accept.
+ * @return the result of visiting a given object.
+ */
+ T accept(Visitor visitor);
+
+  }
+
+  /**
+   * Visitor API for a constraint tree.
+   *
+   * @param  determines the return type of the visit methods.
+   */
+  @Private
+  public interface Visitor {
+T visit(SingleConstraint constraint);
+
+T visit(TargetExpression target);
+
+T visit(TargetConstraint constraint);
+
+T visit(CardinalityConstraint constraint);
+
+T visit(And constraint);
+

[03/32] hadoop git commit: YARN-7783. Add validation step to ensure constraints are not violated due to order in which a request is processed. (asuresh)

[asuresh]

YARN-7783. Add validation step to ensure constraints are not violated due to 
order in which a request is processed. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4c539fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4c539fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4c539fc

Branch: refs/heads/trunk
Commit: a4c539fcdba817e313b2375abf2c4c9a1d13a4fd
Parents: 9b81cb0
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Jan 23 08:15:58 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../algorithm/DefaultPlacementAlgorithm.java| 119 +--
 .../constraint/TestPlacementProcessor.java  |  49 
 2 files changed, 155 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4c539fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
index 9887749..4e6473f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
@@ -22,6 +22,7 @@ import java.util.Iterator;
 import java.util.List;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
@@ -69,13 +70,9 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
   public boolean attemptPlacementOnNode(ApplicationId appId,
   SchedulingRequest schedulingRequest, SchedulerNode schedulerNode)
   throws InvalidAllocationTagsQueryException {
-int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations();
-if (numAllocs > 0) {
-  if (PlacementConstraintsUtil.canSatisfyConstraints(appId,
-  schedulingRequest, schedulerNode,
-  constraintManager, tagsManager)) {
-return true;
-  }
+if (PlacementConstraintsUtil.canSatisfyConstraints(appId,
+schedulingRequest, schedulerNode, constraintManager, tagsManager)) {
+  return true;
 }
 return false;
   }
@@ -93,6 +90,9 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
 int rePlacementCount = RE_ATTEMPT_COUNT;
 while (rePlacementCount > 0) {
   doPlacement(requests, resp, allNodes, rejectedRequests);
+  // Double check if placement constraints are really satisfied
+  validatePlacement(requests.getApplicationId(), resp,
+  rejectedRequests);
   if (rejectedRequests.size() == 0 || rePlacementCount == 1) {
 break;
   }
@@ -122,9 +122,14 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
 break;
   }
   SchedulingRequest schedulingRequest = requestIterator.next();
+  PlacedSchedulingRequest placedReq =
+  new PlacedSchedulingRequest(schedulingRequest);
+  placedReq.setPlacementAttempt(requests.getPlacementAttempt());
+  resp.getPlacedRequests().add(placedReq);
   CircularIterator nodeIter =
   new CircularIterator(lastSatisfiedNode, nIter, allNodes);
-  int numAllocs = 
schedulingRequest.getResourceSizing().getNumAllocations();
+  int numAllocs =
+  schedulingRequest.getResourceSizing().getNumAllocations();
   while (nodeIter.hasNext() && numAllocs > 0) {
 SchedulerNode node = nodeIter.next();
 try {
@@ -135,11 +140,7 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
   requests.getApplicationId(), schedulingRequest, node)) {
 schedulingRequest.getResourceSizing()
 .

[08/32] hadoop git commit: YARN-7448. [API] Add SchedulingRequest to the AllocateRequest. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-7448. [API] Add SchedulingRequest to the AllocateRequest. (Panagiotis 
Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69de9a1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69de9a1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69de9a1b

Branch: refs/heads/trunk
Commit: 69de9a1ba9a587c7e03ae7c7aeae93e04c36d665
Parents: db92855
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Nov 17 10:42:43 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../api/protocolrecords/AllocateRequest.java| 42 ++
 .../hadoop/yarn/api/records/ResourceSizing.java | 27 +++
 .../yarn/api/records/SchedulingRequest.java |  1 +
 .../src/main/proto/yarn_service_protos.proto|  1 +
 .../impl/pb/AllocateRequestPBImpl.java  | 83 
 .../records/impl/pb/ResourceSizingPBImpl.java   |  2 +-
 .../impl/pb/SchedulingRequestPBImpl.java| 16 
 .../hadoop/yarn/api/TestPBImplRecords.java  | 19 +
 8 files changed, 190 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/69de9a1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
index ae0891e..d8d2347 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.api.protocolrecords;
 
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -28,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
 import org.apache.hadoop.yarn.util.Records;
 
@@ -212,6 +214,32 @@ public abstract class AllocateRequest {
   public abstract void setUpdateRequests(
   List updateRequests);
 
+  /**
+   * Get the list of Scheduling requests being sent by the
+   * ApplicationMaster.
+   * @return list of {@link SchedulingRequest} being sent by the
+   * ApplicationMaster.
+   */
+  @Public
+  @Unstable
+  public List getSchedulingRequests() {
+return Collections.EMPTY_LIST;
+  }
+
+  /**
+   * Set the list of Scheduling requests to inform the
+   * ResourceManager about the application's resource requirements
+   * (potentially including allocation tags & placement constraints).
+   * @param schedulingRequests list of SchedulingRequest to update
+   *  the ResourceManager about the application's resource
+   *  requirements.
+   */
+  @Public
+  @Unstable
+  public void setSchedulingRequests(
+  List schedulingRequests) {
+  }
+
   @Public
   @Unstable
   public static AllocateRequestBuilder newBuilder() {
@@ -314,6 +342,20 @@ public abstract class AllocateRequest {
 }
 
 /**
+ * Set the schedulingRequests of the request.
+ * @see AllocateRequest#setSchedulingRequests(List)
+ * @param schedulingRequests SchedulingRequest of the request
+ * @return {@link AllocateRequestBuilder}
+ */
+@Public
+@Unstable
+public AllocateRequestBuilder schedulingRequests(
+List schedulingRequests) {
+  allocateRequest.setSchedulingRequests(schedulingRequests);
+  return this;
+}
+
+/**
  * Return generated {@link AllocateRequest} object.
  * @return {@link AllocateRequest}
  */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69de9a1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
index d82be11..8cdc63f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-y

[11/32] hadoop git commit: YARN-7669. API and interface modifications for placement constraint processor. (asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
new file mode 100644
index 000..9571f0e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Encapsulates the output of the ConstraintPlacementAlgorithm. The Algorithm
+ * is free to produce multiple of output objects at the end of each run and it
+ * must use the provided ConstraintPlacementAlgorithmOutputCollector to
+ * aggregate/collect this output. Similar to the MapReduce Mapper/Reducer
+ * which is provided a collector to collect output.
+ */
+public class ConstraintPlacementAlgorithmOutput {
+
+  private final ApplicationId applicationId;
+
+  public ConstraintPlacementAlgorithmOutput(ApplicationId applicationId) {
+this.applicationId = applicationId;
+  }
+
+  private final List placedRequests =
+  new ArrayList<>();
+
+  private final List rejectedRequests =
+  new ArrayList<>();
+
+  public List getPlacedRequests() {
+return placedRequests;
+  }
+
+  public List getRejectedRequests() {
+return rejectedRequests;
+  }
+
+  public ApplicationId getApplicationId() {
+return applicationId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
new file mode 100644
index 000..131fd42
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing 

[01/32] hadoop git commit: YARN-7780. Documentation for Placement Constraints. (Konstantinos Karanasos via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 7288b0833 -> 8d1e2c640


YARN-7780. Documentation for Placement Constraints. (Konstantinos Karanasos via 
asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8df7666f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8df7666f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8df7666f

Branch: refs/heads/trunk
Commit: 8df7666fe19f124e80bcc63c496607e085fcf804
Parents: add993e
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Jan 30 07:38:27 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../yarn/api/resource/PlacementConstraints.java |  17 ++-
 .../hadoop/yarn/conf/YarnConfiguration.java |  11 +-
 .../site/markdown/PlacementConstraints.md.vm| 149 +++
 3 files changed, 164 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8df7666f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
index 70a8080..c1549c5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
@@ -96,8 +96,9 @@ public final class PlacementConstraints {
* Creates a constraint that restricts the number of allocations within a
* given scope (e.g., node or rack).
*
-   * For example, {@code cardinality(NODE, 3, 10)}, restricts the number of
-   * allocations per node to be no less than 3 and no more than 10.
+   * For example, {@code cardinality(NODE, 3, 10, "zk")} is satisfied on nodes
+   * where there are no less than 3 allocations with tag "zk" and no more than
+   * 10.
*
* @param scope the scope of the constraint
* @param minCardinality determines the minimum number of allocations within
@@ -132,7 +133,7 @@ public final class PlacementConstraints {
 
   /**
* Similar to {@link #cardinality(String, int, int, String...)}, but
-   * determines only the maximum cardinality (the minimum can be as low as 0).
+   * determines only the maximum cardinality (the minimum cardinality is 0).
*
* @param scope the scope of the constraint
* @param maxCardinality determines the maximum number of allocations within
@@ -150,7 +151,7 @@ public final class PlacementConstraints {
*
* Consider a set of nodes N that belongs to the scope specified in the
* constraint. If the target expressions are satisfied at least 
minCardinality
-   * times and at most max-cardinality times in the node set N, then the
+   * times and at most maxCardinality times in the node set N, then the
* constraint is satisfied.
*
* For example, {@code targetCardinality(RACK, 2, 10, allocationTag("zk"))},
@@ -197,7 +198,7 @@ public final class PlacementConstraints {
 
 /**
  * Constructs a target expression on a node partition. It is satisfied if
- * the specified node partition has one of the specified nodePartitions
+ * the specified node partition has one of the specified nodePartitions.
  *
  * @param nodePartitions the set of values that the attribute should take
  *  values from
@@ -211,7 +212,7 @@ public final class PlacementConstraints {
 
 /**
  * Constructs a target expression on an allocation tag. It is satisfied if
- * the there are allocations with one of the given tags.
+ * there are allocations with one of the given tags.
  *
  * @param allocationTags the set of tags that the attribute should take
  *  values from
@@ -224,8 +225,8 @@ public final class PlacementConstraints {
 
 /**
  * Constructs a target expression on an allocation tag. It is satisfied if
- * the there are allocations with one of the given tags. Comparing to
- * {@link PlacementTargets#allocationTag(String...)}, this only check tags
+ * there are allocations with one of the given tags. Comparing to
+ * {@link PlacementTargets#allocationTag(String...)}, this only checks tags
  * within the application.
  *
  * @param allocationTags the set of tags that the attribute should take

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8df7666f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-

[37/37] hadoop git commit: YARN-7861. [UI2] Logs page shows duplicated containers with ATS. (Sunil G via wangda)

[asuresh]

YARN-7861. [UI2] Logs page shows duplicated containers with ATS. (Sunil G via 
wangda)

Change-Id: Id07d1ff8d519e0d9291ab037236641ceb901582c


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1453a04e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1453a04e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1453a04e

Branch: refs/heads/YARN-6592
Commit: 1453a04e92ce88b65995248c5d6a2bc934cbe65f
Parents: 5ca4bf2
Author: Wangda Tan 
Authored: Wed Jan 31 21:44:42 2018 +0800
Committer: Wangda Tan 
Committed: Wed Jan 31 21:44:42 2018 +0800

--
 .../src/main/webapp/app/controllers/yarn-app/logs.js   | 13 +++--
 .../src/main/webapp/app/routes/yarn-app/logs.js|  1 +
 2 files changed, 12 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1453a04e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
index 90ecc12..3b75f76 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
@@ -47,10 +47,14 @@ export default Ember.Controller.extend({
 this.fetchContainersForAttemptId(attemptId)
   .then(hash => {
 let containers = null;
+let containerIdArr = [];
 if (
   hash.rmContainers.get("length") > 0 &&
   hash.rmContainers.get("content")
 ) {
+  hash.rmContainers.get("content").forEach(function(o) {
+containerIdArr.push(o.id);
+  }.bind(this));
   containers = (containers || []).concat(
 hash.rmContainers.get("content")
   );
@@ -59,9 +63,14 @@ export default Ember.Controller.extend({
   hash.tsContainers.get("length") > 0 &&
   hash.tsContainers.get("content")
 ) {
+  let tscontainer = [];
+  hash.tsContainers.get("content").forEach(function(o) {
+if(!containerIdArr.contains(o.id)) {
+  tscontainer.push(o);
+}
+  }.bind(this));
   containers = (containers || []).concat(
-hash.tsContainers.get("content")
-  );
+tscontainer);
 }
 this.set("attemptContainerList", containers);
 this.initializeSelect(".js-fetch-logs-containers");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1453a04e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
index d92fc3c..8837503 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
@@ -36,6 +36,7 @@ export default AbstractRoute.extend(AppAttemptMixin, {
 
   activate() {
 const controller = this.controllerFor("yarn-app.logs");
+controller.resetAfterRefresh();
 controller.initializeSelect();
   },
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[19/37] hadoop git commit: YARN-7653. Node group support for AllocationTagsManager. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-7653. Node group support for AllocationTagsManager. (Panagiotis 
Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37f1a7b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37f1a7b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37f1a7b6

Branch: refs/heads/YARN-6592
Commit: 37f1a7b64fcc93191367330cd59d4d71d7b29ac7
Parents: 06eb63e
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Dec 22 07:24:37 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../server/resourcemanager/ResourceManager.java |   2 +-
 .../constraint/AllocationTagsManager.java   | 282 ++-
 .../rmcontainer/TestRMContainerImpl.java|   2 +-
 .../constraint/TestAllocationTagsManager.java   | 269 --
 4 files changed, 392 insertions(+), 163 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/37f1a7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index a1d3dfc..1d838f0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -496,7 +496,7 @@ public class ResourceManager extends CompositeService 
implements Recoverable {
   }
 
   protected AllocationTagsManager createAllocationTagsManager() {
-return new AllocationTagsManager();
+return new AllocationTagsManager(this.rmContext);
   }
   
   protected DelegationTokenRenewer createDelegationTokenRenewer() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/37f1a7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
index c278606..7b0b959 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.log4j.Logger;
 
 import java.util.HashMap;
@@ -38,9 +39,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.LongBinaryOperator;
 
 /**
- * Support storing maps between container-tags/applications and
- * nodes. This will be required by affinity/anti-affinity implementation and
- * cardinality.
+ * In-memory mapping between applications/container-tags and nodes/racks.
+ * Required by constrained affinity/anti-affinity and cardinality placement.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
@@ -51,48 +51,54 @@ public class AllocationTagsManager {
 
   private ReentrantReadWriteLock.ReadLock readLock;
   private ReentrantReadWriteLock.WriteLock writeLock;
+  private final RMContext rmContext;
 
-  // Application's tags to node
-  private Map<ApplicationId, NodeToCountedTags> perAppMappings =
+  // Application's tags to Node
+  private Map<ApplicationId, NodeToCountedTags> perAppNodeMappings =
+  new HashMap<>();
+  // Application's tags to Rac

[34/37] hadoop git commit: YARN-7763. Allow Constraints specified in the SchedulingRequest to override application level constraints. (Weiwei Yang via asuresh)

[asuresh]

YARN-7763. Allow Constraints specified in the SchedulingRequest to override 
application level constraints. (Weiwei Yang via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bf7c444
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bf7c444
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bf7c444

Branch: refs/heads/YARN-6592
Commit: 8bf7c444368f48f63f8011cf155f551c6b51ee21
Parents: 28fe7f3
Author: Arun Suresh <asur...@apache.org>
Authored: Sun Jan 21 19:11:17 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../scheduler/capacity/CapacityScheduler.java   |  4 +-
 .../constraint/PlacementConstraintsUtil.java| 98 +++-
 .../algorithm/DefaultPlacementAlgorithm.java|  4 +-
 .../SingleConstraintAppPlacementAllocator.java  | 10 +-
 .../TestPlacementConstraintsUtil.java   | 94 ---
 5 files changed, 123 insertions(+), 87 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf7c444/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 429f9f3..a096e2f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2621,9 +2621,9 @@ public class CapacityScheduler extends
 // Validate placement constraint is satisfied before
 // committing the request.
 try {
-  if (!PlacementConstraintsUtil.canSatisfySingleConstraint(
+  if (!PlacementConstraintsUtil.canSatisfyConstraints(
   appAttempt.getApplicationId(),
-  schedulingRequest.getAllocationTags(), schedulerNode,
+  schedulingRequest, schedulerNode,
   rmContext.getPlacementConstraintManager(),
   rmContext.getAllocationTagsManager())) {
 LOG.debug("Failed to allocate container for application "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf7c444/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
index ff5cb67..c07c16f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
@@ -54,7 +55,7 @@ public final class PlacementConstraintsUtil {
   }
 
   /**
-   * Returns true if **single** placement constraint with associated
+   * Returns true if single placement constraint with associated
* allocationTags and scope is satisfied by a s

[17/37] hadoop git commit: YARN-7669. API and interface modifications for placement constraint processor. (asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
new file mode 100644
index 000..9571f0e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Encapsulates the output of the ConstraintPlacementAlgorithm. The Algorithm
+ * is free to produce multiple of output objects at the end of each run and it
+ * must use the provided ConstraintPlacementAlgorithmOutputCollector to
+ * aggregate/collect this output. Similar to the MapReduce Mapper/Reducer
+ * which is provided a collector to collect output.
+ */
+public class ConstraintPlacementAlgorithmOutput {
+
+  private final ApplicationId applicationId;
+
+  public ConstraintPlacementAlgorithmOutput(ApplicationId applicationId) {
+this.applicationId = applicationId;
+  }
+
+  private final List placedRequests =
+  new ArrayList<>();
+
+  private final List rejectedRequests =
+  new ArrayList<>();
+
+  public List getPlacedRequests() {
+return placedRequests;
+  }
+
+  public List getRejectedRequests() {
+return rejectedRequests;
+  }
+
+  public ApplicationId getApplicationId() {
+return applicationId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
new file mode 100644
index 000..131fd42
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing 

[33/37] hadoop git commit: YARN-7780. Documentation for Placement Constraints. (Konstantinos Karanasos via asuresh)

[asuresh]

YARN-7780. Documentation for Placement Constraints. (Konstantinos Karanasos via 
asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8df7666f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8df7666f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8df7666f

Branch: refs/heads/YARN-6592
Commit: 8df7666fe19f124e80bcc63c496607e085fcf804
Parents: add993e
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Jan 30 07:38:27 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../yarn/api/resource/PlacementConstraints.java |  17 ++-
 .../hadoop/yarn/conf/YarnConfiguration.java |  11 +-
 .../site/markdown/PlacementConstraints.md.vm| 149 +++
 3 files changed, 164 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8df7666f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
index 70a8080..c1549c5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
@@ -96,8 +96,9 @@ public final class PlacementConstraints {
* Creates a constraint that restricts the number of allocations within a
* given scope (e.g., node or rack).
*
-   * For example, {@code cardinality(NODE, 3, 10)}, restricts the number of
-   * allocations per node to be no less than 3 and no more than 10.
+   * For example, {@code cardinality(NODE, 3, 10, "zk")} is satisfied on nodes
+   * where there are no less than 3 allocations with tag "zk" and no more than
+   * 10.
*
* @param scope the scope of the constraint
* @param minCardinality determines the minimum number of allocations within
@@ -132,7 +133,7 @@ public final class PlacementConstraints {
 
   /**
* Similar to {@link #cardinality(String, int, int, String...)}, but
-   * determines only the maximum cardinality (the minimum can be as low as 0).
+   * determines only the maximum cardinality (the minimum cardinality is 0).
*
* @param scope the scope of the constraint
* @param maxCardinality determines the maximum number of allocations within
@@ -150,7 +151,7 @@ public final class PlacementConstraints {
*
* Consider a set of nodes N that belongs to the scope specified in the
* constraint. If the target expressions are satisfied at least 
minCardinality
-   * times and at most max-cardinality times in the node set N, then the
+   * times and at most maxCardinality times in the node set N, then the
* constraint is satisfied.
*
* For example, {@code targetCardinality(RACK, 2, 10, allocationTag("zk"))},
@@ -197,7 +198,7 @@ public final class PlacementConstraints {
 
 /**
  * Constructs a target expression on a node partition. It is satisfied if
- * the specified node partition has one of the specified nodePartitions
+ * the specified node partition has one of the specified nodePartitions.
  *
  * @param nodePartitions the set of values that the attribute should take
  *  values from
@@ -211,7 +212,7 @@ public final class PlacementConstraints {
 
 /**
  * Constructs a target expression on an allocation tag. It is satisfied if
- * the there are allocations with one of the given tags.
+ * there are allocations with one of the given tags.
  *
  * @param allocationTags the set of tags that the attribute should take
  *  values from
@@ -224,8 +225,8 @@ public final class PlacementConstraints {
 
 /**
  * Constructs a target expression on an allocation tag. It is satisfied if
- * the there are allocations with one of the given tags. Comparing to
- * {@link PlacementTargets#allocationTag(String...)}, this only check tags
+ * there are allocations with one of the given tags. Comparing to
+ * {@link PlacementTargets#allocationTag(String...)}, this only checks tags
  * within the application.
  *
  * @param allocationTags the set of tags that the attribute should take

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8df7666f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apac

[36/37] hadoop git commit: YARN-7828. Clicking on yarn service should take to component tab. (Sunil G via wangda)

[asuresh]

YARN-7828. Clicking on yarn service should take to component tab. (Sunil G via 
wangda)

Change-Id: Ib52d6727c863bed65de30b02bf60080255af11b8


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ca4bf22
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ca4bf22
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ca4bf22

Branch: refs/heads/YARN-6592
Commit: 5ca4bf22dd45d9ac1328c74204413d90725d1405
Parents: 64344c3
Author: Wangda Tan 
Authored: Wed Jan 31 21:43:08 2018 +0800
Committer: Wangda Tan 
Committed: Wed Jan 31 21:43:08 2018 +0800

--
 .../main/webapp/app/controllers/app-table-columns.js|  2 +-
 .../src/main/webapp/app/controllers/yarn-app-attempt.js |  2 +-
 .../src/main/webapp/app/controllers/yarn-app.js |  2 +-
 .../webapp/app/controllers/yarn-component-instance.js   |  4 ++--
 .../webapp/app/controllers/yarn-component-instances.js  |  2 +-
 .../src/main/webapp/app/templates/yarn-app.hbs  | 12 ++--
 6 files changed, 12 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca4bf22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js
index fb002f9..33e3bb7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js
@@ -122,7 +122,7 @@ export default Ember.Controller.extend({
   getCellContent: function(row) {
 return {
   displayText: row.get('appName'),
-  href: `#/yarn-app/${row.id}/attempts?service=${row.get('appName')}`
+  href: `#/yarn-app/${row.id}/components?service=${row.get('appName')}`
 };
   }
 }, {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca4bf22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js
index 504b66e..4c8b8a1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js
@@ -50,7 +50,7 @@ export default Ember.Controller.extend({
 routeName: 'yarn-services'
   }, {
 text: `${serviceName} [${appId}]`,
-href: `#/yarn-app/${appId}/info?service=${serviceName}`
+href: `#/yarn-app/${appId}/components?service=${serviceName}`
   }, {
 text: "Attempts",
 href: `#/yarn-app/${appId}/attempts?service=${serviceName}`

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca4bf22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js
index b84f328..b2b99b1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js
@@ -102,7 +102,7 @@ export default Ember.Controller.extend({
 routeName: 'yarn-services'
   }, {
 text: `${serviceName} [${appId}]`,
-href: `#/yarn-app/${appId}/info?service=${serviceName}`
+href: `#/yarn-app/${appId}/components?service=${serviceName}`
   });
 } else {
   breadcrumbs.push({

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca4bf22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance.js
index 4b8dbf4..358c6b1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance.js
+++ 

[26/37] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
index 73b4f9e..24c5a5e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -20,6 +20,8 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
 import java.util.Iterator;
 import java.util.Set;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -30,9 +32,12 @@ import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm;
 
+import static 
org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE_PARTITION;
+
 /**
  * This class contains various static methods used by the Placement Algorithms
  * to simplify constrained placement.
@@ -41,16 +46,20 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algori
 @Public
 @Unstable
 public final class PlacementConstraintsUtil {
+  private static final Log LOG =
+  LogFactory.getLog(PlacementConstraintsUtil.class);
 
   // Suppresses default constructor, ensuring non-instantiability.
   private PlacementConstraintsUtil() {
   }
 
   /**
-   * Returns true if **single** application constraint with associated
+   * Returns true if **single** placement constraint with associated
* allocationTags and scope is satisfied by a specific scheduler Node.
*
-   * @param appId the application id
+   * @param targetApplicationId the application id, which could be override by
+   *   target application id specified inside 
allocation
+   *   tags.
* @param sc the placement constraint
* @param te the target expression
* @param node the scheduler node
@@ -59,32 +68,123 @@ public final class PlacementConstraintsUtil {
* @throws InvalidAllocationTagsQueryException
*/
   private static boolean canSatisfySingleConstraintExpression(
-  ApplicationId appId, SingleConstraint sc, TargetExpression te,
-  SchedulerNode node, AllocationTagsManager tm)
+  ApplicationId targetApplicationId, SingleConstraint sc,
+  TargetExpression te, SchedulerNode node, AllocationTagsManager tm)
   throws InvalidAllocationTagsQueryException {
 long minScopeCardinality = 0;
 long maxScopeCardinality = 0;
+
+// Optimizations to only check cardinality if necessary.
+int desiredMinCardinality = sc.getMinCardinality();
+int desiredMaxCardinality = sc.getMaxCardinality();
+boolean checkMinCardinality = desiredMinCardinality > 0;
+boolean checkMaxCardinality = desiredMaxCardinality < Integer.MAX_VALUE;
+
 if (sc.getScope().equals(PlacementConstraints.NODE)) {
-  minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId,
-  te.getTargetValues(), Long::max);
-  maxScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId,
-  te.getTargetValues(), Long::min);
+  if (checkMinCardinality) {
+minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(),
+targetApplicationId, te.getTargetValues(), Long::max);
+  }
+  if (checkMaxCardinality) {
+maxScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(),
+targetApplicationId, te.getTargetValues(), 

[02/37] hadoop git commit: HDFS-12528. Add an option to not disable short-circuit reads on failures. Contributed by Xiao Chen.

[asuresh]

HDFS-12528. Add an option to not disable short-circuit reads on failures. 
Contributed by Xiao Chen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e7331ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e7331ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e7331ca

Branch: refs/heads/YARN-6592
Commit: 2e7331ca264dd366b975f3c8e610cf84eb8cc155
Parents: f9dd5b6
Author: Weiwei Yang 
Authored: Wed Jan 31 07:25:03 2018 +0800
Committer: Weiwei Yang 
Committed: Wed Jan 31 07:25:03 2018 +0800

--
 .../hdfs/client/HdfsClientConfigKeys.java   |   3 +
 .../hdfs/client/impl/BlockReaderFactory.java|  16 ++-
 .../hadoop/hdfs/client/impl/DfsClientConf.java  |  19 +++-
 .../hdfs/shortcircuit/DomainSocketFactory.java  |  14 ++-
 .../hdfs/shortcircuit/ShortCircuitCache.java|  16 ++-
 .../src/main/resources/hdfs-default.xml |  11 ++
 .../client/impl/TestBlockReaderFactory.java | 111 ++-
 7 files changed, 175 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e7331ca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index d6c47b5..38c2435 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -127,6 +127,9 @@ public interface HdfsClientConfigKeys {
   boolean DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC_DEFAULT = false;
   String  DFS_DOMAIN_SOCKET_PATH_KEY = "dfs.domain.socket.path";
   String  DFS_DOMAIN_SOCKET_PATH_DEFAULT = "";
+  String DFS_DOMAIN_SOCKET_DISABLE_INTERVAL_SECOND_KEY =
+  "dfs.domain.socket.disable.interval.seconds";
+  long DFS_DOMAIN_SOCKET_DISABLE_INTERVAL_SECOND_DEFAULT = 600;
   String  DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS =
   "dfs.short.circuit.shared.memory.watcher.interrupt.check.ms";
   int DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e7331ca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
index 6f3fc61..60dde82 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.client.impl;
 
+import static 
org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DOMAIN_SOCKET_DISABLE_INTERVAL_SECOND_KEY;
 import static 
org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitFdResponse.USE_RECEIPT_VERIFICATION;
 
 import java.io.BufferedOutputStream;
@@ -644,10 +645,17 @@ public class BlockReaderFactory implements 
ShortCircuitReplicaCreator {
   LOG.debug("{}:{}", this, msg);
   return new ShortCircuitReplicaInfo(new InvalidToken(msg));
 default:
-  LOG.warn(this + ": unknown response code " + resp.getStatus() +
-  " while attempting to set up short-circuit access. " +
-  resp.getMessage() + ". Disabling short-circuit read for DataNode "
-  + datanode + " temporarily.");
+  final long expiration =
+  clientContext.getDomainSocketFactory().getPathExpireSeconds();
+  String disableMsg = "disabled temporarily for " + expiration + " 
seconds";
+  if (expiration == 0) {
+disableMsg = "not disabled";
+  }
+  LOG.warn("{}: unknown response code {} while attempting to set up "
+  + "short-circuit access. {}. Short-circuit read for "
+  + "DataNode {} is {} based on {}.",
+  this, resp.getStatus(), resp.getMessage(), datanode,
+  disableMsg, DFS_DOMAIN_SOCKET_DISABLE_INTERVAL_SECOND_KEY);
   clientContext.getDomainSocketFactory()
   .disableShortCircuitForPath(pathInfo.getPath());
   return null;

[21/37] hadoop git commit: YARN-7682. Expose canSatisfyConstraints utility function to validate a placement against a constraint. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-7682. Expose canSatisfyConstraints utility function to validate a 
placement against a constraint. (Panagiotis Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bdba01f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bdba01f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bdba01f7

Branch: refs/heads/YARN-6592
Commit: bdba01f73b58d2228e808c6f61377f101b6bac1c
Parents: a52d11f
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Jan 3 08:00:50 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../constraint/PlacementConstraintsUtil.java| 132 +
 .../algorithm/DefaultPlacementAlgorithm.java|  55 +---
 .../TestPlacementConstraintsUtil.java   | 287 +++
 .../constraint/TestPlacementProcessor.java  | 204 +++--
 4 files changed, 601 insertions(+), 77 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdba01f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
new file mode 100644
index 000..956a3c9
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType;
+import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import 
org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm;
+
+/**
+ * This class contains various static methods used by the Placement Algorithms
+ * to simplify constrained placement.
+ * (see also {@link DefaultPlacementAlgorithm}).
+ */
+@Public
+@Unstable
+public final class PlacementConstraintsUtil {
+
+  // Suppresses default constructor, ensuring non-instantiability.
+  private PlacementConstraintsUtil() {
+  }
+
+  /**
+   * Returns true if **single** application constraint with associated
+   * allocationTags and scope is satisfied by a specific scheduler Node.
+   *
+   * @param appId the application id
+   * @param sc the placement constraint
+   * @param te the target expression
+   * @param node the scheduler node
+   * @param tm the allocation tags store
+   * @return true if single application constraint is satisfied by node
+   * @throws InvalidAllocationTagsQueryException
+   */
+  private static boolean canS

[22/37] hadoop git commit: YARN-7613. Implement Basic algorithm for constraint based placement. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-7613. Implement Basic algorithm for constraint based placement. 
(Panagiotis Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a52d11fb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a52d11fb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a52d11fb

Branch: refs/heads/YARN-6592
Commit: a52d11fb8c103f14e42692600a058ba3b56e2ecf
Parents: f9af15d
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Dec 27 22:59:22 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   4 +
 .../src/main/resources/yarn-default.xml |   8 +-
 .../rmcontainer/RMContainerImpl.java|  10 +-
 .../constraint/AllocationTagsManager.java   | 121 ++---
 .../algorithm/DefaultPlacementAlgorithm.java| 172 +++
 .../iterators/PopularTagsIterator.java  |  71 
 .../algorithm/iterators/SerialIterator.java |  53 ++
 .../algorithm/iterators/package-info.java   |  29 
 .../constraint/algorithm/package-info.java  |  29 
 .../constraint/processor/BatchedRequests.java   |  45 -
 .../processor/PlacementProcessor.java   |  32 ++--
 .../processor/SamplePlacementAlgorithm.java | 144 
 .../constraint/TestAllocationTagsManager.java   | 156 -
 .../TestBatchedRequestsIterators.java   |  82 +
 .../constraint/TestPlacementProcessor.java  |   4 +-
 15 files changed, 721 insertions(+), 239 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8fb3c2e..367b1ae 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -536,6 +536,10 @@ public class YarnConfiguration extends Configuration {
   public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS =
   RM_PREFIX + "placement-constraints.algorithm.class";
 
+  /** Used for BasicPlacementAlgorithm - default SERIAL. **/
+  public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_ITERATOR =
+  RM_PREFIX + "placement-constraints.algorithm.iterator";
+
   public static final String RM_PLACEMENT_CONSTRAINTS_ENABLED =
   RM_PREFIX + "placement-constraints.enabled";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 6d52ace..509a040 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -145,7 +145,13 @@
   
 Constraint Placement Algorithm to be used.
 yarn.resourcemanager.placement-constraints.algorithm.class
-
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor.SamplePlacementAlgorithm
+
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm
+  
+
+  
+Placement Algorithm Requests Iterator to be 
used.
+yarn.resourcemanager.placement-constraints.algorithm.iterator
+SERIAL
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index c873509..2c4ef7b 100644
--- 
a/hadoop-yarn-pro

[04/37] hadoop git commit: YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh)

[asuresh]

YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8c5f5b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8c5f5b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8c5f5b2

Branch: refs/heads/YARN-6592
Commit: f8c5f5b23732a1e35f012c1a6850bed09c8a5180
Parents: c23980c
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 25 19:09:21 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../scheduler/AppSchedulingInfo.java| 10 -
 .../scheduler/common/fica/FiCaSchedulerApp.java |  7 +++
 .../constraint/TestPlacementProcessor.java  | 45 
 3 files changed, 60 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c5f5b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 0389895..1efdd8b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -694,6 +694,12 @@ public class AppSchedulingInfo {
   metrics.runAppAttempt(applicationId, user);
 }
 
+updateMetrics(applicationId, type, node, containerAllocated, user, queue);
+  }
+
+  public static void updateMetrics(ApplicationId applicationId, NodeType type,
+  SchedulerNode node, Container containerAllocated, String user,
+  Queue queue) {
 if (LOG.isDebugEnabled()) {
   LOG.debug("allocate: applicationId=" + applicationId + " container="
   + containerAllocated.getId() + " host=" + containerAllocated
@@ -702,10 +708,10 @@ public class AppSchedulingInfo {
   + type);
 }
 if(node != null) {
-  metrics.allocateResources(node.getPartition(), user, 1,
+  queue.getMetrics().allocateResources(node.getPartition(), user, 1,
   containerAllocated.getResource(), true);
 }
-metrics.incrNodeTypeAggregations(user, type);
+queue.getMetrics().incrNodeTypeAggregations(user, type);
   }
 
   // Get AppPlacementAllocator by specified schedulerKey

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c5f5b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 7eb1e31..f3da0a3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -56,6 +56,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRese
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
@@ -548,6 +549,1

[10/37] hadoop git commit: YARN-7783. Add validation step to ensure constraints are not violated due to order in which a request is processed. (asuresh)

[asuresh]

YARN-7783. Add validation step to ensure constraints are not violated due to 
order in which a request is processed. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4c539fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4c539fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4c539fc

Branch: refs/heads/YARN-6592
Commit: a4c539fcdba817e313b2375abf2c4c9a1d13a4fd
Parents: 9b81cb0
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Jan 23 08:15:58 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../algorithm/DefaultPlacementAlgorithm.java| 119 +--
 .../constraint/TestPlacementProcessor.java  |  49 
 2 files changed, 155 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4c539fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
index 9887749..4e6473f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
@@ -22,6 +22,7 @@ import java.util.Iterator;
 import java.util.List;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
@@ -69,13 +70,9 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
   public boolean attemptPlacementOnNode(ApplicationId appId,
   SchedulingRequest schedulingRequest, SchedulerNode schedulerNode)
   throws InvalidAllocationTagsQueryException {
-int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations();
-if (numAllocs > 0) {
-  if (PlacementConstraintsUtil.canSatisfyConstraints(appId,
-  schedulingRequest, schedulerNode,
-  constraintManager, tagsManager)) {
-return true;
-  }
+if (PlacementConstraintsUtil.canSatisfyConstraints(appId,
+schedulingRequest, schedulerNode, constraintManager, tagsManager)) {
+  return true;
 }
 return false;
   }
@@ -93,6 +90,9 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
 int rePlacementCount = RE_ATTEMPT_COUNT;
 while (rePlacementCount > 0) {
   doPlacement(requests, resp, allNodes, rejectedRequests);
+  // Double check if placement constraints are really satisfied
+  validatePlacement(requests.getApplicationId(), resp,
+  rejectedRequests);
   if (rejectedRequests.size() == 0 || rePlacementCount == 1) {
 break;
   }
@@ -122,9 +122,14 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
 break;
   }
   SchedulingRequest schedulingRequest = requestIterator.next();
+  PlacedSchedulingRequest placedReq =
+  new PlacedSchedulingRequest(schedulingRequest);
+  placedReq.setPlacementAttempt(requests.getPlacementAttempt());
+  resp.getPlacedRequests().add(placedReq);
   CircularIterator nodeIter =
   new CircularIterator(lastSatisfiedNode, nIter, allNodes);
-  int numAllocs = 
schedulingRequest.getResourceSizing().getNumAllocations();
+  int numAllocs =
+  schedulingRequest.getResourceSizing().getNumAllocations();
   while (nodeIter.hasNext() && numAllocs > 0) {
 SchedulerNode node = nodeIter.next();
 try {
@@ -135,11 +140,7 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
   requests.getApplicationId(), schedulingRequest, node)) {
 schedulingRequest.getResourceSizing()
 .

[06/37] hadoop git commit: YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and NMContainerStatus to handle all recovery cases. (asuresh)

[asuresh]

YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and 
NMContainerStatus to handle all recovery cases. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5c1fc88
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5c1fc88
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5c1fc88

Branch: refs/heads/YARN-6592
Commit: a5c1fc881e21ebf43da7ead5f3852808fce25492
Parents: 4eda58c
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Jan 12 14:37:06 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/api/records/Container.java  | 15 +
 .../src/main/proto/yarn_protos.proto|  1 +
 .../api/records/impl/pb/ContainerPBImpl.java| 31 +
 .../yarn/security/ContainerTokenIdentifier.java | 69 +++-
 .../src/main/proto/yarn_security_token.proto|  1 +
 .../api/protocolrecords/NMContainerStatus.java  | 14 
 .../impl/pb/NMContainerStatusPBImpl.java| 33 ++
 .../yarn_server_common_service_protos.proto |  1 +
 .../containermanager/ContainerManagerImpl.java  |  3 +-
 .../container/ContainerImpl.java| 19 +++---
 .../rmcontainer/RMContainerImpl.java| 10 ++-
 .../scheduler/SchedulerApplicationAttempt.java  |  3 +-
 .../security/RMContainerTokenSecretManager.java | 21 ++
 .../capacity/TestContainerAllocation.java   |  5 +-
 14 files changed, 194 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
index 4fdc803..b9ca3f9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
@@ -27,6 +27,9 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
 import org.apache.hadoop.yarn.util.Records;
 
+import java.util.Collections;
+import java.util.Set;
+
 /**
  * {@code Container} represents an allocated resource in the cluster.
  * 
@@ -256,4 +259,16 @@ public abstract class Container implements 
Comparable {
   public void setVersion(int version) {
 throw new UnsupportedOperationException();
   }
+
+  @Private
+  @Unstable
+  public Set getAllocationTags() {
+return Collections.EMPTY_SET;
+  }
+
+  @Private
+  @Unstable
+  public void setAllocationTags(Set allocationTags) {
+
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 5cb1177..25c8569 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -152,6 +152,7 @@ message ContainerProto {
   optional ExecutionTypeProto execution_type = 7 [default = GUARANTEED];
   optional int64 allocation_request_id = 8 [default = -1];
   optional int32 version = 9 [default = 0];
+  repeated string allocation_tags = 10;
 }
 
 message ContainerReportProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
index be84938..47be2f0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
@@ -36,6 +36,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.PriorityPr

[13/37] hadoop git commit: YARN-6594. [API] Introduce SchedulingRequest object. (Konstantinos Karanasos via wangda)

[asuresh]

YARN-6594. [API] Introduce SchedulingRequest object. (Konstantinos Karanasos 
via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b57e8bc3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b57e8bc3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b57e8bc3

Branch: refs/heads/YARN-6592
Commit: b57e8bc3002a95d2f2f328554d792151cdc1120d
Parents: 33a796d
Author: Wangda Tan 
Authored: Mon Oct 30 16:54:02 2017 -0700
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/api/records/ResourceSizing.java |  64 +
 .../yarn/api/records/SchedulingRequest.java | 205 ++
 .../src/main/proto/yarn_protos.proto|  14 +
 .../records/impl/pb/ResourceSizingPBImpl.java   | 117 
 .../impl/pb/SchedulingRequestPBImpl.java| 266 +++
 5 files changed, 666 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57e8bc3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
new file mode 100644
index 000..d82be11
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@code ResourceSizing} contains information for the size of a
+ * {@link SchedulingRequest}, such as the number of requested allocations and
+ * the resources for each allocation.
+ */
+@Public
+@Unstable
+public abstract class ResourceSizing {
+
+  @Public
+  @Unstable
+  public static ResourceSizing newInstance(Resource resources) {
+return ResourceSizing.newInstance(1, resources);
+  }
+
+  @Public
+  @Unstable
+  public static ResourceSizing newInstance(int numAllocations, Resource 
resources) {
+ResourceSizing resourceSizing = Records.newRecord(ResourceSizing.class);
+resourceSizing.setNumAllocations(numAllocations);
+resourceSizing.setResources(resources);
+return resourceSizing;
+  }
+
+  @Public
+  @Unstable
+  public abstract int getNumAllocations();
+
+  @Public
+  @Unstable
+  public abstract void setNumAllocations(int numAllocations);
+
+  @Public
+  @Unstable
+  public abstract Resource getResources();
+
+  @Public
+  @Unstable
+  public abstract void setResources(Resource resources);
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57e8bc3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
new file mode 100644
index 000..47a0697
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in 

[03/37] hadoop git commit: HDFS-13083. RBF: Fix doc error setting up client. Contributed by tartarus.

[asuresh]

HDFS-13083. RBF: Fix doc error setting up client. Contributed by tartarus.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5206b2c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5206b2c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5206b2c7

Branch: refs/heads/YARN-6592
Commit: 5206b2c7ca479dd53f614d75bab594043a9866e1
Parents: 2e7331c
Author: Inigo Goiri 
Authored: Tue Jan 30 18:27:18 2018 -0800
Committer: Inigo Goiri 
Committed: Tue Jan 30 18:27:18 2018 -0800

--
 .../hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5206b2c7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md 
b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
index 6b21123..b5fbcc4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
@@ -236,7 +236,7 @@ For example, a cluster with 4 namespaces **ns0, ns1, ns2, 
ns3**, can add a new o
 ns0,ns1,ns2,ns3,ns-fed
   
   
-dfs.namenodes.ns-fed
+dfs.ha.namenodes.ns-fed
 r1,r2
   
   


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[14/37] hadoop git commit: YARN-7448. [API] Add SchedulingRequest to the AllocateRequest. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-7448. [API] Add SchedulingRequest to the AllocateRequest. (Panagiotis 
Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69de9a1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69de9a1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69de9a1b

Branch: refs/heads/YARN-6592
Commit: 69de9a1ba9a587c7e03ae7c7aeae93e04c36d665
Parents: db92855
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Nov 17 10:42:43 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../api/protocolrecords/AllocateRequest.java| 42 ++
 .../hadoop/yarn/api/records/ResourceSizing.java | 27 +++
 .../yarn/api/records/SchedulingRequest.java |  1 +
 .../src/main/proto/yarn_service_protos.proto|  1 +
 .../impl/pb/AllocateRequestPBImpl.java  | 83 
 .../records/impl/pb/ResourceSizingPBImpl.java   |  2 +-
 .../impl/pb/SchedulingRequestPBImpl.java| 16 
 .../hadoop/yarn/api/TestPBImplRecords.java  | 19 +
 8 files changed, 190 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/69de9a1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
index ae0891e..d8d2347 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.api.protocolrecords;
 
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -28,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
 import org.apache.hadoop.yarn.util.Records;
 
@@ -212,6 +214,32 @@ public abstract class AllocateRequest {
   public abstract void setUpdateRequests(
   List updateRequests);
 
+  /**
+   * Get the list of Scheduling requests being sent by the
+   * ApplicationMaster.
+   * @return list of {@link SchedulingRequest} being sent by the
+   * ApplicationMaster.
+   */
+  @Public
+  @Unstable
+  public List getSchedulingRequests() {
+return Collections.EMPTY_LIST;
+  }
+
+  /**
+   * Set the list of Scheduling requests to inform the
+   * ResourceManager about the application's resource requirements
+   * (potentially including allocation tags & placement constraints).
+   * @param schedulingRequests list of SchedulingRequest to update
+   *  the ResourceManager about the application's resource
+   *  requirements.
+   */
+  @Public
+  @Unstable
+  public void setSchedulingRequests(
+  List schedulingRequests) {
+  }
+
   @Public
   @Unstable
   public static AllocateRequestBuilder newBuilder() {
@@ -314,6 +342,20 @@ public abstract class AllocateRequest {
 }
 
 /**
+ * Set the schedulingRequests of the request.
+ * @see AllocateRequest#setSchedulingRequests(List)
+ * @param schedulingRequests SchedulingRequest of the request
+ * @return {@link AllocateRequestBuilder}
+ */
+@Public
+@Unstable
+public AllocateRequestBuilder schedulingRequests(
+List schedulingRequests) {
+  allocateRequest.setSchedulingRequests(schedulingRequests);
+  return this;
+}
+
+/**
  * Return generated {@link AllocateRequest} object.
  * @return {@link AllocateRequest}
  */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69de9a1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
index d82be11..8cdc63f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/had

[11/37] hadoop git commit: YARN-7709. Remove SELF from TargetExpression type. (Konstantinos Karanasos via asuresh)

[asuresh]

YARN-7709. Remove SELF from TargetExpression type. (Konstantinos Karanasos via 
asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8779a357
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8779a357
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8779a357

Branch: refs/heads/YARN-6592
Commit: 8779a35742085fadddccc21342b55d4f17fae5c2
Parents: 29d9e4d
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 18 04:29:57 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../yarn/api/resource/PlacementConstraint.java  | 32 ++
 .../yarn/api/resource/PlacementConstraints.java | 35 +---
 .../api/resource/TestPlacementConstraints.java  |  3 +-
 .../PlacementConstraintTransformations.java | 19 +++
 .../TestPlacementConstraintTransformations.java | 35 +---
 .../constraint/PlacementConstraintsUtil.java| 10 --
 6 files changed, 64 insertions(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8779a357/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
index b6e851a..4d998ac 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
@@ -242,7 +242,7 @@ public class PlacementConstraint {
  * Enum specifying the type of the target expression.
  */
 public enum TargetType {
-  NODE_ATTRIBUTE, ALLOCATION_TAG, SELF
+  NODE_ATTRIBUTE, ALLOCATION_TAG
 }
 
 private TargetType targetType;
@@ -418,23 +418,25 @@ public class PlacementConstraint {
   }
 
   /**
-   * Class that represents a cardinality constraint. Such a constraint the
-   * number of allocations within a given scope to some minimum and maximum
-   * values.
+   * Class that represents a cardinality constraint. Such a constraint allows
+   * the number of allocations with a specific set of tags and within a given
+   * scope to be between some minimum and maximum values.
*
* It is a specialized version of the {@link SingleConstraint}, where the
-   * target is self (i.e., the allocation to which the constraint is attached).
+   * target is a set of allocation tags.
*/
   public static class CardinalityConstraint extends AbstractConstraint {
 private String scope;
 private int minCardinality;
 private int maxCardinality;
+private Set allocationTags;
 
 public CardinalityConstraint(String scope, int minCardinality,
-int maxCardinality) {
+int maxCardinality, Set allocationTags) {
   this.scope = scope;
   this.minCardinality = minCardinality;
   this.maxCardinality = maxCardinality;
+  this.allocationTags = allocationTags;
 }
 
 /**
@@ -464,11 +466,21 @@ public class PlacementConstraint {
   return maxCardinality;
 }
 
+/**
+ * Get the allocation tags of the constraint.
+ *
+ * @return the allocation tags of the constraint
+ */
+public Set getAllocationTags() {
+  return allocationTags;
+}
+
 @Override
 public  T accept(Visitor visitor) {
   return visitor.visit(this);
 }
 
+
 @Override
 public boolean equals(Object o) {
   if (this == o) {
@@ -486,7 +498,11 @@ public class PlacementConstraint {
   if (maxCardinality != that.maxCardinality) {
 return false;
   }
-  return scope != null ? scope.equals(that.scope) : that.scope == null;
+  if (scope != null ? !scope.equals(that.scope) : that.scope != null) {
+return false;
+  }
+  return allocationTags != null ? 
allocationTags.equals(that.allocationTags)
+  : that.allocationTags == null;
 }
 
 @Override
@@ -494,6 +510,8 @@ public class PlacementConstraint {
   int result = scope != null ? scope.hashCode() : 0;
   result = 31 * result + minCardinality;
   result = 31 * result + maxCardinality;
+  result = 31 * result
+  + (allocationTags != null ? allocationTags.hashCode() : 0);
   return result;
 }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8779a357/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementCons

[20/37] hadoop git commit: YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos Karanasos via asuresh)

[asuresh]

YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos 
Karanasos via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1efb2b6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1efb2b6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1efb2b6f

Branch: refs/heads/YARN-6592
Commit: 1efb2b6f250022f41fe5911c1bb3028ec15c5447
Parents: 37f1a7b
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Dec 22 13:26:30 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../resourcemanager/RMActiveServiceContext.java |  15 +
 .../yarn/server/resourcemanager/RMContext.java  |   6 +
 .../server/resourcemanager/RMContextImpl.java   |  13 +
 .../server/resourcemanager/ResourceManager.java |  13 +
 .../MemoryPlacementConstraintManager.java   | 282 +++
 .../constraint/PlacementConstraintManager.java  | 151 ++
 .../PlacementConstraintManagerService.java  |  93 ++
 .../scheduler/constraint/package-info.java  |  29 ++
 .../TestPlacementConstraintManagerService.java  | 182 
 9 files changed, 784 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1efb2b6f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 4d0c230..06a1d00 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -43,6 +43,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator;
 import 
org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
 import 
org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
@@ -109,6 +110,7 @@ public class RMActiveServiceContext {
   private RMAppLifetimeMonitor rmAppLifetimeMonitor;
   private QueueLimitCalculator queueLimitCalculator;
   private AllocationTagsManager allocationTagsManager;
+  private PlacementConstraintManager placementConstraintManager;
 
   public RMActiveServiceContext() {
 queuePlacementManager = new PlacementManager();
@@ -413,6 +415,19 @@ public class RMActiveServiceContext {
 
   @Private
   @Unstable
+  public PlacementConstraintManager getPlacementConstraintManager() {
+return placementConstraintManager;
+  }
+
+  @Private
+  @Unstable
+  public void setPlacementConstraintManager(
+  PlacementConstraintManager placementConstraintManager) {
+this.placementConstraintManager = placementConstraintManager;
+  }
+
+  @Private
+  @Unstable
   public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() {
 return rmDelegatedNodeLabelsUpdater;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1efb2b6f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 00da108..eb91a31 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop

[24/37] hadoop git commit: YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh)

[asuresh]

YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c23980c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c23980c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c23980c4

Branch: refs/heads/YARN-6592
Commit: c23980c4f2cf4c751a99fd310e60149cb32ea7c7
Parents: 644afe5
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Jan 24 14:18:32 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/api/protocolrecords/AllocateRequest.java | 4 ++--
 .../apache/hadoop/yarn/api/records/SchedulingRequest.java| 8 
 .../apache/hadoop/yarn/api/resource/PlacementConstraint.java | 3 +++
 .../hadoop/yarn/api/resource/PlacementConstraints.java   | 3 ---
 .../yarn/api/records/impl/pb/ResourceSizingPBImpl.java   | 3 +++
 .../yarn/api/records/impl/pb/SchedulingRequestPBImpl.java| 3 +++
 .../resourcemanager/scheduler/AbstractYarnScheduler.java | 1 -
 .../server/resourcemanager/scheduler/AppSchedulingInfo.java  | 3 +--
 .../yarn/server/resourcemanager/scheduler/SchedulerNode.java | 8 ++--
 .../scheduler/capacity/CapacityScheduler.java| 3 +--
 .../scheduler/constraint/AllocationTagsManager.java  | 8 +---
 .../scheduler/constraint/PlacementConstraintsUtil.java   | 4 ++--
 .../scheduler/placement/AppPlacementAllocator.java   | 4 ++--
 .../placement/SingleConstraintAppPlacementAllocator.java | 1 -
 .../scheduler/capacity/TestCapacityScheduler.java| 1 -
 .../TestCapacitySchedulerSchedulingRequestUpdate.java| 4 +++-
 .../capacity/TestSchedulingRequestContainerAllocation.java   | 8 
 .../TestSchedulingRequestContainerAllocationAsync.java   | 1 -
 18 files changed, 35 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c23980c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
index d8d2347..876957e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
@@ -229,8 +229,8 @@ public abstract class AllocateRequest {
   /**
* Set the list of Scheduling requests to inform the
* ResourceManager about the application's resource requirements
-   * (potentially including allocation tags & placement constraints).
-   * @param schedulingRequests list of SchedulingRequest to update
+   * (potentially including allocation tags and placement constraints).
+   * @param schedulingRequests list of {@link SchedulingRequest} to update
*  the ResourceManager about the application's resource
*  requirements.
*/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c23980c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
index e32dd24..4bb2b84 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
@@ -77,7 +77,7 @@ public abstract class SchedulingRequest {
 
 /**
  * Set the allocationRequestId of the request.
- * 
+ *
  * @see SchedulingRequest#setAllocationRequestId(long)
  * @param allocationRequestId allocationRequestId of the
  *  request
@@ -107,7 +107,7 @@ public abstract class SchedulingRequest {
 
 /**
  * Set the executionType of the request.
- * 
+ *
  * @see SchedulingRequest#setExecutionType(ExecutionTypeRequest)
  * @param executionType executionType of the request
  * @return {@link SchedulingRequest.SchedulingRequestBuilder}
@@ -119,7 +119,7 @@ public abstract class SchedulingRequest {
   schedulingRequest.

[27/37] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)

[asuresh]

YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda 
Tan via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38af2379
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38af2379
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38af2379

Branch: refs/heads/YARN-6592
Commit: 38af23796971193fa529c3d08ffde8fcd6e607b6
Parents: 8779a35
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 18 14:10:30 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../v2/app/rm/TestRMContainerAllocator.java |  15 +-
 .../sls/scheduler/SLSCapacityScheduler.java |  15 +-
 .../yarn/sls/scheduler/SLSFairScheduler.java|  12 +-
 .../dev-support/findbugs-exclude.xml|   8 +
 .../yarn/api/resource/PlacementConstraints.java |  43 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |   2 +-
 ...SchedulerInvalidResoureRequestException.java |  47 ++
 .../api/impl/TestAMRMClientOnRMRestart.java |   9 +-
 .../impl/pb/AllocateRequestPBImpl.java  |   1 +
 .../server/scheduler/SchedulerRequestKey.java   |  11 +
 .../resourcemanager/DefaultAMSProcessor.java|  13 +-
 .../rmapp/attempt/RMAppAttemptImpl.java |   5 +-
 .../scheduler/AbstractYarnScheduler.java|   3 +-
 .../scheduler/AppSchedulingInfo.java| 205 +--
 .../ApplicationPlacementAllocatorFactory.java   |  68 +++
 .../scheduler/ApplicationPlacementFactory.java  |  63 ---
 .../scheduler/ContainerUpdateContext.java   |   4 +-
 .../scheduler/SchedulerApplicationAttempt.java  |  20 +-
 .../scheduler/YarnScheduler.java|  15 +-
 .../scheduler/capacity/CapacityScheduler.java   |  54 +-
 .../CapacitySchedulerConfiguration.java |   5 +
 .../allocator/RegularContainerAllocator.java|   3 +-
 .../scheduler/common/ContainerRequest.java  |  12 +
 .../scheduler/common/PendingAsk.java|   6 +
 .../scheduler/common/fica/FiCaSchedulerApp.java |   6 +
 .../constraint/AllocationTagsManager.java   |  71 +--
 .../constraint/AllocationTagsNamespaces.java|  31 --
 .../constraint/PlacementConstraintsUtil.java| 165 --
 .../algorithm/DefaultPlacementAlgorithm.java|   2 +-
 .../processor/PlacementProcessor.java   |   8 +-
 .../scheduler/fair/FairScheduler.java   |  12 +-
 .../scheduler/fifo/FifoScheduler.java   |   7 +-
 .../placement/AppPlacementAllocator.java|  66 ++-
 .../LocalityAppPlacementAllocator.java  |  35 +-
 .../SingleConstraintAppPlacementAllocator.java  | 531 +++
 .../server/resourcemanager/Application.java |   9 +-
 .../yarn/server/resourcemanager/MockAM.java |  51 ++
 .../attempt/TestRMAppAttemptTransitions.java|  10 +-
 .../rmcontainer/TestRMContainerImpl.java|   6 +-
 .../scheduler/TestAppSchedulingInfo.java|   4 +-
 .../capacity/CapacitySchedulerTestBase.java |  79 +++
 .../capacity/TestCapacityScheduler.java |  90 +---
 .../TestCapacitySchedulerAsyncScheduling.java   |   2 +-
 .../TestCapacitySchedulerAutoQueueCreation.java |   2 +-
 ...apacitySchedulerSchedulingRequestUpdate.java | 260 +
 .../capacity/TestIncreaseAllocationExpirer.java |   2 +-
 ...estSchedulingRequestContainerAllocation.java | 277 ++
 ...hedulingRequestContainerAllocationAsync.java | 139 +
 .../scheduler/capacity/TestUtils.java   |   2 +
 .../constraint/TestAllocationTagsManager.java   |  30 +-
 .../TestPlacementConstraintsUtil.java   |  36 +-
 .../scheduler/fair/FairSchedulerTestBase.java   |   6 +-
 .../fair/TestContinuousScheduling.java  |  10 +-
 .../scheduler/fair/TestFairScheduler.java   |  30 +-
 .../scheduler/fifo/TestFifoScheduler.java   |  28 +-
 ...stSingleConstraintAppPlacementAllocator.java | 403 ++
 56 files changed, 2557 insertions(+), 492 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 85e4181..7875917 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ 
b/hadoop-mapreduce-pr

[25/37] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index a3b88c0..01d5e6c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -170,7 +170,7 @@ public class TestCapacitySchedulerAutoQueueCreation
   1 * GB, 1, true, priority, recordFactory);
 
   cs.allocate(appAttemptId, Collections.singletonList(r1),
-  Collections.emptyList(), 
Collections.singletonList(host),
+  null, Collections.emptyList(), 
Collections.singletonList(host),
   null, NULL_UPDATE_REQUESTS);
 
   //And this will result in container assignment for app1

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java
new file mode 100644
index 000..b6ac4b6
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+public class 

[15/37] hadoop git commit: YARN-7522. Introduce AllocationTagsManager to associate allocation tags to nodes. (Wangda Tan via asuresh)

[asuresh]

YARN-7522. Introduce AllocationTagsManager to associate allocation tags to 
nodes. (Wangda Tan via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/801c0988
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/801c0988
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/801c0988

Branch: refs/heads/YARN-6592
Commit: 801c0988b5ad1eff1e896a2635c2937721c96b04
Parents: 69de9a1
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Dec 8 00:24:00 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../resourcemanager/RMActiveServiceContext.java |  15 +
 .../yarn/server/resourcemanager/RMContext.java  |   5 +
 .../server/resourcemanager/RMContextImpl.java   |  12 +
 .../server/resourcemanager/ResourceManager.java |   9 +
 .../constraint/AllocationTagsManager.java   | 431 +++
 .../constraint/AllocationTagsNamespaces.java|  31 ++
 .../InvalidAllocationTagsQueryException.java|  35 ++
 .../rmcontainer/RMContainer.java|   8 +
 .../rmcontainer/RMContainerImpl.java|  21 +
 .../constraint/TestAllocationTagsManager.java   | 328 ++
 .../rmcontainer/TestRMContainerImpl.java| 124 ++
 .../scheduler/capacity/TestUtils.java   |   9 +
 .../scheduler/fifo/TestFifoScheduler.java   |   5 +
 13 files changed, 1033 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/801c0988/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 9dc5945..6ee3a4c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.event.Dispatcher;
 import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater;
 import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import 
org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
@@ -107,6 +108,7 @@ public class RMActiveServiceContext {
 
   private RMAppLifetimeMonitor rmAppLifetimeMonitor;
   private QueueLimitCalculator queueLimitCalculator;
+  private AllocationTagsManager allocationTagsManager;
 
   public RMActiveServiceContext() {
 queuePlacementManager = new PlacementManager();
@@ -398,6 +400,19 @@ public class RMActiveServiceContext {
 
   @Private
   @Unstable
+  public AllocationTagsManager getAllocationTagsManager() {
+return allocationTagsManager;
+  }
+
+  @Private
+  @Unstable
+  public void setAllocationTagsManager(
+  AllocationTagsManager allocationTagsManager) {
+this.allocationTagsManager = allocationTagsManager;
+  }
+
+  @Private
+  @Unstable
   public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() {
 return rmDelegatedNodeLabelsUpdater;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/801c0988/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index ec94030..62899d9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext

[23/37] hadoop git commit: YARN-6597. Add RMContainer recovery test to verify tag population in the AllocationTagsManager. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-6597. Add RMContainer recovery test to verify tag population in the 
AllocationTagsManager. (Panagiotis Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/add993e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/add993e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/add993e2

Branch: refs/heads/YARN-6592
Commit: add993e26a3c96f77dfd42086f186a139966019e
Parents: f8c5f5b
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 25 23:01:43 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../rmcontainer/RMContainerImpl.java|  8 +++
 .../rmcontainer/TestRMContainerImpl.java| 25 ++--
 2 files changed, 26 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/add993e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index a504221..541621b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -20,7 +20,6 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
 
 import java.util.Collections;
 import java.util.EnumSet;
-import java.util.List;
 import java.util.Set;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -40,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -533,7 +531,7 @@ public class RMContainerImpl implements RMContainer {
 RMContainerEvent event) {
   NMContainerStatus report =
   ((RMContainerRecoverEvent) event).getContainerReport();
-  // Set the allocation tags from the
+  // Set the allocation tags from the NMContainerStatus
   container.setAllocationTags(report.getAllocationTags());
   // Notify AllocationTagsManager
   container.rmContext.getAllocationTagsManager().addContainer(
@@ -689,7 +687,7 @@ public class RMContainerImpl implements RMContainer {
 // Something wrong happened, kill the container
 LOG.warn("Something wrong happened, container size reported by NM"
 + " is not expected, ContainerID=" + container.getContainerId()
-+ " rm-size-resource:" + rmContainerResource + " nm-size-reosurce:"
++ " rm-size-resource:" + rmContainerResource + " nm-size-resource:"
 + nmContainerResource);
 container.eventHandler.handle(new RMNodeCleanContainerEvent(
 container.nodeId, container.getContainerId()));
@@ -702,7 +700,7 @@ public class RMContainerImpl implements RMContainer {
 
 @Override
 public void transition(RMContainerImpl container, RMContainerEvent event) {
-  // Notify placementManager
+  // Notify AllocationTagsManager
   container.rmContext.getAllocationTagsManager().removeContainer(
   container.getNodeId(), container.getContainerId(),
   container.getAllocationTags());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/add993e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
 

[31/37] hadoop git commit: YARN-7670. Modifications to the ResourceScheduler API to support SchedulingRequests. (asuresh)

[asuresh]

YARN-7670. Modifications to the ResourceScheduler API to support 
SchedulingRequests. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88d8d3f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88d8d3f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88d8d3f4

Branch: refs/heads/YARN-6592
Commit: 88d8d3f40b2923fab23a933bce1cd2e9c320ae84
Parents: 801c098
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Dec 19 08:59:23 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../scheduler/AbstractYarnScheduler.java| 18 +
 .../scheduler/ResourceScheduler.java| 13 
 .../scheduler/capacity/CapacityScheduler.java   | 78 ++--
 .../common/ResourceAllocationCommitter.java | 12 ++-
 .../scheduler/common/fica/FiCaSchedulerApp.java | 30 +---
 .../TestCapacitySchedulerAsyncScheduling.java   | 10 +--
 6 files changed, 138 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/88d8d3f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 4b76327..213d784 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceOption;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerError;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -295,6 +296,10 @@ public abstract class AbstractYarnScheduler
 return nodeTracker.getNodes(nodeFilter);
   }
 
+  public List getNodes(final NodeFilter filter) {
+return nodeTracker.getNodes(filter);
+  }
+
   public boolean shouldContainersBeAutoUpdated() {
 return this.autoUpdateContainers;
   }
@@ -1443,4 +1448,17 @@ public abstract class AbstractYarnScheduler
   throw new IOException(e);
 }
   }
+
+  /**
+   * Default implementation. Always returns false.
+   * @param appAttempt ApplicationAttempt.
+   * @param schedulingRequest SchedulingRequest.
+   * @param schedulerNode SchedulerNode.
+   * @return Success or not.
+   */
+  @Override
+  public boolean attemptAllocationOnNode(SchedulerApplicationAttempt 
appAttempt,
+  SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) {
+return false;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/88d8d3f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
index d96d625..5a56ac7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
@@ -25,6 +25,7 @@ import 
org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.conf.Configurat

[01/37] hadoop git commit: YARN-7811. Fixed a bug in user defined docker network settings. (Contributed by Billie Rinaldi) [Forced Update!]

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-6592 6ae4cc995 -> 8df7666fe (forced update)


YARN-7811.  Fixed a bug in user defined docker network settings.  (Contributed 
by Billie Rinaldi)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9dd5b61
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9dd5b61
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9dd5b61

Branch: refs/heads/YARN-6592
Commit: f9dd5b61f4ed0288cc01cb1a676df8c9cd69cdd9
Parents: 901d15a
Author: Eric Yang 
Authored: Tue Jan 30 12:42:38 2018 -0500
Committer: Eric Yang 
Committed: Tue Jan 30 12:42:38 2018 -0500

--
 .../containerlaunch/AbstractLauncher.java   | 22 ++--
 .../service/provider/docker/DockerKeys.java |  7 ---
 .../provider/docker/DockerProviderService.java  |  2 +-
 3 files changed, 12 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9dd5b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
index e1e88cd..f497985 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -39,8 +39,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import static 
org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
-
 /**
  * Launcher of applications: base class
  */
@@ -60,9 +58,9 @@ public class AbstractLauncher {
   private final Map serviceData = new HashMap<>();
   protected boolean yarnDockerMode = false;
   protected String dockerImage;
-  protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
+  protected String dockerNetwork;
   protected String dockerHostname;
-  protected String runPrivilegedContainer;
+  protected boolean runPrivilegedContainer = false;
   private ServiceContext context;
 
   public AbstractLauncher(ServiceContext context) {
@@ -145,10 +143,16 @@ public class AbstractLauncher {
   Map env = containerLaunchContext.getEnvironment();
   env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
   env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);
-  env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", 
dockerNetwork);
+  if (ServiceUtils.isSet(dockerNetwork)) {
+env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
+dockerNetwork);
+  }
   env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME",
   dockerHostname);
-  env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", 
runPrivilegedContainer);
+  if (runPrivilegedContainer) {
+env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
+"true");
+  }
   StringBuilder sb = new StringBuilder();
   for (Entry mount : mountPaths.entrySet()) {
 if (sb.length() > 0) {
@@ -238,11 +242,7 @@ public class AbstractLauncher {
   }
 
   public void setRunPrivilegedContainer(boolean runPrivilegedContainer) {
-if (runPrivilegedContainer) {
-  this.runPrivilegedContainer = Boolean.toString(true);
-} else {
-  this.runPrivilegedContainer = Boolean.toString(false);
-}
+this.runPrivilegedContainer = runPrivilegedContainer;
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9dd5b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
 

[29/37] hadoop git commit: YARN-7745. Allow DistributedShell to take a placement specification for containers it wants to launch. (Arun Suresh via wangda)

[asuresh]

YARN-7745. Allow DistributedShell to take a placement specification for 
containers it wants to launch. (Arun Suresh via wangda)

Change-Id: Ided146d662e944a8a4692e5d6885f23fd9bbcad5


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e60f5129
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e60f5129
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e60f5129

Branch: refs/heads/YARN-6592
Commit: e60f51299dba360d13aa39f9ab714fdfc666b532
Parents: 38af237
Author: Wangda Tan 
Authored: Thu Jan 18 14:22:45 2018 -0800
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../distributedshell/ApplicationMaster.java | 124 +++--
 .../applications/distributedshell/Client.java   |  14 ++
 .../distributedshell/PlacementSpec.java | 137 +++
 3 files changed, 263 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e60f5129/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 270ef1b..9ba2138 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -42,6 +42,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
@@ -87,8 +88,11 @@ import 
org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.ProfileCapability;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.UpdatedContainer;
 import org.apache.hadoop.yarn.api.records.ExecutionType;
@@ -99,6 +103,7 @@ import 
org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
 import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.client.api.TimelineV2Client;
@@ -274,6 +279,10 @@ public class ApplicationMaster {
   @VisibleForTesting
   protected AtomicInteger numRequestedContainers = new AtomicInteger();
 
+  protected AtomicInteger numIgnore = new AtomicInteger();
+
+  protected AtomicInteger totalRetries = new AtomicInteger(10);
+
   // Shell command to be executed
   private String shellCommand = "";
   // Args to be passed to the shell command
@@ -289,6 +298,9 @@ public class ApplicationMaster {
   // File length needed for local resource
   private long shellScriptPathLen = 0;
 
+  // Placement Specifications
+  private Map placementSpecs = null;
+
   // Container retry options
   private ContainerRetryPolicy containerRetryPolicy =
   ContainerRetryPolicy.NEVER_RETRY;
@@ -334,6 +346,7 @@ public class ApplicationMaster {
   private final String windows_command = "cmd /c";
 
   private int yarnShellIdCounter = 1;
+  private final AtomicLong allocIdCounter = new AtomicLong(1);
 
   @VisibleForTesting
   protected final Set launchedContainers =
@@ -457,6 +470,7 @@ public class ApplicationMaster {
 "If container could retry, it specifies max 

[08/37] hadoop git commit: YARN-6619. AMRMClient Changes to use the PlacementConstraint and SchcedulingRequest objects. (Arun Suresh via wangda)

[asuresh]

YARN-6619. AMRMClient Changes to use the PlacementConstraint and 
SchcedulingRequest objects. (Arun Suresh via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29d9e4d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29d9e4d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29d9e4d5

Branch: refs/heads/YARN-6592
Commit: 29d9e4d5814900d5c59d77fe05d32186d4ad9385
Parents: a5c1fc8
Author: Wangda Tan 
Authored: Wed Jan 17 11:36:26 2018 -0800
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/client/api/AMRMClient.java  |  38 +++-
 .../yarn/client/api/async/AMRMClientAsync.java  |  48 +
 .../api/async/impl/AMRMClientAsyncImpl.java |  49 -
 .../yarn/client/api/impl/AMRMClientImpl.java| 142 -
 .../client/api/impl/BaseAMRMClientTest.java | 212 +++
 .../yarn/client/api/impl/TestAMRMClient.java| 156 +-
 .../TestAMRMClientPlacementConstraints.java | 204 ++
 .../rmcontainer/RMContainerImpl.java|   3 +
 .../scheduler/AbstractYarnScheduler.java|   1 +
 .../scheduler/SchedulerApplicationAttempt.java  |   1 +
 .../constraint/PlacementConstraintsUtil.java|   4 +-
 11 files changed, 700 insertions(+), 158 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d9e4d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index d3d1974..914a146 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.client.api;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Supplier;
 import java.util.List;
 
@@ -39,7 +41,9 @@ import 
org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.ProfileCapability;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
 import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.resource.Resources;
@@ -554,6 +558,18 @@ public abstract class AMRMClient extends
   }
 
   /**
+   * Add a Collection of SchedulingRequests. The AMRMClient will ensure that
+   * all requests in the same batch are sent in the same allocate call.
+   * @param schedulingRequests Collection of Scheduling Requests.
+   */
+  @Public
+  @InterfaceStability.Unstable
+  public void addSchedulingRequests(
+  Collection schedulingRequests) {
+
+  }
+
+  /**
* Register the application master. This must be called before any 
* other interaction
* @param appHostName Name of the host on which master is running
@@ -568,7 +584,27 @@ public abstract class AMRMClient extends
  int appHostPort,
  String appTrackingUrl) 
throws YarnException, IOException;
-  
+
+  /**
+   * Register the application master. This must be called before any
+   * other interaction
+   * @param appHostName Name of the host on which master is running
+   * @param appHostPort Port master is listening on
+   * @param appTrackingUrl URL at which the master info can be seen
+   * @param placementConstraints Placement Constraints mappings.
+   * @return RegisterApplicationMasterResponse
+   * @throws YarnException
+   * @throws IOException
+   */
+  @Public
+  @InterfaceStability.Unstable
+  public RegisterApplicationMasterResponse registerApplicationMaster(
+  String appHostName, int appHostPort, String appTrackingUrl,
+  Map placementConstraints)
+  throws YarnException, IOException {
+throw new YarnException("Not supported");
+  }
+
   /**
* Request additional containers and receive new container allocations.
* Requests made via addContainerRequest are sent to the

[32/37] hadoop git commit: YARN-7612. Add Processor Framework for Rich Placement Constraints. (asuresh)

[asuresh]

YARN-7612. Add Processor Framework for Rich Placement Constraints. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9af15d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9af15d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9af15d6

Branch: refs/heads/YARN-6592
Commit: f9af15d659f59fd0cf564fe1ecc8e06c6429ba68
Parents: 1efb2b6
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Dec 22 15:51:20 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  26 ++
 .../src/main/resources/yarn-default.xml |  30 ++
 .../ApplicationMasterService.java   |  15 +
 .../rmcontainer/RMContainerImpl.java|   7 +-
 .../scheduler/capacity/CapacityScheduler.java   |   2 +
 .../constraint/processor/BatchedRequests.java   | 105 +
 .../processor/NodeCandidateSelector.java|  38 ++
 .../processor/PlacementDispatcher.java  | 145 +++
 .../processor/PlacementProcessor.java   | 343 
 .../processor/SamplePlacementAlgorithm.java | 144 +++
 .../constraint/processor/package-info.java  |  29 ++
 .../yarn/server/resourcemanager/MockAM.java |  26 ++
 .../yarn/server/resourcemanager/MockRM.java |  14 +
 .../constraint/TestPlacementProcessor.java  | 394 +++
 14 files changed, 1316 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9af15d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index bbbfc52..8fb3c2e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -531,6 +531,32 @@ public class YarnConfiguration extends Configuration {
   /** The class to use as the resource scheduler.*/
   public static final String RM_SCHEDULER = 
 RM_PREFIX + "scheduler.class";
+
+  /** Placement Algorithm. */
+  public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS =
+  RM_PREFIX + "placement-constraints.algorithm.class";
+
+  public static final String RM_PLACEMENT_CONSTRAINTS_ENABLED =
+  RM_PREFIX + "placement-constraints.enabled";
+
+  public static final boolean DEFAULT_RM_PLACEMENT_CONSTRAINTS_ENABLED = true;
+
+  public static final String RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS =
+  RM_PREFIX + "placement-constraints.retry-attempts";
+
+  public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS = 3;
+
+  public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE =
+  RM_PREFIX + "placement-constraints.algorithm.pool-size";
+
+  public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE 
=
+  1;
+
+  public static final String RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE =
+  RM_PREFIX + "placement-constraints.scheduler.pool-size";
+
+  public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE 
=
+  1;
  
   public static final String DEFAULT_RM_SCHEDULER = 
   
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9af15d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0bb4fca..6d52ace 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -131,6 +131,36 @@
   
 
   
+Enable Constraint Placement.
+yarn.resourcemanager.placement-constraints.enabled
+false
+  
+
+  
+Number of times to retry placing of rejected 
SchedulingRequests
+yarn.resourcemanager.placement-constraints.retry-attempts
+3
+  
+
+  
+Constraint Placement Algorithm to be used.
+yarn.resourcemanager.placement-constraints.algorithm.class
+
org.apache.hadoop.y

[16/37] hadoop git commit: YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun Suresh via kkaranasos)

[asuresh]

YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun 
Suresh via kkaranasos)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adbe87ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adbe87ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adbe87ab

Branch: refs/heads/YARN-6592
Commit: adbe87abf8b2814e0e2988d09ef8a8569190c80e
Parents: 8bf7c44
Author: Konstantinos Karanasos 
Authored: Mon Jan 22 23:51:02 2018 -0800
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../constraint/AllocationTagsManager.java   | 110 +++-
 .../algorithm/DefaultPlacementAlgorithm.java|   8 +-
 .../algorithm/LocalAllocationTagsManager.java   | 167 +++
 .../constraint/TestAllocationTagsManager.java   |  82 -
 .../TestLocalAllocationTagsManager.java | 139 +++
 5 files changed, 336 insertions(+), 170 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/adbe87ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
index 962e548..7ad5e8c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
@@ -24,17 +24,14 @@ import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.log4j.Logger;
 
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -61,9 +58,6 @@ public class AllocationTagsManager {
   // Application's tags to Rack
   private Map perAppRackMappings =
   new HashMap<>();
-  // Application's Temporary containers mapping
-  private Map>>
-  appTempMappings = new HashMap<>();
 
   // Global tags to node mapping (used to fast return aggregated tags
   // cardinality across apps)
@@ -76,7 +70,7 @@ public class AllocationTagsManager {
* Currently used both for NodeId to Tag, Count and Rack to Tag, Count
*/
   @VisibleForTesting
-  static class TypeToCountedTags {
+  public static class TypeToCountedTags {
 // Map>
 private Map> typeToTagsWithCount = new HashMap<>();
 
@@ -214,7 +208,7 @@ public class AllocationTagsManager {
   }
 
   @VisibleForTesting
-  Map getPerAppNodeMappings() {
+  public Map getPerAppNodeMappings() {
 return perAppNodeMappings;
   }
 
@@ -233,12 +227,6 @@ public class AllocationTagsManager {
 return globalRackMapping;
   }
 
-  @VisibleForTesting
-  public Map> getAppTempMappings(
-  ApplicationId applicationId) {
-return appTempMappings.get(applicationId);
-  }
-
   public AllocationTagsManager(RMContext context) {
 ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
 readLock = lock.readLock();
@@ -246,39 +234,6 @@ public class AllocationTagsManager {
 rmContext = context;
   }
 
-  //
-
-  /**
-   * Method adds a temporary fake-container tag to Node mapping.
-   * Used by the constrained placement algorithm to keep track of containers
-   * that are currently placed on nodes but are not yet 

[28/37] hadoop git commit: YARN-7807. Assume intra-app anti-affinity as default for scheduling request inside AppPlacementAllocator. (Wangda Tan via asuresh)

[asuresh]

YARN-7807. Assume intra-app anti-affinity as default for scheduling request 
inside AppPlacementAllocator. (Wangda Tan via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/644afe5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/644afe5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/644afe5f

Branch: refs/heads/YARN-6592
Commit: 644afe5fd800ac4f2b873a99f9b3868c3a8c5c40
Parents: a4c539f
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Jan 24 12:55:01 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../placement/SingleConstraintAppPlacementAllocator.java| 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/644afe5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
index 9e7d71c..b02cb00 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
@@ -334,14 +334,15 @@ public class SingleConstraintAppPlacementAllocator
 targetAllocationTags = new HashSet<>(
 targetExpression.getTargetValues());
 
-if (targetExpression.getTargetKey() == null || !targetExpression
+if (targetExpression.getTargetKey() != null && !targetExpression
 .getTargetKey().equals(APPLICATION_LABEL_INTRA_APPLICATION)) {
   throwExceptionWithMetaInfo(
   "As of now, the only accepted target key for targetKey of "
   + "allocation_tag target expression is: ["
   + APPLICATION_LABEL_INTRA_APPLICATION
   + "]. Please make changes to placement constraints "
-  + "accordingly.");
+  + "accordingly. If this is null, it will be set to "
+  + APPLICATION_LABEL_INTRA_APPLICATION + " by default.");
 }
   }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[07/37] hadoop git commit: YARN-7681. Double-check placement constraints in scheduling phase before actual allocation is made. (Weiwei Yang via asuresh)

[asuresh]

YARN-7681. Double-check placement constraints in scheduling phase before actual 
allocation is made. (Weiwei Yang via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4eda58c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4eda58c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4eda58c1

Branch: refs/heads/YARN-6592
Commit: 4eda58c13641c14c4b248843a2589781cbcd343f
Parents: bdba01f
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Jan 10 09:04:30 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../scheduler/capacity/CapacityScheduler.java   | 23 
 1 file changed, 23 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eda58c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index e682d0f..d2713c8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -124,6 +124,8 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCo
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintsUtil;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
@@ -2574,6 +2576,27 @@ public class CapacityScheduler extends
 ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>
 resourceCommitRequest = createResourceCommitRequest(
 appAttempt, schedulingRequest, schedulerNode);
+
+// Validate placement constraint is satisfied before
+// committing the request.
+try {
+  if (!PlacementConstraintsUtil.canSatisfyConstraints(
+  appAttempt.getApplicationId(),
+  schedulingRequest.getAllocationTags(),
+  schedulerNode,
+  rmContext.getPlacementConstraintManager(),
+  rmContext.getAllocationTagsManager())) {
+LOG.debug("Failed to allocate container for application "
++ appAttempt.getApplicationId() + " on node "
++ schedulerNode.getNodeName()
++ " because this allocation violates the"
++ " placement constraint.");
+return false;
+  }
+} catch (InvalidAllocationTagsQueryException e) {
+  LOG.warn("Unable to allocate container", e);
+  return false;
+}
 return tryCommit(getClusterResource(), resourceCommitRequest, false);
   }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[12/37] hadoop git commit: YARN-6595. [API] Add Placement Constraints at the application level. (Arun Suresh via kkaranasos)

[asuresh]

YARN-6595. [API] Add Placement Constraints at the application level. (Arun 
Suresh via kkaranasos)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db928556
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db928556
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db928556

Branch: refs/heads/YARN-6592
Commit: db928556c81e5950b3fe374fa5b99ab26791ef3a
Parents: b57e8bc
Author: Konstantinos Karanasos 
Authored: Mon Nov 13 15:25:24 2017 -0800
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../RegisterApplicationMasterRequest.java   |  42 -
 .../yarn/api/resource/PlacementConstraint.java  | 156 +++
 .../src/main/proto/yarn_protos.proto|   6 +
 .../src/main/proto/yarn_service_protos.proto|   1 +
 .../RegisterApplicationMasterRequestPBImpl.java | 106 -
 .../hadoop/yarn/api/BasePBImplRecordsTest.java  |  11 ++
 6 files changed, 313 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/db928556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
index 395e190..f2d537a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
@@ -18,11 +18,16 @@
 
 package org.apache.hadoop.yarn.api.protocolrecords;
 
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
 import org.apache.hadoop.yarn.util.Records;
-
 /**
  * The request sent by the {@code ApplicationMaster} to {@code ResourceManager}
  * on registration.
@@ -132,4 +137,39 @@ public abstract class RegisterApplicationMasterRequest {
   @Public
   @Stable
   public abstract void setTrackingUrl(String trackingUrl);
+
+  /**
+   * Return all Placement Constraints specified at the Application level. The
+   * mapping is from a set of allocation tags to a
+   * PlacementConstraint associated with the tags, i.e., each
+   * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest} that has 
those
+   * tags will be placed taking into account the corresponding constraint.
+   *
+   * @return A map of Placement Constraints.
+   */
+  @Public
+  @Unstable
+  public Map getPlacementConstraints() {
+return new HashMap<>();
+  }
+
+  /**
+   * Set Placement Constraints applicable to the
+   * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}s
+   * of this application.
+   * The mapping is from a set of allocation tags to a
+   * PlacementConstraint associated with the tags.
+   * For example:
+   *  Map 
+   *   hb_regionserver - node_anti_affinity,
+   *   hb_regionserver, hb_master - rack_affinity,
+   *   ...
+   *  
+   * @param placementConstraints Placement Constraint Mapping.
+   */
+  @Public
+  @Unstable
+  public void setPlacementConstraints(
+  Map placementConstraints) {
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db928556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
index f0e3982..b6e851a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
@@ -54,6 +54,26 @@ public class PlacementConstraint {
 return constraintExpr;
   }
 
+  @Override
+  

[30/37] hadoop git commit: YARN-7774. Miscellaneous fixes to the PlacementProcessor. (asuresh)

[asuresh]

YARN-7774. Miscellaneous fixes to the PlacementProcessor. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28fe7f33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28fe7f33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28fe7f33

Branch: refs/heads/YARN-6592
Commit: 28fe7f331837b36e78fa34ed990993677dddeaee
Parents: e60f512
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 18 11:01:36 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../scheduler/SchedulerNode.java| 16 +++-
 .../scheduler/capacity/CapacityScheduler.java   |  4 +
 .../constraint/PlacementConstraintsUtil.java|  5 +-
 .../constraint/algorithm/CircularIterator.java  | 86 
 .../algorithm/DefaultPlacementAlgorithm.java| 50 ++--
 .../constraint/processor/BatchedRequests.java   |  8 ++
 .../SingleConstraintAppPlacementAllocator.java  |  2 +-
 .../yarn/server/resourcemanager/MockAM.java |  4 +-
 .../constraint/TestPlacementProcessor.java  | 24 +++---
 .../algorithm/TestCircularIterator.java | 84 +++
 ...stSingleConstraintAppPlacementAllocator.java | 28 +++
 11 files changed, 271 insertions(+), 40 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/28fe7f33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 89f748d..96a8e34 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -469,6 +469,20 @@ public abstract class SchedulerNode {
 this.lastHeartbeatMonotonicTime = Time.monotonicNow();
   }
 
+  @Override
+  public boolean equals(Object o) {
+if (this == o) return true;
+if (!(o instanceof SchedulerNode)) return false;
+
+SchedulerNode that = (SchedulerNode) o;
+
+return getNodeID().equals(that.getNodeID());
+  }
+
+  @Override
+  public int hashCode() {
+return getNodeID().hashCode();
+  }
 
   private static class ContainerInfo {
 private final RMContainer container;
@@ -479,4 +493,4 @@ public abstract class SchedulerNode {
   this.launchedOnNode = launchedOnNode;
 }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/28fe7f33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index c713036..429f9f3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2610,6 +2610,10 @@ public class CapacityScheduler extends
 " but only 1 will be attempted !!");
   }
   if (!appAttempt.isStopped()) {
+Resource resource =
+schedulingRequest.getResourceSizing().getResources();
+schedulingRequest.getResourceSizing().setResources(
+getNormalizedResource(resource));
 ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>
 resourceCommitRequest = createResourceCommitRequest(
 appAttempt, schedulingRequest, schedulerNode);

http://git-wip-us.apache.org/repos/asf/hado

[05/37] hadoop git commit: YARN-7779. Display allocation tags in RM web UI and expose same through REST API. Contributed by Weiwei Yang.

[asuresh]

YARN-7779. Display allocation tags in RM web UI and expose same through REST 
API. Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b81cb05
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b81cb05
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b81cb05

Branch: refs/heads/YARN-6592
Commit: 9b81cb0537e5b731581e6a375bf0a59abf61c359
Parents: adbe87a
Author: Sunil G 
Authored: Tue Jan 23 17:09:58 2018 +0530
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../hadoop/yarn/sls/nodemanager/NodeInfo.java   |  6 ++
 .../yarn/sls/scheduler/RMNodeWrapper.java   |  6 ++
 .../server/resourcemanager/rmnode/RMNode.java   |  7 ++
 .../resourcemanager/rmnode/RMNodeImpl.java  |  6 ++
 .../constraint/AllocationTagsManager.java   | 11 +++
 .../resourcemanager/webapp/NodesPage.java   |  3 +
 .../webapp/dao/AllocationTagInfo.java   | 56 ++
 .../webapp/dao/AllocationTagsInfo.java  | 59 +++
 .../resourcemanager/webapp/dao/NodeInfo.java| 15 
 .../yarn/server/resourcemanager/MockNodes.java  |  6 ++
 .../resourcemanager/webapp/TestNodesPage.java   |  4 +-
 .../webapp/TestRMWebServicesNodes.java  | 77 +++-
 12 files changed, 253 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 1016ce1..0c99139 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.sls.nodemanager;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -213,6 +214,11 @@ public class NodeInfo {
 }
 
 @Override
+public Map getAllocationTagsWithCount() {
+  return null;
+}
+
+@Override
 public Resource getPhysicalResource() {
   return null;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index fdad826..92f9b0f 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 @Private
@@ -203,6 +204,11 @@ public class RMNodeWrapper implements RMNode {
   }
 
   @Override
+  public Map getAllocationTagsWithCount() {
+return node.getAllocationTagsWithCount();
+  }
+
+  @Override
   public Resource getPhysicalResource() {
 return null;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index a5615ef..872f2a6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
 
 
 import java.util.List;
+import java.util.Map;
 import 

[35/37] hadoop git commit: YARN-7802. [UI2] Application regex search did not work properly with app name. Contributed by Sreenath Somarajapuram.

[asuresh]

YARN-7802. [UI2] Application regex search did not work properly with app name. 
Contributed by  Sreenath Somarajapuram.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64344c34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64344c34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64344c34

Branch: refs/heads/YARN-6592
Commit: 64344c345d69bcdcf72a08cec326e6d1a5c25fab
Parents: 5206b2c
Author: Sunil G 
Authored: Wed Jan 31 17:40:36 2018 +0530
Committer: Sunil G 
Committed: Wed Jan 31 17:40:36 2018 +0530

--
 .../src/main/webapp/app/components/timeline-view.js| 5 +
 .../src/main/webapp/app/controllers/yarn-app/components.js | 5 +
 .../src/main/webapp/app/controllers/yarn-apps/apps.js  | 1 +
 .../webapp/app/controllers/yarn-component-instances/info.js| 5 +
 .../src/main/webapp/app/controllers/yarn-flowrun/info.js   | 5 +
 .../src/main/webapp/app/controllers/yarn-flowrun/metrics.js| 5 +
 .../src/main/webapp/app/controllers/yarn-nodes/table.js| 4 +++-
 .../src/main/webapp/app/controllers/yarn-queue/apps.js | 1 +
 .../src/main/webapp/app/controllers/yarn-services.js   | 1 +
 .../src/main/webapp/app/controllers/yarn-tools/yarn-conf.js| 4 
 .../src/main/webapp/app/templates/components/timeline-view.hbs | 2 +-
 .../src/main/webapp/app/templates/yarn-app/components.hbs  | 2 +-
 .../webapp/app/templates/yarn-component-instances/info.hbs | 2 +-
 .../src/main/webapp/app/templates/yarn-flowrun/info.hbs| 2 +-
 .../src/main/webapp/app/templates/yarn-flowrun/metrics.hbs | 6 +++---
 .../src/main/webapp/app/templates/yarn-tools/yarn-conf.hbs | 6 +++---
 .../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json| 6 +++---
 .../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock   | 6 +++---
 18 files changed, 51 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/64344c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js
index e23d991..65a8cb1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js
@@ -19,8 +19,13 @@
 import Ember from 'ember';
 import Converter from 'yarn-ui/utils/converter';
 import ColumnDef from 'em-table/utils/column-definition';
+import TableDefinition from 'em-table/utils/table-definition';
 
 export default Ember.Component.extend({
+  tableDefinition: TableDefinition.create({
+searchType: 'manual',
+  }),
+
   canvas: {
 svg: undefined,
 h: 0,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/64344c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
index 812f1e6..5981eb5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
@@ -18,11 +18,16 @@
 
 import Ember from 'ember';
 import ColumnDef from 'em-table/utils/column-definition';
+import TableDefinition from 'em-table/utils/table-definition';
 
 export default Ember.Controller.extend({
   queryParams: ["service"],
   service: undefined,
 
+  tableDefinition: TableDefinition.create({
+searchType: 'manual',
+  }),
+
   tableColumns: Ember.computed('model.appId', 'model.serviceName', function() {
 var cols = [];
 var service = this.get('model.serviceName');

http://git-wip-us.apache.org/repos/asf/hadoop/blob/64344c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
index bfd6299..55d5a88 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
+++ 

[18/37] hadoop git commit: YARN-7669. API and interface modifications for placement constraint processor. (asuresh)

[asuresh]

YARN-7669. API and interface modifications for placement constraint processor. 
(asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06eb63e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06eb63e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06eb63e6

Branch: refs/heads/YARN-6592
Commit: 06eb63e64b05e2e8bb8a76c15360ab0495f11317
Parents: 88d8d3f
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Dec 19 22:47:46 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../yarn/ams/ApplicationMasterServiceUtils.java |  16 +
 .../api/protocolrecords/AllocateResponse.java   |  23 +
 .../api/records/RejectedSchedulingRequest.java  |  70 +++
 .../yarn/api/records/RejectionReason.java   |  44 ++
 .../src/main/proto/yarn_protos.proto|  10 +
 .../src/main/proto/yarn_service_protos.proto|   1 +
 .../impl/pb/AllocateResponsePBImpl.java |  85 
 .../yarn/api/records/impl/pb/ProtoUtils.java|  16 +
 .../pb/RejectedSchedulingRequestPBImpl.java | 148 +++
 .../records/impl/pb/ResourceSizingPBImpl.java   |   8 +
 .../impl/pb/SchedulingRequestPBImpl.java|  11 +
 .../hadoop/yarn/api/TestPBImplRecords.java  |   2 +
 .../resourcemanager/RMActiveServiceContext.java |   2 +-
 .../yarn/server/resourcemanager/RMContext.java  |   2 +-
 .../server/resourcemanager/RMContextImpl.java   |   2 +-
 .../server/resourcemanager/ResourceManager.java |   2 +-
 .../constraint/AllocationTagsManager.java   | 431 ---
 .../constraint/AllocationTagsNamespaces.java|  31 --
 .../InvalidAllocationTagsQueryException.java|  35 --
 .../constraint/AllocationTagsManager.java   | 431 +++
 .../constraint/AllocationTagsNamespaces.java|  31 ++
 .../InvalidAllocationTagsQueryException.java|  35 ++
 .../api/ConstraintPlacementAlgorithm.java   |  43 ++
 .../api/ConstraintPlacementAlgorithmInput.java  |  32 ++
 .../api/ConstraintPlacementAlgorithmOutput.java |  58 +++
 ...traintPlacementAlgorithmOutputCollector.java |  32 ++
 .../constraint/api/PlacedSchedulingRequest.java |  79 
 .../constraint/api/SchedulingResponse.java  |  70 +++
 .../scheduler/constraint/api/package-info.java  |  28 ++
 .../constraint/TestAllocationTagsManager.java   | 328 --
 .../rmcontainer/TestRMContainerImpl.java|   2 +-
 .../scheduler/capacity/TestUtils.java   |   2 +-
 .../constraint/TestAllocationTagsManager.java   | 328 ++
 .../scheduler/fifo/TestFifoScheduler.java   |   2 +-
 34 files changed, 1608 insertions(+), 832 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
index 476da8b..8bdfaf3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.ams;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerError;
 import org.apache.hadoop.yarn.api.records.UpdatedContainer;
 
@@ -86,4 +87,19 @@ public final class ApplicationMasterServiceUtils {
 }
 allocateResponse.setAllocatedContainers(allocatedContainers);
   }
+
+  /**
+   * Add rejected Scheduling Requests to {@link AllocateResponse}.
+   * @param allocateResponse Allocate Response.
+   * @param rejectedRequests Rejected SchedulingRequests.
+   */
+  public static void addToRejectedSchedulingRequests(
+  AllocateResponse allocateResponse,
+  List rejectedRequests) {
+if (allocateResponse.getRejectedSchedulingRequests() != null
+&& !allocateResponse.getRejectedSchedulingRequests().isEmpty()) {
+  
rejectedRequests.addAll(allocateResponse.getRejectedSchedulingRequests());
+}
+allocateResponse.setRejectedSchedulingRequests(rejectedRequests);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn

[09/37] hadoop git commit: YARN-6593. [API] Introduce Placement Constraint object. (Konstantinos Karanasos via wangda)

[asuresh]

YARN-6593. [API] Introduce Placement Constraint object. (Konstantinos Karanasos 
via wangda)

Change-Id: Id00edb7185fdf01cce6e40f920cac3585f8cbe9c


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33a796d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33a796d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33a796d9

Branch: refs/heads/YARN-6592
Commit: 33a796d9b778bf7350e87a4e36ca30c925cf7036
Parents: 1453a04
Author: Wangda Tan 
Authored: Thu Aug 3 14:03:55 2017 -0700
Committer: Arun Suresh 
Committed: Wed Jan 31 01:30:17 2018 -0800

--
 .../yarn/api/resource/PlacementConstraint.java  | 567 +++
 .../yarn/api/resource/PlacementConstraints.java | 286 ++
 .../hadoop/yarn/api/resource/package-info.java  |  23 +
 .../src/main/proto/yarn_protos.proto|  55 ++
 .../api/resource/TestPlacementConstraints.java  | 106 
 .../PlacementConstraintFromProtoConverter.java  | 116 
 .../pb/PlacementConstraintToProtoConverter.java | 174 ++
 .../apache/hadoop/yarn/api/pb/package-info.java |  23 +
 .../yarn/api/records/impl/pb/ProtoUtils.java|  27 +
 .../PlacementConstraintTransformations.java | 209 +++
 .../hadoop/yarn/api/resource/package-info.java  |  23 +
 .../TestPlacementConstraintPBConversion.java| 195 +++
 .../TestPlacementConstraintTransformations.java | 183 ++
 13 files changed, 1987 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/33a796d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
new file mode 100644
index 000..f0e3982
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
@@ -0,0 +1,567 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * {@code PlacementConstraint} represents a placement constraint for a resource
+ * allocation.
+ */
+@Public
+@Unstable
+public class PlacementConstraint {
+
+  /**
+   * The constraint expression tree.
+   */
+  private AbstractConstraint constraintExpr;
+
+  public PlacementConstraint(AbstractConstraint constraintExpr) {
+this.constraintExpr = constraintExpr;
+  }
+
+  /**
+   * Get the constraint expression of the placement constraint.
+   *
+   * @return the constraint expression
+   */
+  public AbstractConstraint getConstraintExpr() {
+return constraintExpr;
+  }
+
+  /**
+   * Interface used to enable the elements of the constraint tree to be 
visited.
+   */
+  @Private
+  public interface Visitable {
+/**
+ * Visitor pattern.
+ *
+ * @param visitor visitor to be used
+ * @param  defines the type that the visitor will use and the return 
type
+ *  of the accept.
+ * @return the result of visiting a given object.
+ */
+ T accept(Visitor visitor);
+
+  }
+
+  /**
+   * Visitor API for a constraint tree.
+   *
+   * @param  determines the return type of the visit methods.
+   */
+  @Private
+  public interface Visitor {
+T visit(SingleConstraint constraint);
+
+T visit(TargetExpression target);
+
+T visit(TargetConstraint constraint);
+
+T visit(CardinalityConstraint constraint);
+
+T visit(And 

hadoop git commit: YARN-7822. Constraint satisfaction checker support for composite OR and AND constraints. (Weiwei Yang via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-7812 [created] e6d2d26a1


YARN-7822. Constraint satisfaction checker support for composite OR and AND 
constraints. (Weiwei Yang via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6d2d26a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6d2d26a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6d2d26a

Branch: refs/heads/YARN-7812
Commit: e6d2d26a13337adc995efa7bcb77181871930796
Parents: 6ae4cc9
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Jan 30 10:15:33 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Tue Jan 30 10:15:33 2018 -0800

--
 .../TestPlacementConstraintTransformations.java |   2 +-
 .../constraint/PlacementConstraintsUtil.java|  53 +++-
 .../TestPlacementConstraintsUtil.java   | 278 ---
 .../constraint/TestPlacementProcessor.java  | 159 ++-
 4 files changed, 444 insertions(+), 48 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6d2d26a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
index 62da092..aa92d7a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
@@ -156,7 +156,7 @@ public class TestPlacementConstraintTransformations {
 SingleConstraintTransformer singleTransformer =
 new SingleConstraintTransformer(specConstraint);
 PlacementConstraint simConstraint = singleTransformer.transform();
-Assert.assertTrue(constraintExpr instanceof Or);
+Assert.assertTrue(simConstraint.getConstraintExpr() instanceof Or);
 Or simOrExpr = (Or) specConstraint.getConstraintExpr();
 for (AbstractConstraint child : simOrExpr.getChildren()) {
   Assert.assertTrue(child instanceof SingleConstraint);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6d2d26a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
index 199dd62..6396e57 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType;
@@ -149,6 +151,48 @@ public final class PlacementConstraintsUtil {
 return true;
   }
 
+  /**
+   * Returns true if all child constraints are satisfied.
+   * @param appId application id
+   * @param constraint Or constraint
+   * @param node node
+   * @param atm allocation tags manager
+   * @return true if all child constraints are satisfied, false otherwise
+   * @throws InvalidAllocationTagsQueryException
+   */
+  private st

[40/50] [abbrv] hadoop git commit: YARN-7681. Double-check placement constraints in scheduling phase before actual allocation is made. (Weiwei Yang via asuresh)

[asuresh]

YARN-7681. Double-check placement constraints in scheduling phase before actual 
allocation is made. (Weiwei Yang via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30b8d4fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30b8d4fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30b8d4fc

Branch: refs/heads/YARN-6592
Commit: 30b8d4fc57bf9d25ef9093bb6c7ef5b2868c3602
Parents: 5df2556
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Jan 10 09:04:30 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Tue Jan 30 07:53:34 2018 -0800

--
 .../scheduler/capacity/CapacityScheduler.java   | 23 
 1 file changed, 23 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/30b8d4fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index e682d0f..d2713c8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -124,6 +124,8 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCo
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintsUtil;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
@@ -2574,6 +2576,27 @@ public class CapacityScheduler extends
 ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>
 resourceCommitRequest = createResourceCommitRequest(
 appAttempt, schedulingRequest, schedulerNode);
+
+// Validate placement constraint is satisfied before
+// committing the request.
+try {
+  if (!PlacementConstraintsUtil.canSatisfyConstraints(
+  appAttempt.getApplicationId(),
+  schedulingRequest.getAllocationTags(),
+  schedulerNode,
+  rmContext.getPlacementConstraintManager(),
+  rmContext.getAllocationTagsManager())) {
+LOG.debug("Failed to allocate container for application "
++ appAttempt.getApplicationId() + " on node "
++ schedulerNode.getNodeName()
++ " because this allocation violates the"
++ " placement constraint.");
+return false;
+  }
+} catch (InvalidAllocationTagsQueryException e) {
+  LOG.warn("Unable to allocate container", e);
+  return false;
+}
 return tryCommit(getClusterResource(), resourceCommitRequest, false);
   }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[33/50] [abbrv] hadoop git commit: YARN-7670. Modifications to the ResourceScheduler API to support SchedulingRequests. (asuresh)

[asuresh]

YARN-7670. Modifications to the ResourceScheduler API to support 
SchedulingRequests. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f9139cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f9139cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f9139cc

Branch: refs/heads/YARN-6592
Commit: 9f9139ccad3de707d6c5a1bc4664794468dcb1dd
Parents: 2615da8
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Dec 19 08:59:23 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Tue Jan 30 07:53:34 2018 -0800

--
 .../scheduler/AbstractYarnScheduler.java| 18 +
 .../scheduler/ResourceScheduler.java| 13 
 .../scheduler/capacity/CapacityScheduler.java   | 78 ++--
 .../common/ResourceAllocationCommitter.java | 12 ++-
 .../scheduler/common/fica/FiCaSchedulerApp.java | 30 +---
 .../TestCapacitySchedulerAsyncScheduling.java   | 10 +--
 6 files changed, 138 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f9139cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 4b76327..213d784 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceOption;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerError;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -295,6 +296,10 @@ public abstract class AbstractYarnScheduler
 return nodeTracker.getNodes(nodeFilter);
   }
 
+  public List getNodes(final NodeFilter filter) {
+return nodeTracker.getNodes(filter);
+  }
+
   public boolean shouldContainersBeAutoUpdated() {
 return this.autoUpdateContainers;
   }
@@ -1443,4 +1448,17 @@ public abstract class AbstractYarnScheduler
   throw new IOException(e);
 }
   }
+
+  /**
+   * Default implementation. Always returns false.
+   * @param appAttempt ApplicationAttempt.
+   * @param schedulingRequest SchedulingRequest.
+   * @param schedulerNode SchedulerNode.
+   * @return Success or not.
+   */
+  @Override
+  public boolean attemptAllocationOnNode(SchedulerApplicationAttempt 
appAttempt,
+  SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) {
+return false;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f9139cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
index d96d625..5a56ac7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
@@ -25,6 +25,7 @@ import 
org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.conf.Configurat

[45/50] [abbrv] hadoop git commit: YARN-6597. Add RMContainer recovery test to verify tag population in the AllocationTagsManager. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-6597. Add RMContainer recovery test to verify tag population in the 
AllocationTagsManager. (Panagiotis Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06d22eb2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06d22eb2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06d22eb2

Branch: refs/heads/YARN-6592
Commit: 06d22eb2784d4e2386b50926286409adb8b47973
Parents: 3663239
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 25 23:01:43 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Tue Jan 30 07:54:37 2018 -0800

--
 .../rmcontainer/RMContainerImpl.java|  8 +++
 .../rmcontainer/TestRMContainerImpl.java| 25 ++--
 2 files changed, 26 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/06d22eb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index a504221..541621b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -20,7 +20,6 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
 
 import java.util.Collections;
 import java.util.EnumSet;
-import java.util.List;
 import java.util.Set;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -40,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -533,7 +531,7 @@ public class RMContainerImpl implements RMContainer {
 RMContainerEvent event) {
   NMContainerStatus report =
   ((RMContainerRecoverEvent) event).getContainerReport();
-  // Set the allocation tags from the
+  // Set the allocation tags from the NMContainerStatus
   container.setAllocationTags(report.getAllocationTags());
   // Notify AllocationTagsManager
   container.rmContext.getAllocationTagsManager().addContainer(
@@ -689,7 +687,7 @@ public class RMContainerImpl implements RMContainer {
 // Something wrong happened, kill the container
 LOG.warn("Something wrong happened, container size reported by NM"
 + " is not expected, ContainerID=" + container.getContainerId()
-+ " rm-size-resource:" + rmContainerResource + " nm-size-reosurce:"
++ " rm-size-resource:" + rmContainerResource + " nm-size-resource:"
 + nmContainerResource);
 container.eventHandler.handle(new RMNodeCleanContainerEvent(
 container.nodeId, container.getContainerId()));
@@ -702,7 +700,7 @@ public class RMContainerImpl implements RMContainer {
 
 @Override
 public void transition(RMContainerImpl container, RMContainerEvent event) {
-  // Notify placementManager
+  // Notify AllocationTagsManager
   container.rmContext.getAllocationTagsManager().removeContainer(
   container.getNodeId(), container.getContainerId(),
   container.getAllocationTags());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06d22eb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
 

[46/50] [abbrv] hadoop git commit: YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun Suresh via kkaranasos)

[asuresh]

YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun 
Suresh via kkaranasos)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fca7371f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fca7371f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fca7371f

Branch: refs/heads/YARN-6592
Commit: fca7371f48cbd1faad6d06287b89a2526a0ff9d3
Parents: 3803dea
Author: Konstantinos Karanasos 
Authored: Mon Jan 22 23:51:02 2018 -0800
Committer: Arun Suresh 
Committed: Tue Jan 30 07:54:37 2018 -0800

--
 .../constraint/AllocationTagsManager.java   | 110 +++-
 .../algorithm/DefaultPlacementAlgorithm.java|   8 +-
 .../algorithm/LocalAllocationTagsManager.java   | 167 +++
 .../constraint/TestAllocationTagsManager.java   |  82 -
 .../TestLocalAllocationTagsManager.java | 139 +++
 5 files changed, 336 insertions(+), 170 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fca7371f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
index 962e548..7ad5e8c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
@@ -24,17 +24,14 @@ import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.log4j.Logger;
 
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -61,9 +58,6 @@ public class AllocationTagsManager {
   // Application's tags to Rack
   private Map perAppRackMappings =
   new HashMap<>();
-  // Application's Temporary containers mapping
-  private Map>>
-  appTempMappings = new HashMap<>();
 
   // Global tags to node mapping (used to fast return aggregated tags
   // cardinality across apps)
@@ -76,7 +70,7 @@ public class AllocationTagsManager {
* Currently used both for NodeId to Tag, Count and Rack to Tag, Count
*/
   @VisibleForTesting
-  static class TypeToCountedTags {
+  public static class TypeToCountedTags {
 // Map>
 private Map> typeToTagsWithCount = new HashMap<>();
 
@@ -214,7 +208,7 @@ public class AllocationTagsManager {
   }
 
   @VisibleForTesting
-  Map getPerAppNodeMappings() {
+  public Map getPerAppNodeMappings() {
 return perAppNodeMappings;
   }
 
@@ -233,12 +227,6 @@ public class AllocationTagsManager {
 return globalRackMapping;
   }
 
-  @VisibleForTesting
-  public Map> getAppTempMappings(
-  ApplicationId applicationId) {
-return appTempMappings.get(applicationId);
-  }
-
   public AllocationTagsManager(RMContext context) {
 ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
 readLock = lock.readLock();
@@ -246,39 +234,6 @@ public class AllocationTagsManager {
 rmContext = context;
   }
 
-  //
-
-  /**
-   * Method adds a temporary fake-container tag to Node mapping.
-   * Used by the constrained placement algorithm to keep track of containers
-   * that are currently placed on nodes but are not yet 

[34/50] [abbrv] hadoop git commit: YARN-7669. API and interface modifications for placement constraint processor. (asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c83ef6f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
new file mode 100644
index 000..9571f0e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Encapsulates the output of the ConstraintPlacementAlgorithm. The Algorithm
+ * is free to produce multiple of output objects at the end of each run and it
+ * must use the provided ConstraintPlacementAlgorithmOutputCollector to
+ * aggregate/collect this output. Similar to the MapReduce Mapper/Reducer
+ * which is provided a collector to collect output.
+ */
+public class ConstraintPlacementAlgorithmOutput {
+
+  private final ApplicationId applicationId;
+
+  public ConstraintPlacementAlgorithmOutput(ApplicationId applicationId) {
+this.applicationId = applicationId;
+  }
+
+  private final List placedRequests =
+  new ArrayList<>();
+
+  private final List rejectedRequests =
+  new ArrayList<>();
+
+  public List getPlacedRequests() {
+return placedRequests;
+  }
+
+  public List getRejectedRequests() {
+return rejectedRequests;
+  }
+
+  public ApplicationId getApplicationId() {
+return applicationId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c83ef6f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
new file mode 100644
index 000..131fd42
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing 

[47/50] [abbrv] hadoop git commit: YARN-7783. Add validation step to ensure constraints are not violated due to order in which a request is processed. (asuresh)

[asuresh]

YARN-7783. Add validation step to ensure constraints are not violated due to 
order in which a request is processed. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d04ec492
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d04ec492
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d04ec492

Branch: refs/heads/YARN-6592
Commit: d04ec492f2d9e8a7d6ad1490ff6e13cf07d8fd7c
Parents: 416f2aa
Author: Arun Suresh <asur...@apache.org>
Authored: Tue Jan 23 08:15:58 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Tue Jan 30 07:54:37 2018 -0800

--
 .../algorithm/DefaultPlacementAlgorithm.java| 119 +--
 .../constraint/TestPlacementProcessor.java  |  49 
 2 files changed, 155 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04ec492/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
index 9887749..4e6473f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
@@ -22,6 +22,7 @@ import java.util.Iterator;
 import java.util.List;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
@@ -69,13 +70,9 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
   public boolean attemptPlacementOnNode(ApplicationId appId,
   SchedulingRequest schedulingRequest, SchedulerNode schedulerNode)
   throws InvalidAllocationTagsQueryException {
-int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations();
-if (numAllocs > 0) {
-  if (PlacementConstraintsUtil.canSatisfyConstraints(appId,
-  schedulingRequest, schedulerNode,
-  constraintManager, tagsManager)) {
-return true;
-  }
+if (PlacementConstraintsUtil.canSatisfyConstraints(appId,
+schedulingRequest, schedulerNode, constraintManager, tagsManager)) {
+  return true;
 }
 return false;
   }
@@ -93,6 +90,9 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
 int rePlacementCount = RE_ATTEMPT_COUNT;
 while (rePlacementCount > 0) {
   doPlacement(requests, resp, allNodes, rejectedRequests);
+  // Double check if placement constraints are really satisfied
+  validatePlacement(requests.getApplicationId(), resp,
+  rejectedRequests);
   if (rejectedRequests.size() == 0 || rePlacementCount == 1) {
 break;
   }
@@ -122,9 +122,14 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
 break;
   }
   SchedulingRequest schedulingRequest = requestIterator.next();
+  PlacedSchedulingRequest placedReq =
+  new PlacedSchedulingRequest(schedulingRequest);
+  placedReq.setPlacementAttempt(requests.getPlacementAttempt());
+  resp.getPlacedRequests().add(placedReq);
   CircularIterator nodeIter =
   new CircularIterator(lastSatisfiedNode, nIter, allNodes);
-  int numAllocs = 
schedulingRequest.getResourceSizing().getNumAllocations();
+  int numAllocs =
+  schedulingRequest.getResourceSizing().getNumAllocations();
   while (nodeIter.hasNext() && numAllocs > 0) {
 SchedulerNode node = nodeIter.next();
 try {
@@ -135,11 +140,7 @@ public class DefaultPlacementAlgorithm implements 
ConstraintPlacementAlgorithm {
   requests.getApplicationId(), schedulingRequest, node)) {
 schedulingRequest.getResourceSizing()
 .

[36/50] [abbrv] hadoop git commit: YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos Karanasos via asuresh)

[asuresh]

YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos 
Karanasos via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80af031d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80af031d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80af031d

Branch: refs/heads/YARN-6592
Commit: 80af031dccf5b264949d5a996d40ed993f0e21c3
Parents: e724972
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Dec 22 13:26:30 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Tue Jan 30 07:53:34 2018 -0800

--
 .../resourcemanager/RMActiveServiceContext.java |  15 +
 .../yarn/server/resourcemanager/RMContext.java  |   6 +
 .../server/resourcemanager/RMContextImpl.java   |  13 +
 .../server/resourcemanager/ResourceManager.java |  13 +
 .../MemoryPlacementConstraintManager.java   | 282 +++
 .../constraint/PlacementConstraintManager.java  | 151 ++
 .../PlacementConstraintManagerService.java  |  93 ++
 .../scheduler/constraint/package-info.java  |  29 ++
 .../TestPlacementConstraintManagerService.java  | 182 
 9 files changed, 784 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/80af031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 4d0c230..06a1d00 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -43,6 +43,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator;
 import 
org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
 import 
org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
@@ -109,6 +110,7 @@ public class RMActiveServiceContext {
   private RMAppLifetimeMonitor rmAppLifetimeMonitor;
   private QueueLimitCalculator queueLimitCalculator;
   private AllocationTagsManager allocationTagsManager;
+  private PlacementConstraintManager placementConstraintManager;
 
   public RMActiveServiceContext() {
 queuePlacementManager = new PlacementManager();
@@ -413,6 +415,19 @@ public class RMActiveServiceContext {
 
   @Private
   @Unstable
+  public PlacementConstraintManager getPlacementConstraintManager() {
+return placementConstraintManager;
+  }
+
+  @Private
+  @Unstable
+  public void setPlacementConstraintManager(
+  PlacementConstraintManager placementConstraintManager) {
+this.placementConstraintManager = placementConstraintManager;
+  }
+
+  @Private
+  @Unstable
   public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() {
 return rmDelegatedNodeLabelsUpdater;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/80af031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 00da108..eb91a31 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop

[48/50] [abbrv] hadoop git commit: YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh)

[asuresh]

YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c6644fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c6644fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c6644fe

Branch: refs/heads/YARN-6592
Commit: 7c6644fedf8243e907d26e2d3d2194ab13fbf213
Parents: c7cee3e
Author: Arun Suresh <asur...@apache.org>
Authored: Wed Jan 24 14:18:32 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Tue Jan 30 07:54:37 2018 -0800

--
 .../hadoop/yarn/api/protocolrecords/AllocateRequest.java | 4 ++--
 .../apache/hadoop/yarn/api/records/SchedulingRequest.java| 8 
 .../apache/hadoop/yarn/api/resource/PlacementConstraint.java | 3 +++
 .../hadoop/yarn/api/resource/PlacementConstraints.java   | 3 ---
 .../yarn/api/records/impl/pb/ResourceSizingPBImpl.java   | 3 +++
 .../yarn/api/records/impl/pb/SchedulingRequestPBImpl.java| 3 +++
 .../resourcemanager/scheduler/AbstractYarnScheduler.java | 1 -
 .../server/resourcemanager/scheduler/AppSchedulingInfo.java  | 3 +--
 .../yarn/server/resourcemanager/scheduler/SchedulerNode.java | 8 ++--
 .../scheduler/capacity/CapacityScheduler.java| 3 +--
 .../scheduler/constraint/AllocationTagsManager.java  | 8 +---
 .../scheduler/constraint/PlacementConstraintsUtil.java   | 4 ++--
 .../scheduler/placement/AppPlacementAllocator.java   | 4 ++--
 .../placement/SingleConstraintAppPlacementAllocator.java | 1 -
 .../scheduler/capacity/TestCapacityScheduler.java| 1 -
 .../TestCapacitySchedulerSchedulingRequestUpdate.java| 4 +++-
 .../capacity/TestSchedulingRequestContainerAllocation.java   | 8 
 .../TestSchedulingRequestContainerAllocationAsync.java   | 1 -
 18 files changed, 35 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c6644fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
index d8d2347..876957e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
@@ -229,8 +229,8 @@ public abstract class AllocateRequest {
   /**
* Set the list of Scheduling requests to inform the
* ResourceManager about the application's resource requirements
-   * (potentially including allocation tags & placement constraints).
-   * @param schedulingRequests list of SchedulingRequest to update
+   * (potentially including allocation tags and placement constraints).
+   * @param schedulingRequests list of {@link SchedulingRequest} to update
*  the ResourceManager about the application's resource
*  requirements.
*/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c6644fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
index e32dd24..4bb2b84 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
@@ -77,7 +77,7 @@ public abstract class SchedulingRequest {
 
 /**
  * Set the allocationRequestId of the request.
- * 
+ *
  * @see SchedulingRequest#setAllocationRequestId(long)
  * @param allocationRequestId allocationRequestId of the
  *  request
@@ -107,7 +107,7 @@ public abstract class SchedulingRequest {
 
 /**
  * Set the executionType of the request.
- * 
+ *
  * @see SchedulingRequest#setExecutionType(ExecutionTypeRequest)
  * @param executionType executionType of the request
  * @return {@link SchedulingRequest.SchedulingRequestBuilder}
@@ -119,7 +119,7 @@ public abstract class SchedulingRequest {
   schedulingRequest.

[42/50] [abbrv] hadoop git commit: YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh)

[asuresh]

YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3663239d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3663239d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3663239d

Branch: refs/heads/YARN-6592
Commit: 3663239d8726a435bd4dda71c0553e7965b5d628
Parents: 7c6644f
Author: Arun Suresh <asur...@apache.org>
Authored: Thu Jan 25 19:09:21 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Tue Jan 30 07:54:37 2018 -0800

--
 .../scheduler/AppSchedulingInfo.java| 10 -
 .../scheduler/common/fica/FiCaSchedulerApp.java |  7 +++
 .../constraint/TestPlacementProcessor.java  | 45 
 3 files changed, 60 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3663239d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 0389895..1efdd8b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -694,6 +694,12 @@ public class AppSchedulingInfo {
   metrics.runAppAttempt(applicationId, user);
 }
 
+updateMetrics(applicationId, type, node, containerAllocated, user, queue);
+  }
+
+  public static void updateMetrics(ApplicationId applicationId, NodeType type,
+  SchedulerNode node, Container containerAllocated, String user,
+  Queue queue) {
 if (LOG.isDebugEnabled()) {
   LOG.debug("allocate: applicationId=" + applicationId + " container="
   + containerAllocated.getId() + " host=" + containerAllocated
@@ -702,10 +708,10 @@ public class AppSchedulingInfo {
   + type);
 }
 if(node != null) {
-  metrics.allocateResources(node.getPartition(), user, 1,
+  queue.getMetrics().allocateResources(node.getPartition(), user, 1,
   containerAllocated.getResource(), true);
 }
-metrics.incrNodeTypeAggregations(user, type);
+queue.getMetrics().incrNodeTypeAggregations(user, type);
   }
 
   // Get AppPlacementAllocator by specified schedulerKey

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3663239d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 7eb1e31..f3da0a3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -56,6 +56,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRese
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
@@ -548,6 +549,1

[32/50] [abbrv] hadoop git commit: YARN-7653. Node group support for AllocationTagsManager. (Panagiotis Garefalakis via asuresh)

[asuresh]

YARN-7653. Node group support for AllocationTagsManager. (Panagiotis 
Garefalakis via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e724972b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e724972b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e724972b

Branch: refs/heads/YARN-6592
Commit: e724972bf756bafaf395d05748d940d8d68b09bd
Parents: c83ef6f
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Dec 22 07:24:37 2017 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Tue Jan 30 07:53:34 2018 -0800

--
 .../server/resourcemanager/ResourceManager.java |   2 +-
 .../constraint/AllocationTagsManager.java   | 282 ++-
 .../rmcontainer/TestRMContainerImpl.java|   2 +-
 .../constraint/TestAllocationTagsManager.java   | 269 --
 4 files changed, 392 insertions(+), 163 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e724972b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index a1d3dfc..1d838f0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -496,7 +496,7 @@ public class ResourceManager extends CompositeService 
implements Recoverable {
   }
 
   protected AllocationTagsManager createAllocationTagsManager() {
-return new AllocationTagsManager();
+return new AllocationTagsManager(this.rmContext);
   }
   
   protected DelegationTokenRenewer createDelegationTokenRenewer() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e724972b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
index c278606..7b0b959 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.log4j.Logger;
 
 import java.util.HashMap;
@@ -38,9 +39,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.LongBinaryOperator;
 
 /**
- * Support storing maps between container-tags/applications and
- * nodes. This will be required by affinity/anti-affinity implementation and
- * cardinality.
+ * In-memory mapping between applications/container-tags and nodes/racks.
+ * Required by constrained affinity/anti-affinity and cardinality placement.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
@@ -51,48 +51,54 @@ public class AllocationTagsManager {
 
   private ReentrantReadWriteLock.ReadLock readLock;
   private ReentrantReadWriteLock.WriteLock writeLock;
+  private final RMContext rmContext;
 
-  // Application's tags to node
-  private Map<ApplicationId, NodeToCountedTags> perAppMappings =
+  // Application's tags to Node
+  private Map<ApplicationId, NodeToCountedTags> perAppNodeMappings =
+  new HashMap<>();
+  // Application's tags to Rac

[21/50] [abbrv] hadoop git commit: YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and NMContainerStatus to handle all recovery cases. (asuresh)

[asuresh]

YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and 
NMContainerStatus to handle all recovery cases. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/126eb8d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/126eb8d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/126eb8d7

Branch: refs/heads/YARN-6592
Commit: 126eb8d7abfb5b97a50fa9a1be3d0e630bc6259e
Parents: 30b8d4f
Author: Arun Suresh <asur...@apache.org>
Authored: Fri Jan 12 14:37:06 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Tue Jan 30 07:53:34 2018 -0800

--
 .../hadoop/yarn/api/records/Container.java  | 15 +
 .../src/main/proto/yarn_protos.proto|  1 +
 .../api/records/impl/pb/ContainerPBImpl.java| 31 +
 .../yarn/security/ContainerTokenIdentifier.java | 69 +++-
 .../src/main/proto/yarn_security_token.proto|  1 +
 .../api/protocolrecords/NMContainerStatus.java  | 14 
 .../impl/pb/NMContainerStatusPBImpl.java| 33 ++
 .../yarn_server_common_service_protos.proto |  1 +
 .../containermanager/ContainerManagerImpl.java  |  3 +-
 .../container/ContainerImpl.java| 19 +++---
 .../rmcontainer/RMContainerImpl.java| 10 ++-
 .../scheduler/SchedulerApplicationAttempt.java  |  3 +-
 .../security/RMContainerTokenSecretManager.java | 21 ++
 .../capacity/TestContainerAllocation.java   |  5 +-
 14 files changed, 194 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/126eb8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
index 4fdc803..b9ca3f9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
@@ -27,6 +27,9 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
 import org.apache.hadoop.yarn.util.Records;
 
+import java.util.Collections;
+import java.util.Set;
+
 /**
  * {@code Container} represents an allocated resource in the cluster.
  * 
@@ -256,4 +259,16 @@ public abstract class Container implements 
Comparable {
   public void setVersion(int version) {
 throw new UnsupportedOperationException();
   }
+
+  @Private
+  @Unstable
+  public Set getAllocationTags() {
+return Collections.EMPTY_SET;
+  }
+
+  @Private
+  @Unstable
+  public void setAllocationTags(Set allocationTags) {
+
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/126eb8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 5cb1177..25c8569 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -152,6 +152,7 @@ message ContainerProto {
   optional ExecutionTypeProto execution_type = 7 [default = GUARANTEED];
   optional int64 allocation_request_id = 8 [default = -1];
   optional int32 version = 9 [default = 0];
+  repeated string allocation_tags = 10;
 }
 
 message ContainerReportProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/126eb8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
index be84938..47be2f0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
@@ -36,6 +36,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.PriorityPr

[05/50] [abbrv] hadoop git commit: HDFS-13054. Handling PathIsNotEmptyDirectoryException in DFSClient delete call. Contributed by Nanda kumar.

[asuresh]

HDFS-13054. Handling PathIsNotEmptyDirectoryException in DFSClient delete call. 
Contributed by Nanda kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e990904d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e990904d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e990904d

Branch: refs/heads/YARN-6592
Commit: e990904dd568a1d8f98efb55c1dd2d598ae4752b
Parents: a37e7f0
Author: Arpit Agarwal 
Authored: Fri Jan 26 11:42:27 2018 -0800
Committer: Arpit Agarwal 
Committed: Fri Jan 26 13:09:13 2018 -0800

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java |  4 +++-
 .../hadoop/hdfs/protocol/ClientProtocol.java   |  3 +++
 .../hadoop/hdfs/TestDistributedFileSystem.java | 17 +
 3 files changed, 23 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e990904d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index f0769c1..92bb99e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -82,6 +82,7 @@ import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.Options.ChecksumOpt;
 import org.apache.hadoop.fs.ParentNotDirectoryException;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
 import org.apache.hadoop.fs.QuotaUsage;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.StorageType;
@@ -1620,7 +1621,8 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
   FileNotFoundException.class,
   SafeModeException.class,
   UnresolvedPathException.class,
-  SnapshotAccessControlException.class);
+  SnapshotAccessControlException.class,
+  PathIsNotEmptyDirectoryException.class);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e990904d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index fbef037..0d77037 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.crypto.CryptoProtocolVersion;
 import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
 import org.apache.hadoop.hdfs.AddBlockFlag;
 import org.apache.hadoop.fs.CacheFlag;
 import org.apache.hadoop.fs.ContentSummary;
@@ -625,6 +626,8 @@ public interface ClientProtocol {
* @throws org.apache.hadoop.fs.UnresolvedLinkException If src
*   contains a symlink
* @throws SnapshotAccessControlException if path is in RO snapshot
+   * @throws PathIsNotEmptyDirectoryException if path is a non-empty directory
+   *   and recursive is set to false
* @throws IOException If an I/O error occurred
*/
   @AtMostOnce

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e990904d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 823c747..072ee9f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
 import org.apache.hadoop.fs.Options.ChecksumOpt;
 import org.apache.hadoop.fs.Path;
+import 

[50/50] [abbrv] hadoop git commit: YARN-7763. Allow Constraints specified in the SchedulingRequest to override application level constraints. (Weiwei Yang via asuresh)

[asuresh]

YARN-7763. Allow Constraints specified in the SchedulingRequest to override 
application level constraints. (Weiwei Yang via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3803dea4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3803dea4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3803dea4

Branch: refs/heads/YARN-6592
Commit: 3803dea47f3f498f5b4165bef677fed12e66a36e
Parents: 7445139
Author: Arun Suresh <asur...@apache.org>
Authored: Sun Jan 21 19:11:17 2018 -0800
Committer: Arun Suresh <asur...@apache.org>
Committed: Tue Jan 30 07:54:37 2018 -0800

--
 .../scheduler/capacity/CapacityScheduler.java   |  4 +-
 .../constraint/PlacementConstraintsUtil.java| 98 +++-
 .../algorithm/DefaultPlacementAlgorithm.java|  4 +-
 .../SingleConstraintAppPlacementAllocator.java  | 10 +-
 .../TestPlacementConstraintsUtil.java   | 94 ---
 5 files changed, 123 insertions(+), 87 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3803dea4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 429f9f3..a096e2f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2621,9 +2621,9 @@ public class CapacityScheduler extends
 // Validate placement constraint is satisfied before
 // committing the request.
 try {
-  if (!PlacementConstraintsUtil.canSatisfySingleConstraint(
+  if (!PlacementConstraintsUtil.canSatisfyConstraints(
   appAttempt.getApplicationId(),
-  schedulingRequest.getAllocationTags(), schedulerNode,
+  schedulingRequest, schedulerNode,
   rmContext.getPlacementConstraintManager(),
   rmContext.getAllocationTagsManager())) {
 LOG.debug("Failed to allocate container for application "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3803dea4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
index ff5cb67..c07c16f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
 import 
org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
@@ -54,7 +55,7 @@ public final class PlacementConstraintsUtil {
   }
 
   /**
-   * Returns true if **single** placement constraint with associated
+   * Returns true if single placement constraint with associated
* allocationTags and scope is satisfied by a s