[
https://issues.apache.org/jira/browse/APEXCORE-10?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15212769#comment-15212769
]
ASF GitHub Bot commented on APEXCORE-10:
----------------------------------------
Github user tweise commented on a diff in the pull request:
https://github.com/apache/incubator-apex-core/pull/250#discussion_r57508599
--- Diff:
engine/src/main/java/com/datatorrent/stram/ResourceRequestHandler.java ---
@@ -170,19 +205,69 @@ public String getHost(ContainerStartRequest csr,
boolean first)
containers.add(nodeLocalOper.getContainer());
}
}
- for (Map.Entry<String, NodeReport> nodeEntry :
nodeReportMap.entrySet()) {
- int memAvailable =
nodeEntry.getValue().getCapability().getMemory() -
nodeEntry.getValue().getUsed().getMemory();
- int vCoresAvailable =
nodeEntry.getValue().getCapability().getVirtualCores() -
nodeEntry.getValue().getUsed().getVirtualCores();
- if (memAvailable >= aggrMemory && vCoresAvailable >= vCores) {
- host = nodeEntry.getKey();
- grpObj.setHost(host);
- nodeLocalMapping.put(nodeLocalSet, host);
- return host;
- }
+ host = assignHost(host, antiHosts, antiPreferredHosts, grpObj,
nodeLocalSet, aggrMemory, vCores);
+
+ if (host == null && !antiPreferredHosts.isEmpty() &&
!antiHosts.isEmpty()) {
+ // Drop the preferred constraint and try allocation
+ antiPreferredHosts.clear();
+ host = assignHost(host, antiHosts, antiPreferredHosts, grpObj,
nodeLocalSet, aggrMemory, vCores);
+ }
+ if (host != null) {
+ antiAffinityMapping.put(c, host);
+ } else {
+ host = INVALID_HOST;
}
}
}
+ LOG.info("Found host {}", host);
return host;
}
+ public void populateAntiHostList(PTContainer c, List<String> antiHosts)
+ {
+ for (PTContainer container : c.getStrictAntiPrefs()) {
+ if (antiAffinityMapping.containsKey(container)) {
+ antiHosts.add(antiAffinityMapping.get(container));
+ } else {
+ // Check if there is an anti-affinity with host locality
+ String antiHost = getAntiHostsList(container);
+ if (antiHost != null) {
+ antiHosts.add(antiHost);
+ }
+ }
+ }
+ }
+
+ public String getAntiHostsList(PTContainer container)
--- End diff --
javadoc
> Enable non-affinity of operators per node (not containers)
> ----------------------------------------------------------
>
> Key: APEXCORE-10
> URL: https://issues.apache.org/jira/browse/APEXCORE-10
> Project: Apache Apex Core
> Issue Type: Task
> Reporter: Amol Kekre
> Assignee: Isha Arkatkar
> Labels: roadmap
>
> The issue happens on cloud which provides virtual cores with software like
> Xen underneath. In effect if CPU intensive operators land up on same node we
> have a resource bottleneck,
> Need to create an attribute that does the following
> - Operators A & B should not be on same node
> - Stram should use this attribute to try to get containers on different node
> It is understood that the user is making an explicit choice to use NIC
> instead of stream local optimization
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
