minni31 commented on a change in pull request #3760:
URL: https://github.com/apache/hadoop/pull/3760#discussion_r797321327
##########
File path:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
##########
@@ -795,6 +797,106 @@ public void testSubClusterExpiry() throws Exception {
checkTotalContainerAllocation(response, 100);
}
+ @After
+ public void cleanup() {
+ ((FederationAMRMProxyPolicy) getPolicy()).shutdown();
+ }
+
+ @Test
+ public void testLoadbasedSubClusterReroute() throws YarnException {
+ int pendingThreshold = 1000;
+
+ LocalityMulticastAMRMProxyPolicy policy =
+ (LocalityMulticastAMRMProxyPolicy) getPolicy();
+ initializePolicy();
+
+ SubClusterId sc0 = SubClusterId.newInstance("0");
+ SubClusterId sc1 = SubClusterId.newInstance("1");
+ SubClusterId sc2 = SubClusterId.newInstance("2");
+ SubClusterId sc3 = SubClusterId.newInstance("3");
+ SubClusterId sc4 = SubClusterId.newInstance("4");
+
+ Set<SubClusterId> scList = new HashSet<>();
+ scList.add(sc0);
+ scList.add(sc1);
+ scList.add(sc2);
+ scList.add(sc3);
+ scList.add(sc4);
+
+ // This cluster is the most overloaded - 4 times the threshold.
+ policy.notifyOfResponse(sc0,
+ getAllocateResponseWithEnhancedHeadroom(4 * pendingThreshold, 0));
+
+ // This cluster is the most overloaded - 4 times the threshold.
+ policy.notifyOfResponse(sc1,
+ getAllocateResponseWithEnhancedHeadroom(4 * pendingThreshold, 0));
+
+ // This cluster is 2 times the threshold, but not the most loaded.
+ policy.notifyOfResponse(sc2,
+ getAllocateResponseWithEnhancedHeadroom(2 * pendingThreshold, 0));
+
+ // This cluster is at the threshold, but not the most loaded.
+ policy.notifyOfResponse(sc3,
+ getAllocateResponseWithEnhancedHeadroom(pendingThreshold, 0));
+
+ // This cluster has zero pending.
+ policy.notifyOfResponse(sc4, getAllocateResponseWithEnhancedHeadroom(0,
0));
+
+ // sc2, sc3 and sc4 should just return the original subcluster.
+ Assert.assertEquals(
+ policy.routeNodeRequestIfNeeded(sc2, pendingThreshold, scList), sc2);
+ Assert.assertEquals(
+ policy.routeNodeRequestIfNeeded(sc3, pendingThreshold, scList), sc3);
+ Assert.assertEquals(
+ policy.routeNodeRequestIfNeeded(sc4, pendingThreshold, scList), sc4);
+
+ // sc0 and sc1 must select from sc0/sc1/sc2/sc3/sc4 according to weights
+ // 1/4, 1/4, 1/2, 1, 2. Let's run tons of random of samples, and verify
that
+ // the proportion approximately holds.
+ Map<SubClusterId, Integer> counts = new HashMap<SubClusterId, Integer>();
+ counts.put(sc0, 0);
+ counts.put(sc1, 0);
+ counts.put(sc2, 0);
+ counts.put(sc3, 0);
+ counts.put(sc4, 0);
+ int n = 100000;
+ for (int i = 0; i < n; i++) {
+ SubClusterId selectedId =
+ policy.routeNodeRequestIfNeeded(sc0, pendingThreshold, scList);
+ counts.put(selectedId, counts.get(selectedId) + 1);
+
+ selectedId =
+ policy.routeNodeRequestIfNeeded(sc1, pendingThreshold, scList);
+ counts.put(selectedId, counts.get(selectedId) + 1);
+
+ // Also try a new SCId that's not active and enabled. Should be rerouted
+ // to sc0-4 with the same distribution as above
+ selectedId = policy
+ .routeNodeRequestIfNeeded(SubClusterId.newInstance("10"),
+ pendingThreshold, scList);
+ counts.put(selectedId, counts.get(selectedId) + 1);
+ }
+
+ // The probability should be 1/16, 1/16, 1/8, 1/4, 1/2
+ Assert.assertEquals((double) counts.get(sc0) / n / 3, 0.0625, 0.01);
+ Assert.assertEquals((double) counts.get(sc1) / n / 3, 0.0625, 0.01);
+ Assert.assertEquals((double) counts.get(sc2) / n / 3, 0.125, 0.01);
+ Assert.assertEquals((double) counts.get(sc3) / n / 3, 0.25, 0.01);
+ Assert.assertEquals((double) counts.get(sc4) / n / 3, 0.5, 0.01);
Review comment:
I didn't get the rationale behind that, can you please explain that ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
