SLIDER-963 Write mock/unit tests for AA placement
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a3591eb1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a3591eb1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a3591eb1 Branch: refs/heads/feature/SLIDER-82-pass-3.1 Commit: a3591eb16408213927b01f3f33c11bd016f13a87 Parents: 55d50fe Author: Steve Loughran <[email protected]> Authored: Thu Nov 5 18:16:43 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Thu Nov 5 18:16:43 2015 +0000 ---------------------------------------------------------------------- .../apache/slider/common/tools/Comparators.java | 13 ++++- .../apache/slider/common/tools/SliderUtils.java | 1 + .../server/appmaster/state/NodeEntry.java | 7 +++ .../server/appmaster/state/NodeInstance.java | 32 +++++------ .../appmaster/state/OutstandingRequest.java | 3 +- .../server/appmaster/state/RoleHistory.java | 2 +- .../server/appmaster/state/RoleStatus.java | 3 ++ .../TestMockAppStateContainerFailure.groovy | 6 +-- .../history/TestRoleHistoryNIComparators.groovy | 45 +++++++++------- ...tRoleHistoryOutstandingRequestTracker.groovy | 7 +-- .../model/history/TestRoleHistoryRW.groovy | 4 +- .../history/TestRoleHistoryRWOrdering.groovy | 8 +-- .../TestRoleHistoryRequestTracking.groovy | 57 +++++++++++--------- .../model/mock/BaseMockAppStateTest.groovy | 4 ++ 14 files changed, 112 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java index 0ccca0f..6380d0c 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java @@ -21,6 +21,9 @@ package org.apache.slider.common.tools; import java.io.Serializable; import java.util.Comparator; +/** + * Some general comparators + */ public class Comparators { public static class LongComparator implements Comparator<Long>, Serializable { @@ -30,12 +33,20 @@ public class Comparators { // need to comparisons with a diff greater than integer size if (result < 0 ) { return -1; - } else if (result >0) { + } else if (result > 0) { return 1; } return 0; } } +public static class InvertedLongComparator implements Comparator<Long>, Serializable { + private static final LongComparator inner = new LongComparator(); + @Override + public int compare(Long o1, Long o2) { + return -inner.compare(o1, o2); + } + } + /** * Little template class to reverse any comparitor http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 1f97982..76668bf 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -2482,6 +2482,7 @@ public final class SliderUtils { } public static String requestToString(AMRMClient.ContainerRequest request) { + Preconditions.checkArgument(request != null, "Null request"); StringBuffer buffer = new StringBuffer(request.toString()); buffer.append("; "); buffer.append("relaxLocality=").append(request.getRelaxLocality()).append("; "); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java index 8ff0895..6dae3c6 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java @@ -18,6 +18,7 @@ package org.apache.slider.server.appmaster.state; +import com.google.common.annotations.VisibleForTesting; import org.apache.slider.api.types.NodeEntryInformation; /** @@ -256,10 +257,16 @@ public class NodeEntry { return failedRecently; } + @VisibleForTesting + public synchronized void setFailedRecently(int failedRecently) { + this.failedRecently = failedRecently; + } + public synchronized int getPreempted() { return preempted; } + /** * Reset the failed recently count. */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java index 2afdc42..7fc912d 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.slider.api.types.NodeEntryInformation; import org.apache.slider.api.types.NodeInformation; +import org.apache.slider.common.tools.Comparators; import org.apache.slider.common.tools.SliderUtils; import java.io.Serializable; @@ -69,7 +70,7 @@ public class NodeInstance { * @param report latest node report * @return true if the node state changed */ - public boolean updateNode(NodeReport report) { + public synchronized boolean updateNode(NodeReport report) { nodeReport = report; NodeState oldState = nodeState; nodeState = report.getNodeState(); @@ -272,16 +273,18 @@ public class NodeInstance { /** * A comparator for sorting entries where the node is preferred over another. - * <p> - * The exact algorithm may change - * + * + * The exact algorithm may change: current policy is "most recent first", so sorted + * on the lastUsed + * * the comparision is a positive int if left is preferred to right; * negative if right over left, 0 for equal */ - public static class Preferred implements Comparator<NodeInstance>, - Serializable { + public static class Preferred implements Comparator<NodeInstance>, Serializable { - final int role; + private static final Comparators.InvertedLongComparator comparator = + new Comparators.InvertedLongComparator(); + private final int role; public Preferred(int role) { this.role = role; @@ -291,16 +294,9 @@ public class NodeInstance { public int compare(NodeInstance o1, NodeInstance o2) { NodeEntry left = o1.get(role); NodeEntry right = o2.get(role); - long ageL = left != null ? left.getLastUsed() : 0; - long ageR = right != null ? right.getLastUsed() : 0; - - if (ageL > ageR) { - return -1; - } else if (ageL < ageR) { - return 1; - } - // equal - return 0; + long ageL = left != null ? left.getLastUsed() : -1; + long ageR = right != null ? right.getLastUsed() : -1; + return comparator.compare(ageL, ageR); } } @@ -313,7 +309,7 @@ public class NodeInstance { public static class MoreActiveThan implements Comparator<NodeInstance>, Serializable { - final int role; + private final int role; public MoreActiveThan(int role) { this.role = role; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java index f5689dd..602b48e 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java @@ -353,7 +353,8 @@ public final class OutstandingRequest { sb.append(", mayEscalate=").append(mayEscalate); sb.append(", escalated=").append(escalated); sb.append(", escalationTimeoutMillis=").append(escalationTimeoutMillis); - sb.append(", issuedRequest=").append(SliderUtils.requestToString(issuedRequest)); + sb.append(", issuedRequest=").append( + issuedRequest != null ? SliderUtils.requestToString(issuedRequest) : "(null)"); sb.append('}'); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index 53c2689..6880b69 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -955,7 +955,7 @@ public class RoleHistory { * @return a clone of the list */ @VisibleForTesting - public List<NodeInstance> cloneAvailableList(int role) { + public List<NodeInstance> cloneRecentNodeList(int role) { return new LinkedList<>(listRecentNodesForRoleId(role)); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java index 98a8311..52df406 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java @@ -61,6 +61,9 @@ public final class RoleStatus implements Cloneable { */ private int pendingAntiAffineRequestCount = 0; + /** any pending AA request */ + public OutstandingRequest outstandingAArequest = null; + private String failureMessage = ""; public RoleStatus(ProviderRole providerRole) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy index 1b79115..5b24a59 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy @@ -95,7 +95,7 @@ class TestMockAppStateContainerFailure extends BaseMockAppStateTest //view the world appState.getRoleHistory().dump(); - List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0) + List<NodeInstance> queue = appState.roleHistory.cloneRecentNodeList(0) assert queue.size() == 0 } @@ -123,7 +123,7 @@ class TestMockAppStateContainerFailure extends BaseMockAppStateTest //view the world appState.getRoleHistory().dump(); - List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0) + List<NodeInstance> queue = appState.roleHistory.cloneRecentNodeList(0) assert queue.size() == 1 } @@ -148,7 +148,7 @@ class TestMockAppStateContainerFailure extends BaseMockAppStateTest RoleHistory history = appState.roleHistory history.dump(); - List<NodeInstance> queue = history.cloneAvailableList(0) + List<NodeInstance> queue = history.cloneRecentNodeList(0) assert queue.size() == 0 NodeInstance ni = history.getOrCreateNodeInstance(instance.container) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy index b26b2f0..ee910e4 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy @@ -18,6 +18,7 @@ package org.apache.slider.server.appmaster.model.history +import groovy.transform.CompileStatic import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest import org.apache.slider.server.appmaster.model.mock.MockFactory import org.apache.slider.server.appmaster.state.NodeInstance @@ -27,17 +28,19 @@ import org.junit.Test /** * Unit test to verify the comparators sort as expected */ +@CompileStatic class TestRoleHistoryNIComparators extends BaseMockAppStateTest { - NodeInstance age1Active4 = nodeInstance(1000, 4, 0, 0) - NodeInstance age2Active2 = nodeInstance(1001, 2, 0, 0) - NodeInstance age3Active0 = nodeInstance(1002, 0, 0, 0) - NodeInstance age4Active1 = nodeInstance(1005, 0, 0, 0) + NodeInstance age1Active4 = nodeInstance(1001, 4, 0, 0) + NodeInstance age2Active2 = nodeInstance(1002, 2, 0, 0) + NodeInstance age3Active0 = nodeInstance(1003, 0, 0, 0) + NodeInstance age4Active1 = nodeInstance(1004, 1, 0, 0) NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT) NodeInstance age6failing = nodeInstance(1006, 0, 0, 0) - NodeInstance age1failing = nodeInstance(1000, 0, 0, 0) + NodeInstance age1failing = nodeInstance(1001, 0, 0, 0) List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0] + List<NodeInstance> allnodes = [age6failing, age2Active2, age4Active1, age1Active4, age3Active0, age1failing] @Before public void setup() { @@ -51,43 +54,49 @@ class TestRoleHistoryNIComparators extends BaseMockAppStateTest { } @Test - public void testNewerThan() throws Throwable { - + public void testPreferred() throws Throwable { Collections.sort(nodes, new NodeInstance.Preferred(0)) - assertListEquals(nodes, - [age4Active1, age3Active0, age2Active2, age1Active4]) + assertListEquals(nodes, [age4Active1, age3Active0, age2Active2, age1Active4]) + } + + /** + * The preferred sort still includes failures; up to next phase in process + * to handle that + * @throws Throwable + */ + @Test + public void testPreferredWithFailures() throws Throwable { + Collections.sort(allnodes, new NodeInstance.Preferred(0)) + assert allnodes[0] == age6failing + assert allnodes[1] == age4Active1 } @Test - public void testFailureCountFirst() throws Throwable { + public void testPreferredComparatorDowngradesFailures() throws Throwable { def preferred = new NodeInstance.Preferred(0) assert preferred.compare(age6failing, age1failing) == -1 assert preferred.compare(age1failing, age6failing) == 1 } - + @Test public void testNewerThanNoRole() throws Throwable { - nodes << empty Collections.sort(nodes, new NodeInstance.Preferred(0)) - assertListEquals(nodes, - [age4Active1, age3Active0, age2Active2, age1Active4, empty]) + assertListEquals(nodes, [age4Active1, age3Active0, age2Active2, age1Active4, empty]) } @Test public void testMoreActiveThan() throws Throwable { Collections.sort(nodes, new NodeInstance.MoreActiveThan(0)) - assertListEquals(nodes, - [age1Active4, age2Active2, age4Active1, age3Active0],) + assertListEquals(nodes, [age1Active4, age2Active2, age4Active1, age3Active0]) } @Test public void testMoreActiveThanEmpty() throws Throwable { nodes << empty Collections.sort(nodes, new NodeInstance.MoreActiveThan(0)) - assertListEquals(nodes, - [age1Active4, age2Active2, age4Active1, age3Active0, empty]) + assertListEquals(nodes, [age1Active4, age2Active2, age4Active1, age3Active0, empty]) } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy index 653af84..33cacd7 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy @@ -35,17 +35,12 @@ import org.apache.slider.server.appmaster.state.OutstandingRequestTracker import org.apache.slider.server.appmaster.state.RoleStatus import org.junit.Test -class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { +class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { NodeInstance host1 = new NodeInstance("host1", 3) NodeInstance host2 = new NodeInstance("host2", 3) OutstandingRequestTracker tracker = new OutstandingRequestTracker() - - @Override - String getTestName() { - return "TestOutstandingRequestTracker" - } @Test public void testAddRetrieveEntry() throws Throwable { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy index 7afcfc1..a1e424f 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy @@ -137,13 +137,13 @@ class TestRoleHistoryRW extends BaseMockAppStateTest { rh2.buildRecentNodeLists(); describe("starting") rh2.dump(); - List<NodeInstance> available0 = rh2.cloneAvailableList(0) + List<NodeInstance> available0 = rh2.cloneRecentNodeList(0) assert available0.size() == 1 NodeInstance entry = available0.get(0) assert entry.hostname == "localhost" assert entry == localhost - List<NodeInstance> available1 = rh2.cloneAvailableList(1) + List<NodeInstance> available1 = rh2.cloneRecentNodeList(1) assert available1.size() == 2 //and verify that even if last used was set, the save time is picked up assert entry.get(1).lastUsed == roleHistory.saveTime http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy index aef22fb..0655531 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy @@ -18,6 +18,7 @@ package org.apache.slider.server.appmaster.model.history +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.fs.Path import org.apache.slider.common.SliderKeys @@ -34,9 +35,10 @@ import java.util.regex.Matcher import java.util.regex.Pattern @Slf4j +@CompileStatic class TestRoleHistoryRWOrdering extends BaseMockAppStateTest { - def paths = pathlist( + List<Path> paths = pathlist( [ "hdfs://localhost/history-0406c.json", "hdfs://localhost/history-5fffa.json", @@ -60,7 +62,6 @@ class TestRoleHistoryRWOrdering extends BaseMockAppStateTest { return "TestHistoryRWOrdering" } - /** * This tests regexp pattern matching. It uses the current time so isn't * repeatable -but it does test a wider range of values in the process @@ -79,7 +80,6 @@ class TestRoleHistoryRWOrdering extends BaseMockAppStateTest { } } - @Test public void testWriteSequenceReadData() throws Throwable { describe "test that if multiple entries are written, the newest is picked up" @@ -116,7 +116,7 @@ class TestRoleHistoryRWOrdering extends BaseMockAppStateTest { public void testPathStructure() throws Throwable { assert h_5fffa.getName() == "history-5fffa.json" } - + @Test public void testPathnameComparator() throws Throwable { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy index db795d0..42d0c50 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy @@ -18,6 +18,7 @@ package org.apache.slider.server.appmaster.model.history +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.api.records.Container import org.apache.hadoop.yarn.api.records.Resource @@ -41,6 +42,7 @@ import org.junit.Test * get removed and added */ @Slf4j +@CompileStatic class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { String roleName = "test" @@ -54,6 +56,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0] RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) + /** 1MB, 1 vcore*/ Resource resource = Resource.newInstance(1, 1) ProviderRole provRole = new ProviderRole(roleName, 0) @@ -72,27 +75,25 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { @Test public void testAvailableListBuiltForRoles() throws Throwable { - List<NodeInstance> available0 = roleHistory.cloneAvailableList(0) + List<NodeInstance> available0 = roleHistory.cloneRecentNodeList(0) assertListEquals([age3Active0, age2Active0], available0) } @Test public void testRequestedNodeOffList() throws Throwable { - List<NodeInstance> available0 = roleHistory.cloneAvailableList(0) NodeInstance ni = roleHistory.findNodeForNewInstance(roleStatus) assert age3Active0 == ni - AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni, + assertListEquals([age2Active0], roleHistory.cloneRecentNodeList(0)) + roleHistory.requestInstanceOnNode(ni, roleStatus, resource, "") - List<NodeInstance> a2 = roleHistory.cloneAvailableList(0) - assertListEquals([age2Active0], a2) } @Test public void testRequestedNodeOffListWithFailures() throws Throwable { assert 0 == roleStatus.key - assert !roleHistory.cloneAvailableList(0).isEmpty() + assert !roleHistory.cloneRecentNodeList(0).empty NodeEntry age3role0 = recordAsFailed(age3Active0, 0, 4) assert age3Active0.isConsideredUnreliable(0, roleStatus.nodeFailureThreshold) @@ -115,51 +116,55 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { roleHistory.dump() assert 0 == age3role0.failedRecently assert !age3Active0.isConsideredUnreliable(0, roleStatus.nodeFailureThreshold) - assert !roleHistory.cloneAvailableList(0).isEmpty() + assert !roleHistory.cloneRecentNodeList(0).empty // looking for a node should now find one ni = roleHistory.findNodeForNewInstance(roleStatus) assert ni == age3Active0 - req = roleHistory.requestInstanceOnNode(ni, - roleStatus, - resource, - "") + req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource, "") assert 1 == req.nodes.size() } + /** + * verify that strict placement policies generate requests for nodes irrespective + * of their failed status + * @throws Throwable + */ @Test public void testStrictPlacementIgnoresFailures() throws Throwable { def targetRole = role1Status final ProviderRole providerRole1 = targetRole.providerRole assert providerRole1.placementPolicy == PlacementPolicy.STRICT - int key = targetRole.key + int key1 = targetRole.key + def key0 = role0Status.key - recordAsFailed(age1Active4, key, 4) - recordAsFailed(age2Active0, key, 4) - recordAsFailed(age2Active2, key, 4) - recordAsFailed(age3Active0, key, 4) - recordAsFailed(age4Active1, key, 4) + def nodes = [age1Active4, age2Active0, age2Active2, age3Active0, age4Active1] + recordAllFailed(key0, 4, nodes) + recordAllFailed(key1, 4, nodes) // trigger a list rebuild roleHistory.buildRecentNodeLists(); + def recentRole0 = roleHistory.cloneRecentNodeList(key0) + assert recentRole0.indexOf(age3Active0) < recentRole0.indexOf(age2Active0) - assert !roleHistory.cloneAvailableList(key).isEmpty() + // the non-strict role has no suitable nodes + assert null == roleHistory.findNodeForNewInstance(role0Status) - NodeInstance ni = roleHistory.findNodeForNewInstance(targetRole) - assert ni == age4Active1!= null - // next lookup returns next node - ni = roleHistory.findNodeForNewInstance(roleStatus) - assert ni == age3Active0 - } + def ni = roleHistory.findNodeForNewInstance(targetRole) + assert ni + def ni2 = roleHistory.findNodeForNewInstance(targetRole) + assert ni2 + assert ni != ni2 + } @Test public void testFindAndRequestNode() throws Throwable { AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource) assert age3Active0.hostname == req.nodes[0] - List<NodeInstance> a2 = roleHistory.cloneAvailableList(0) + List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(0) assertListEquals([age2Active0], a2) } @@ -239,7 +244,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { roleHistory.listOpenRequests().empty // and the remainder goes onto the available list - List<NodeInstance> a2 = roleHistory.cloneAvailableList(0) + List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(0) assertListEquals([age2Active0], a2) } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3591eb1/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy index 33ea0a0..babce9b 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy @@ -356,4 +356,8 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles } entry } + + def recordAllFailed(int id, int count, List<NodeInstance> nodes) { + nodes.each { NodeInstance node -> recordAsFailed(node, id, count)} + } }
