SLIDER-1130 Hadoop 2.8 YARN changes have broken slider. See also YARN-5135: ContainerRequest labels ignored -broken constructor
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/80fe8c7e Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/80fe8c7e Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/80fe8c7e Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: 80fe8c7e04a933885ab0f623216940861b233b5f Parents: 9937bbc Author: Steve Loughran <ste...@apache.org> Authored: Tue May 24 17:59:58 2016 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Tue May 24 17:59:58 2016 +0100 ---------------------------------------------------------------------- .../appmaster/state/OutstandingRequest.java | 2 ++ .../appstate/TestMockAppStateAAPlacement.groovy | 6 ++--- .../appstate/TestMockLabelledAAPlacement.groovy | 4 +-- .../TestOutstandingRequestValidation.groovy | 26 -------------------- ...tRoleHistoryOutstandingRequestTracker.groovy | 5 +++- .../appmaster/model/mock/MockYarnCluster.groovy | 2 +- 6 files changed, 12 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java index e0b01a1..c6524be 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java @@ -283,6 +283,7 @@ public final class OutstandingRequest extends RoleHostnamePair { */ public synchronized AMRMClient.ContainerRequest escalate() { Preconditions.checkNotNull(issuedRequest, "cannot escalate if request not issued " + this); + log.debug("Escalating {}", this.toString()); escalated = true; // this is now the priority @@ -352,6 +353,7 @@ public final class OutstandingRequest extends RoleHostnamePair { } sb.append(", node=").append(node); sb.append(", hasLocation=").append(requestHasLocation); + sb.append(", label=").append(label); sb.append(", requestedTimeMillis=").append(requestedTimeMillis); sb.append(", mayEscalate=").append(mayEscalate); sb.append(", escalated=").append(escalated); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy index 85c09ff..125b1f4 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy @@ -57,7 +57,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest } /** - * This is the simplest AA allocation: no lables, so allocate anywhere + * This is the simplest AA allocation: no labels, so allocate anywhere * @throws Throwable */ @Test @@ -218,7 +218,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest // flex down so that the next request should be cancelled aaRole.desired = 0 - // expect: no new reqests, pending count -- + // expect: no new requests, pending count -- List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes() assert aaRole.pendingAntiAffineRequests == 0 assert !aaRole.AARequestOutstanding @@ -281,7 +281,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest } protected AppState.NodeUpdatedOutcome addNewNode() { - updateNodes(MockFactory.newNodeReport("4", NodeState.RUNNING, "gpu")) + updateNodes(MockFactory.instance.newNodeReport("4", NodeState.RUNNING, "gpu")) } @Test http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy index b2e2a74..e159dbb 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy @@ -48,8 +48,8 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest super.setup() // node 1 is GPU - updateNodes(MockFactory.newNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU)) - updateNodes(MockFactory.newNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU)) + updateNodes(MockFactory.instance.newNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU)) + updateNodes(MockFactory.instance.newNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU)) } @Override http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy index 9202b5c..fb9d976 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy @@ -55,21 +55,6 @@ class TestOutstandingRequestValidation extends SliderTestBase { createAndValidate(H1, "", true) } - @Test - public void testLabelAndHostRelaxed() throws Throwable { - expectValidationFailure(H1, "gpu", true) - } - - @Test - public void testLabelAndHostNonRelaxed() throws Throwable { - expectValidationFailure(H1, "gpu", false) - } - - @Test - public void testComplexLabelExpression() throws Throwable { - expectValidationFailure(null, "gpu||ps4", true) - } - /** * Use varargs for simple list to array conversion * @param hostnames host names @@ -79,17 +64,6 @@ class TestOutstandingRequestValidation extends SliderTestBase { hostnames } - void expectValidationFailure( - String[] hosts, - String labels, - boolean relaxLocality) { - try { - def result = createAndValidate(hosts, labels, relaxLocality) - fail("Expected an exception, got $result") - } catch (InvalidContainerRequestException expected) { - } - } - void expectCreationFailure( String[] hosts, String labels, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy index 6969b38..0969824 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy @@ -238,12 +238,15 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { def workerRole = lookupRole(WORKER.name) // initial request def yarnRequest = req1.buildContainerRequest(resource, workerRole, 0) + assert (req1.label == WORKERS_LABEL) + assert (yarnRequest.nodeLabelExpression == null) assert (!yarnRequest.relaxLocality) // escalation def yarnRequest2 = req1.escalate() - assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL) + assert yarnRequest2.nodes == null assert (yarnRequest2.relaxLocality) + assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL) } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy index e1dfa25..f0a1494 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy @@ -182,7 +182,7 @@ public class MockYarnCluster { containers[i] = new MockYarnClusterContainer(mci) } - nodeReport = MockFactory.newNodeReport(hostname, nodeId, NodeState.RUNNING, "") + nodeReport = MockFactory.instance.newNodeReport(hostname, nodeId, NodeState.RUNNING, "") } /**
