SLIDER-1130 Hadoop 2.8 YARN RPC changes have broken slider mock classes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9937bbca Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9937bbca Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9937bbca Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: 9937bbcae3ac1575a82b0140558264f2507962ff Parents: 1154470 Author: Steve Loughran <ste...@apache.org> Authored: Mon May 23 22:40:49 2016 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Mon May 23 22:40:49 2016 +0100 ---------------------------------------------------------------------- .../appstate/TestMockAppStateAAPlacement.groovy | 4 +- .../appstate/TestMockLabelledAAPlacement.groovy | 11 +-- .../model/history/TestRoleHistoryAA.groovy | 12 ++-- .../model/mock/MockContainerStatus.groovy | 34 --------- .../appmaster/model/mock/MockFactory.groovy | 60 +++++++++++++++- .../appmaster/model/mock/MockNodeReport.groovy | 75 -------------------- .../model/mock/MockRecordFactory.groovy | 37 ++++++++++ .../appmaster/model/mock/MockYarnCluster.groovy | 9 ++- .../appmaster/model/mock/MockYarnEngine.groovy | 5 +- 9 files changed, 114 insertions(+), 133 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy index e43d894..85c09ff 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy @@ -24,12 +24,10 @@ import org.apache.hadoop.yarn.api.records.Container import org.apache.hadoop.yarn.api.records.NodeState import org.apache.hadoop.yarn.client.api.AMRMClient import org.apache.slider.api.ResourceKeys -import org.apache.slider.api.types.NodeInformationList import org.apache.slider.core.conf.ConfTreeOperations import org.apache.slider.providers.PlacementPolicy import org.apache.slider.server.appmaster.model.mock.MockAppState import org.apache.slider.server.appmaster.model.mock.MockFactory -import org.apache.slider.server.appmaster.model.mock.MockNodeReport import org.apache.slider.server.appmaster.model.mock.MockRoles import org.apache.slider.server.appmaster.model.mock.MockYarnEngine import org.apache.slider.server.appmaster.operations.AbstractRMOperation @@ -283,7 +281,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest } protected AppState.NodeUpdatedOutcome addNewNode() { - updateNodes(new MockNodeReport("4", NodeState.RUNNING, "gpu")) + updateNodes(MockFactory.newNodeReport("4", NodeState.RUNNING, "gpu")) } @Test http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy index f0fed95..b2e2a74 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy @@ -22,7 +22,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.api.records.Container import org.apache.hadoop.yarn.api.records.NodeState -import org.apache.slider.server.appmaster.model.mock.MockNodeReport +import org.apache.slider.server.appmaster.model.mock.MockFactory import org.apache.slider.server.appmaster.model.mock.MockRoles import org.apache.slider.server.appmaster.model.mock.MockYarnEngine import org.apache.slider.server.appmaster.operations.AbstractRMOperation @@ -48,8 +48,8 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest super.setup() // node 1 is GPU - updateNodes(new MockNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU)) - updateNodes(new MockNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU)) + updateNodes(MockFactory.newNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU)) + updateNodes(MockFactory.newNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU)) } @Override @@ -108,7 +108,8 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest assert 0 == appState.reviewRequestAndReleaseNodes().size() // switch node 2 into being labelled - def outcome = updateNodes(new MockNodeReport("00000002", NodeState.RUNNING, "gpu")) + def outcome = updateNodes(MockFactory.instance. + newNodeReport("00000002", NodeState.RUNNING, "gpu")) assert cloneNodemap().size() == NODES assert outcome.clusterChanged @@ -118,7 +119,7 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest } protected AppState.NodeUpdatedOutcome addNewNode() { - updateNodes(new MockNodeReport("00000004", NodeState.RUNNING, "gpu")) + updateNodes(MockFactory.instance.newNodeReport("00000004", NodeState.RUNNING, "gpu")) } @Test http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy index db84b0b..9c60ea5 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy @@ -21,12 +21,9 @@ package org.apache.slider.server.appmaster.model.history import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.api.records.NodeReport import org.apache.hadoop.yarn.api.records.NodeState -import org.apache.slider.api.proto.Messages import org.apache.slider.api.proto.RestTypeMarshalling -import org.apache.slider.api.types.NodeInformation import org.apache.slider.api.types.NodeInformationList import org.apache.slider.server.appmaster.model.mock.MockFactory -import org.apache.slider.server.appmaster.model.mock.MockNodeReport import org.apache.slider.server.appmaster.model.mock.MockRoleHistory import org.apache.slider.server.appmaster.state.NodeEntry import org.apache.slider.server.appmaster.state.NodeInstance @@ -72,7 +69,7 @@ class TestRoleHistoryAA extends SliderTestBase { } protected boolean setNodeState(NodeInstance node, NodeState state) { - node.updateNode(new MockNodeReport(node.hostname, state)) + node.updateNode(MockFactory.instance.newNodeReport(node.hostname, state)) } @Test @@ -84,7 +81,8 @@ class TestRoleHistoryAA extends SliderTestBase { @Test public void testFindSomeNodesSomeLabel() throws Throwable { // all three will surface at first - update(nodeMap, [new MockNodeReport("1", NodeState.RUNNING, "GPU")]) + update(nodeMap, + [MockFactory.instance.newNodeReport("1", NodeState.RUNNING, "GPU")]) def gpuNodes = nodeMap.findAllNodesForRole(1, "GPU") verifyResultSize(1, gpuNodes) def instance = gpuNodes[0] @@ -159,7 +157,7 @@ class TestRoleHistoryAA extends SliderTestBase { def role1 = node1.getOrCreate(1) def role2 = node1.getOrCreate(2) nodeMap.values().each { - it.updateNode(new MockNodeReport("0", NodeState.UNHEALTHY)) + it.updateNode(MockFactory.instance.newNodeReport("0", NodeState.UNHEALTHY)) } assertNoAvailableNodes(1) assertNoAvailableNodes(2) @@ -249,6 +247,6 @@ class TestRoleHistoryAA extends SliderTestBase { def NodeMap createNodeMap(List<String> hosts, NodeState state, String label = "") { - createNodeMap(MockNodeReport.createInstances(hosts, state, label)) + createNodeMap(MockFactory.instance.createNodeReports(hosts, state, label)) } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy deleted file mode 100644 index 4237f1e..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.mock - -import org.apache.hadoop.yarn.api.records.ContainerId -import org.apache.hadoop.yarn.api.records.ContainerState -import org.apache.hadoop.yarn.api.records.ContainerStatus -import org.apache.hadoop.yarn.api.records.Resource - -class MockContainerStatus extends ContainerStatus { - - ContainerId containerId - ContainerState state - String diagnostics - int exitStatus - - Resource capability; -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy index d873390..3ba6e31 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy @@ -23,8 +23,13 @@ import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.api.records.ApplicationAttemptId import org.apache.hadoop.yarn.api.records.ApplicationId import org.apache.hadoop.yarn.api.records.ContainerId +import org.apache.hadoop.yarn.api.records.ContainerState +import org.apache.hadoop.yarn.api.records.ContainerStatus import org.apache.hadoop.yarn.api.records.NodeId +import org.apache.hadoop.yarn.api.records.NodeReport +import org.apache.hadoop.yarn.api.records.NodeState import org.apache.hadoop.yarn.api.records.Priority +import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl import org.apache.hadoop.yarn.client.api.AMRMClient import org.apache.slider.api.ClusterDescription import org.apache.slider.api.ResourceKeys @@ -40,6 +45,8 @@ import org.apache.slider.providers.ProviderRole @Slf4j class MockFactory implements MockRoles { + public static MockFactory instance = new MockFactory(); + /* Ignore any IDE hints about needless references to the ROLE values; groovyc fails without them. */ @@ -239,7 +246,56 @@ class MockFactory implements MockRoles { return new MockResource(memory, vcores) } - MockContainerStatus newContainerStatus() { - return new MockContainerStatus() + ContainerStatus newContainerStatus() { + return newContainerStatus(null, null, "", 0) + } + + ContainerStatus newContainerStatus(ContainerId containerId, + ContainerState containerState, String diagnostics, int exitStatus) { + ContainerStatus.newInstance(containerId, containerState, diagnostics, exitStatus) + } + + /** + * Create a single instance + * @param hostname + * @param nodeState + * @param label + */ + NodeReport newNodeReport(String hostname, NodeState nodeState, String label = "") { + def nodeId = NodeId.newInstance(hostname, 80) + Integer.valueOf(hostname, 16) + newNodeReport(hostname, nodeId, nodeState, label) } + + NodeReport newNodeReport( + String hostname, + NodeId nodeId, + NodeState nodeState, + String label) { + def report = new NodeReportPBImpl(); + def nodeLabels = new HashSet<>() + nodeLabels.add(label) + report.nodeId = nodeId + report.nodeLabels = nodeLabels + report.nodeState = nodeState + report.httpAddress = "http$hostname:80" + report + } + + /** + * Create a list of instances -one for each hostname + * @param hostnames hosts + * @param nodeState state of all of them + * @param label label for all of them + * @return + */ + List<NodeReport> createNodeReports( + List<String> hostnames, + NodeState nodeState = NodeState.RUNNING, + String label = "") { + hostnames.collect { String name -> + newNodeReport(name, nodeState, label) + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy deleted file mode 100644 index 8c3b712..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.mock - -import groovy.transform.CompileStatic -import org.apache.hadoop.yarn.api.records.NodeId -import org.apache.hadoop.yarn.api.records.NodeReport -import org.apache.hadoop.yarn.api.records.NodeState -import org.apache.hadoop.yarn.api.records.Resource - -/** - * Node report for testing - */ -@CompileStatic -class MockNodeReport extends NodeReport { - NodeId nodeId; - NodeState nodeState; - String httpAddress; - String rackName; - Resource used; - Resource capability; - int numContainers; - String healthReport; - long lastHealthReportTime; - Set<String> nodeLabels; - - MockNodeReport() { - } - - /** - * Create a single instance - * @param hostname - * @param nodeState - * @param label - */ - MockNodeReport(String hostname, NodeState nodeState, String label ="") { - nodeId = NodeId.newInstance(hostname, 80) - Integer.valueOf(hostname, 16) - this.nodeState = nodeState - this.httpAddress = "http$hostname:80" - this.nodeLabels = new HashSet<>() - nodeLabels.add(label) - } - - /** - * Create a list of instances -one for each hostname - * @param hostnames hosts - * @param nodeState state of all of them - * @param label label for all of them - * @return - */ - static List<MockNodeReport> createInstances( - List<String> hostnames, - NodeState nodeState = NodeState.RUNNING, - String label = "") { - hostnames.collect { String name -> - new MockNodeReport(name, nodeState, label)} - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy new file mode 100644 index 0000000..a608e5a --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock + +import groovy.transform.CompileStatic +import org.apache.hadoop.yarn.api.records.ContainerId +import org.apache.hadoop.yarn.api.records.ContainerState +import org.apache.hadoop.yarn.api.records.ContainerStatus +import org.apache.hadoop.yarn.api.records.NodeId +import org.apache.hadoop.yarn.api.records.NodeReport +import org.apache.hadoop.yarn.api.records.NodeState +import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl + +/** + * Node report for testing + */ +@CompileStatic +class MockRecordFactory { + + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy index 265a796..e1dfa25 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy @@ -22,6 +22,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.api.records.ContainerId import org.apache.hadoop.yarn.api.records.NodeId +import org.apache.hadoop.yarn.api.records.NodeReport import org.apache.hadoop.yarn.api.records.NodeState /** @@ -149,7 +150,7 @@ public class MockYarnCluster { * Get the list of node reports. These are not cloned; updates will persist in the nodemap * @return current node report list */ - List<MockNodeReport> getNodeReports() { + List<NodeReport> getNodeReports() { nodes.collect { MockYarnClusterNode n -> n.nodeReport } } @@ -167,7 +168,7 @@ public class MockYarnCluster { public final MockNodeId nodeId; public final MockYarnClusterContainer[] containers; private boolean offline; - public MockNodeReport nodeReport + public NodeReport nodeReport public MockYarnClusterNode(int index, int size) { nodeIndex = index; @@ -181,9 +182,7 @@ public class MockYarnCluster { containers[i] = new MockYarnClusterContainer(mci) } - nodeReport = new MockNodeReport() - nodeReport.nodeId = nodeId - nodeReport.nodeState = NodeState.RUNNING + nodeReport = MockFactory.newNodeReport(hostname, nodeId, NodeState.RUNNING, "") } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy index 7ab97fa..d946c6b 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId import org.apache.hadoop.yarn.api.records.ApplicationId import org.apache.hadoop.yarn.api.records.Container import org.apache.hadoop.yarn.api.records.ContainerId +import org.apache.hadoop.yarn.api.records.NodeReport import org.apache.hadoop.yarn.client.api.AMRMClient import org.apache.slider.server.appmaster.operations.AbstractRMOperation import org.apache.slider.server.appmaster.operations.CancelSingleRequest @@ -166,7 +167,7 @@ class MockYarnEngine { * Get the list of node reports. These are not cloned; updates will persist in the nodemap * @return current node report list */ - List<MockNodeReport> getNodeReports() { + List<NodeReport> getNodeReports() { cluster.nodeReports } -} \ No newline at end of file +}
