http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationId.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationId.java new file mode 100644 index 0000000..01da470 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationId.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.ApplicationId; + +/** + * Mock app id. + */ +public class MockApplicationId extends ApplicationId { + + private int id; + private long clusterTimestamp; + + public MockApplicationId() { + } + + public MockApplicationId(int id) { + this.id = id; + } + + public MockApplicationId(int id, long clusterTimestamp) { + this.id = id; + this.clusterTimestamp = clusterTimestamp; + } + + @Override + public int getId() { + return id; + } + + @Override + public void setId(int id) { + this.id = id; + } + + @Override + public long getClusterTimestamp() { + return clusterTimestamp; + } + + @Override + public void setClusterTimestamp(long clusterTimestamp) { + this.clusterTimestamp = clusterTimestamp; + } + + @Override + public void build() { + + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockClusterServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockClusterServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockClusterServices.java new file mode 100644 index 0000000..2578595 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockClusterServices.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.slider.server.appmaster.state.AbstractClusterServices; + +/** + * Mock cluster services. + */ +public class MockClusterServices extends AbstractClusterServices { + + @Override + public Resource newResource() { + return new MockResource(0, 0); + } + + @Override + public Resource newResource(int memory, int cores) { + return new MockResource(memory, cores); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainer.java new file mode 100644 index 0000000..148b7f6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainer.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; + +/** + * Mock container. + */ +public class MockContainer extends Container { + + private ContainerId id; + private NodeId nodeId; + private String nodeHttpAddress; + private Resource resource; + private Priority priority; + private Token containerToken; + + @Override + public int compareTo(Container other) { + if (this.getId().compareTo(other.getId()) == 0) { + if (this.getNodeId().compareTo(other.getNodeId()) == 0) { + return this.getResource().compareTo(other.getResource()); + } else { + return this.getNodeId().compareTo(other.getNodeId()); + } + } else { + return this.getId().compareTo(other.getId()); + } + } + + @Override + public String toString() { + return "MockContainer{ id=" + id + + ", nodeHttpAddress='" + nodeHttpAddress + "'," + + " priority=" + priority + " }"; + } + + @Override + public ContainerId getId() { + return id; + } + + @Override + public void setId(ContainerId id) { + this.id = id; + } + + @Override + public NodeId getNodeId() { + return nodeId; + } + + @Override + public void setNodeId(NodeId nodeId) { + this.nodeId = nodeId; + } + + @Override + public String getNodeHttpAddress() { + return nodeHttpAddress; + } + + @Override + public void setNodeHttpAddress(String nodeHttpAddress) { + this.nodeHttpAddress = nodeHttpAddress; + } + + @Override + public Resource getResource() { + return resource; + } + + @Override + public void setResource(Resource resource) { + this.resource = resource; + } + + @Override + public Priority getPriority() { + return priority; + } + + @Override + public void setPriority(Priority priority) { + this.priority = priority; + } + + @Override + public Token getContainerToken() { + return containerToken; + } + + @Override + public void setContainerToken(Token containerToken) { + this.containerToken = containerToken; + } + + @Override + public ExecutionType getExecutionType() { + return null; + } + + @Override + public void setExecutionType(ExecutionType executionType) { + + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainerId.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainerId.java new file mode 100644 index 0000000..3cbc7e5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockContainerId.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ContainerId; + +/** + * Mock container id. + */ +public class MockContainerId extends ContainerId implements Cloneable { + + private static final MockApplicationAttemptId DEFAULT_APP_ATTEMPT_ID = + new MockApplicationAttemptId(new MockApplicationId(1), 1); + + private long containerId; + private ApplicationAttemptId applicationAttemptId; + + MockContainerId() { + } + + /** + * Sets up a default app Attempt ID. + * @param containerId + */ + MockContainerId(long containerId) { + this.containerId = containerId; + this.applicationAttemptId = DEFAULT_APP_ATTEMPT_ID; + } + + public MockContainerId(ApplicationAttemptId applicationAttemptId, + long containerId) { + this.containerId = containerId; + this.applicationAttemptId = applicationAttemptId; + } + + MockContainerId(ContainerId that) { + containerId = that.getContainerId(); + applicationAttemptId = that.getApplicationAttemptId(); + } + + @Deprecated + @Override + public int getId() { + return (int) containerId; + } + + // TODO: Temporarily adding it back + void setId(int id) { + containerId = (long) id; + } + + @Override + public long getContainerId() { + return this.containerId; + } + + @Override + public void setContainerId(long id) { + this.containerId = id; + } + + @Override + public ApplicationAttemptId getApplicationAttemptId() { + return applicationAttemptId; + } + + @Override + public void setApplicationAttemptId(ApplicationAttemptId + applicationAttemptId) { + this.applicationAttemptId = applicationAttemptId; + } + + @Override + public void build() { + + } + + @Override + public String toString() { + return "mockcontainer_" + containerId; + } + + @Override + protected Object clone() throws CloneNotSupportedException { + return super.clone(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java new file mode 100644 index 0000000..2ac5087 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFactory.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; +import org.apache.slider.providers.PlacementPolicy; +import org.apache.slider.providers.ProviderRole; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; + +import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY; + +/** + * Factory for creating things. + */ +public class MockFactory implements MockRoles { + + public static final int NODE_FAILURE_THRESHOLD = 2; + + public static final MockFactory INSTANCE = new MockFactory(); + + /** + * Basic role. + */ + public static final ProviderRole PROVIDER_ROLE0 = new ProviderRole( + MockRoles.ROLE0, + 0, + PlacementPolicy.DEFAULT, + NODE_FAILURE_THRESHOLD, + 1, + ResourceKeys.DEF_YARN_LABEL_EXPRESSION); + /** + * role 1 is strict. timeout should be irrelevant; same as failures + */ + public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole( + MockRoles.ROLE1, + 1, + PlacementPolicy.STRICT, + NODE_FAILURE_THRESHOLD, + 1, + ResourceKeys.DEF_YARN_LABEL_EXPRESSION); + + /** + * role 2: longer delay. + */ + public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole( + MockRoles.ROLE2, + 2, + PlacementPolicy.ANYWHERE, + NODE_FAILURE_THRESHOLD, + 2, + ResourceKeys.DEF_YARN_LABEL_EXPRESSION); + + /** + * Patch up a "role2" role to have anti-affinity set. + */ + public static final ProviderRole AAROLE_2 = new ProviderRole( + MockRoles.ROLE2, + 2, + PlacementPolicy.ANTI_AFFINITY_REQUIRED, + NODE_FAILURE_THRESHOLD, + 2, + null); + + /** + * Patch up a "role1" role to have anti-affinity set and GPI as the label. + */ + public static final ProviderRole AAROLE_1_GPU = new ProviderRole( + MockRoles.ROLE1, + 1, + PlacementPolicy.ANTI_AFFINITY_REQUIRED, + NODE_FAILURE_THRESHOLD, + 1, + MockRoles.LABEL_GPU); + + private int appIdCount; + private int attemptIdCount; + private int containerIdCount; + + private ApplicationId appId = newAppId(); + private ApplicationAttemptId attemptId = newApplicationAttemptId(appId); + + /** + * List of roles. + */ + public static final List<ProviderRole> ROLES = Arrays.asList( + PROVIDER_ROLE0, + PROVIDER_ROLE1, + PROVIDER_ROLE2 + ); + + public static final int ROLE_COUNT = ROLES.size(); + + MockContainerId newContainerId() { + return newContainerId(attemptId); + } + + MockContainerId newContainerId(ApplicationAttemptId attemptId0) { + MockContainerId cid = new MockContainerId(attemptId0, containerIdCount++); + return cid; + } + + MockApplicationAttemptId newApplicationAttemptId(ApplicationId appId0) { + MockApplicationAttemptId id = new MockApplicationAttemptId(appId0, + attemptIdCount++); + return id; + } + + MockApplicationId newAppId() { + MockApplicationId id = new MockApplicationId(); + id.setId(appIdCount++); + return id; + } + + public MockNodeId newNodeId(String host) { + return new MockNodeId(host); + } + + MockContainer newContainer(ContainerId cid) { + MockContainer c = new MockContainer(); + c.setId(cid); + return c; + } + + public MockContainer newContainer() { + return newContainer(newContainerId()); + } + + public MockContainer newContainer(NodeId nodeId, Priority priority) { + MockContainer container = newContainer(newContainerId()); + container.setNodeId(nodeId); + container.setPriority(priority); + return container; + } + + /** + * Build a new container using the request to supply priority and resource. + * @param req request + * @param host hostname to assign to + * @return the container + */ + public MockContainer newContainer(AMRMClient.ContainerRequest req, String + host) { + MockContainer container = newContainer(newContainerId()); + container.setResource(req.getCapability()); + container.setPriority(req.getPriority()); + container.setNodeId(new MockNodeId(host)); + return container; + } + + /** + * Create a new instance with the given components definined in the + * resources section. + * @param r1 + * @param r2 + * @param r3 + * @return + */ + public Application newApplication(long r1, long r2, long r3) { + Application application = new Application(); + application.getConfiguration().setProperty(ResourceKeys + .NODE_FAILURE_THRESHOLD, Integer.toString(NODE_FAILURE_THRESHOLD)); + List<Component> components = application.getComponents(); + Component c1 = new Component().name(ROLE0).numberOfContainers(r1); + c1.getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY, + Integer.toString(PlacementPolicy.DEFAULT)); + Component c2 = new Component().name(ROLE1).numberOfContainers(r2); + c2.getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY, + Integer.toString(PlacementPolicy.STRICT)); + Component c3 = new Component().name(ROLE2).numberOfContainers(r3); + c3.getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY, + Integer.toString(PlacementPolicy.ANYWHERE)); + components.add(c1); + components.add(c2); + components.add(c3); + return application; + } + + public MockResource newResource(int memory, int vcores) { + return new MockResource(memory, vcores); + } + + ContainerStatus newContainerStatus() { + return newContainerStatus(null, null, "", 0); + } + + ContainerStatus newContainerStatus(ContainerId containerId, + ContainerState containerState, String diagnostics, int exitStatus) { + return ContainerStatus.newInstance(containerId, containerState, + diagnostics, exitStatus); + } + + /** + * Create a single instance. + * @param hostname + * @param nodeState + * @param label + */ + public NodeReport newNodeReport(String hostname, NodeState nodeState, + String label) { + NodeId nodeId = NodeId.newInstance(hostname, 80); + Integer.valueOf(hostname, 16); + return newNodeReport(hostname, nodeId, nodeState, label); + } + + NodeReport newNodeReport( + String hostname, + NodeId nodeId, + NodeState nodeState, + String label) { + NodeReport report = new NodeReportPBImpl(); + HashSet<String> nodeLabels = new HashSet<>(); + nodeLabels.add(label); + report.setNodeId(nodeId); + report.setNodeLabels(nodeLabels); + report.setNodeState(nodeState); + report.setHttpAddress("http$hostname:80"); + return report; + } + + /** + * Create a list of instances -one for each hostname. + * @param hostnames hosts + * @return + */ + public List<NodeReport> createNodeReports( + List<String> hostnames, NodeState nodeState, String label) { + if (nodeState == null) { + nodeState = NodeState.RUNNING; + } + List<NodeReport> reports = new ArrayList<>(); + for (String name : hostnames) { + reports.add(newNodeReport(name, nodeState, label)); + } + return reports; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFileSystem.java new file mode 100644 index 0000000..72d1665 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockFileSystem.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.fs.FilterFileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; + +/** + * + */ +class MockFileSystem extends FilterFileSystem{ + @Override + public Path resolvePath(Path p) throws IOException { + return new Path("hdfs://localhost/", p); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockNodeId.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockNodeId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockNodeId.java new file mode 100644 index 0000000..9d2379a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockNodeId.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.NodeId; + +/** + * Mock node id. + */ +public class MockNodeId extends NodeId { + private String host; + private int port; + + public MockNodeId() { + } + + MockNodeId(String host) { + this.host = host; + } + + public MockNodeId(String host, int port) { + this.host = host; + this.port = port; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + @Override + protected void build() { + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockPriority.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockPriority.java new file mode 100644 index 0000000..36f97cc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockPriority.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.Priority; + +/** + * Mock priority. + */ +public class MockPriority extends Priority { + + private int priority; + + public MockPriority(int priority) { + this.priority = priority; + } + + MockPriority() { + } + + @Override + public int getPriority() { + return priority; + } + + @Override + public void setPriority(int priority) { + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java new file mode 100644 index 0000000..112a5ac --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockProviderService.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.LifecycleEvent; +import org.apache.hadoop.service.ServiceStateChangeListener; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.slider.api.resource.Application; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.providers.ProviderRole; +import org.apache.slider.providers.ProviderService; +import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Mock provider service. + */ +public class MockProviderService implements ProviderService { + + @Override + public String getName() { + return null; + } + + @Override + public void init(Configuration config) { + } + + @Override + public void start() { + } + + @Override + public void stop() { + } + + @Override + public void close() throws IOException { + } + + @Override + public void registerServiceListener(ServiceStateChangeListener listener) { + } + + @Override + public void unregisterServiceListener(ServiceStateChangeListener listener) { + } + + @Override + public Configuration getConfig() { + return null; + } + + public STATE getServiceState() { + return null; + } + + @Override + public long getStartTime() { + return 0; + } + + @Override + public boolean isInState(STATE state) { + return false; + } + + @Override + public Throwable getFailureCause() { + return null; + } + + @Override + public STATE getFailureState() { + return null; + } + + @Override + public boolean waitForServiceToStop(long timeout) { + return false; + } + + @Override + public List<LifecycleEvent> getLifecycleHistory() { + return null; + } + + @Override + public Map<String, String> getBlockers() { + return null; + } + + @Override + public void buildContainerLaunchContext(ContainerLauncher containerLauncher, + Application application, Container container, ProviderRole providerRole, + SliderFileSystem sliderFileSystem) throws IOException, SliderException { + + } + + @Override + public void setAMState(StateAccessForProviders stateAccessForProviders) { + + } + + @Override + public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) { + + } + + @Override + public boolean processContainerStatus(ContainerId containerId, + ContainerStatus status) { + return false; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.java new file mode 100644 index 0000000..3dd764a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; +import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; +import org.apache.slider.server.appmaster.operations.RMOperationHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * Mock RM operation handler. + */ +public class MockRMOperationHandler extends RMOperationHandler { + protected static final Logger LOG = + LoggerFactory.getLogger(MockRMOperationHandler.class); + + private List<AbstractRMOperation> operations = new ArrayList<>(); + private int requests; + private int releases; + // number available to cancel + private int availableToCancel = 0; + // count of cancelled values. This must be explicitly set + private int cancelled; + // number blacklisted + private int blacklisted = 0; + + @Override + public void releaseAssignedContainer(ContainerId containerId) { + operations.add(new ContainerReleaseOperation(containerId)); + LOG.info("Releasing container ID " + containerId.getContainerId()); + releases++; + } + + @Override + public void addContainerRequest(AMRMClient.ContainerRequest req) { + operations.add(new ContainerRequestOperation(req)); + LOG.info("Requesting container role #" + req.getPriority()); + requests++; + } + + @Override + public int cancelContainerRequests( + Priority priority1, + Priority priority2, + int count) { + int releaseable = Math.min(count, availableToCancel); + availableToCancel -= releaseable; + cancelled += releaseable; + return releaseable; + } + + @Override + public void cancelSingleRequest(AMRMClient.ContainerRequest request) { + // here assume that there is a copy of this request in the list + if (availableToCancel > 0) { + availableToCancel--; + cancelled++; + } + } + + @Override + public void updateBlacklist(List<String> blacklistAdditions, List<String> + blacklistRemovals) { + blacklisted += blacklistAdditions.size(); + blacklisted -= blacklistRemovals.size(); + } + + /** + * Clear the history. + */ + public void clear() { + operations.clear(); + releases = 0; + requests = 0; + } + + public AbstractRMOperation getFirstOp() { + return operations.get(0); + } + + public int getNumReleases() { + return releases; + } + + public void setAvailableToCancel(int num) { + this.availableToCancel = num; + } + + public int getAvailableToCancel() { + return availableToCancel; + } + + public int getBlacklisted() { + return blacklisted; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.java new file mode 100644 index 0000000..eb34586 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +/** + * Node report for testing. + */ +class MockRecordFactory { + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.java new file mode 100644 index 0000000..4917f1b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.types.RegistryPathStatus; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.service.AbstractService; + +import java.util.List; + +/** + * Simple stub registry for when one is needed for its API, but the operations + * are not actually required. + */ +class MockRegistryOperations extends AbstractService implements + RegistryOperations { + + MockRegistryOperations() { + super("mock"); + } + + @Override + public boolean mknode(String path, boolean createParents) { + return true; + } + + @Override + public void bind(String path, ServiceRecord record, int flags) { + } + + @Override + public ServiceRecord resolve(String path) throws PathNotFoundException { + throw new PathNotFoundException(path); + } + + @Override + public RegistryPathStatus stat(String path) throws PathNotFoundException { + throw new PathNotFoundException(path); + } + + @Override + public boolean exists(String path) { + return false; + } + + @Override + public List<String> list(String path) throws PathNotFoundException { + throw new PathNotFoundException(path); + } + + @Override + public void delete(String path, boolean recursive) { + + } + + @Override + public boolean addWriteAccessor(String id, String pass) { + return true; + } + + @Override + public void clearWriteAccessors() { + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockResource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockResource.java new file mode 100644 index 0000000..3a2ccd7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockResource.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.Resource; + +/** + * Mock resource. + */ +public class MockResource extends Resource { + private int memory; + private int virtualCores; + + public MockResource(int memory, int vcores) { + this.memory = memory; + this.virtualCores = vcores; + } + + @Override + public int compareTo(Resource other) { + long diff = this.getMemorySize() - other.getMemorySize(); + if (diff == 0) { + diff = this.getVirtualCores() - other.getVirtualCores(); + } + return diff == 0 ? 0 : (diff > 0 ? 1 : -1); + } + + @Override + public long getMemorySize() { + return memory; + } + + @Override + public void setMemorySize(long memorySize) { + memory = (int) memorySize; + } + + @Override + public int getVirtualCores() { + return virtualCores; + } + + @Override + public void setVirtualCores(int vCores) { + this.virtualCores = vCores; + } + + @Deprecated + @Override + public int getMemory() { + return memory; + } + + @Deprecated + @Override + public void setMemory(int memory) { + this.memory = memory; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.java new file mode 100644 index 0000000..8e88b0d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.providers.ProviderRole; +import org.apache.slider.server.appmaster.state.RoleHistory; +import org.apache.slider.server.appmaster.state.RoleStatus; + +import java.util.ArrayList; +import java.util.List; + +/** + * Subclass to enable access to some of the protected methods. + */ +public class MockRoleHistory extends RoleHistory { + + /** + * Take a list of provider roles and build the history from them, + * dynamically creating the role status entries on the way. + * @param providerRoles provider role list + * @throws BadConfigException configuration problem with the role list + */ + public MockRoleHistory(List<ProviderRole> providerRoles) throws + BadConfigException { + super(convertRoles(providerRoles), new MockClusterServices()); + } + + static List<RoleStatus> convertRoles(List<ProviderRole> providerRoles) { + List<RoleStatus> statuses = new ArrayList<>(); + for (ProviderRole role : providerRoles) { + statuses.add(new RoleStatus(role)); + } + return statuses; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoles.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoles.java new file mode 100644 index 0000000..bad82bd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockRoles.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.appmaster.model.mock; + +/** + * Mock role constants. + */ +public interface MockRoles { + + String ROLE0 = "role0"; + String ROLE1 = "role1"; + String ROLE2 = "role2"; + String LABEL_GPU = "gpu"; + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.java new file mode 100644 index 0000000..6b685a0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * Models the cluster itself: a set of mock cluster nodes. + * + * nodes retain the slot model with a limit of 2^8 slots/host -this + * lets us use 24 bits of the container ID for hosts, and so simulate + * larger hosts. + * + * upper 32: index into nodes in the cluster + * NodeID hostname is the index in hex format; this is parsed down to the index + * to resolve the host + * + * Important: container IDs will be reused as containers get recycled. This + * is not an attempt to realistically mimic a real YARN cluster, just + * simulate it enough for Slider to explore node re-use and its handling + * of successful and unsuccessful allocations. + * + * There is little or no checking of valid parameters in here -this is for + * test use, not production. + */ +public class MockYarnCluster { + protected static final Logger LOG = + LoggerFactory.getLogger(MockYarnCluster.class); + + private final int clusterSize; + private final int containersPerNode; + private MockYarnClusterNode[] nodes; + + MockYarnCluster(int clusterSize, int containersPerNode) { + this.clusterSize = clusterSize; + this.containersPerNode = containersPerNode; + build(); + } + + public int getClusterSize() { + return clusterSize; + } + + @Override + public String toString() { + return "MockYarnCluster size=" + clusterSize + ", capacity=" + + totalClusterCapacity()+ ", in use=" + containersInUse(); + } + + /** + * Build the cluster. + */ + private void build() { + nodes = new MockYarnClusterNode[clusterSize]; + for (int i = 0; i < clusterSize; i++) { + nodes[i] = new MockYarnClusterNode(i, containersPerNode); + } + } + + public MockYarnClusterNode nodeAt(int index) { + return nodes[index]; + } + + MockYarnClusterNode lookup(String hostname) { + int index = Integer.valueOf(hostname, 16); + return nodeAt(index); + } + + MockYarnClusterNode lookup(NodeId nodeId) { + return lookup(nodeId.getHost()); + } + + MockYarnClusterNode lookupOwner(ContainerId cid) { + return nodeAt(extractHost(cid.getContainerId())); + } + + /** + * Release a container: return true if it was actually in use. + * @param cid container ID + * @return the container released + */ + MockYarnClusterContainer release(ContainerId cid) { + int host = extractHost(cid.getContainerId()); + MockYarnClusterContainer inUse = nodeAt(host).release(cid.getContainerId()); + LOG.debug("Released {} inuse={}", cid, inUse); + return inUse; + } + + int containersInUse() { + int count = 0; + for (MockYarnClusterNode it : nodes) { + count += it.containersInUse(); + } + return count; + } + + /** + * Containers free. + * @return + */ + int containersFree() { + return totalClusterCapacity() - containersInUse(); + } + + int totalClusterCapacity() { + return clusterSize * containersPerNode; + } + + /** + * Reset all the containers. + */ + public void reset() { + for (MockYarnClusterNode node : nodes) { + node.reset(); + } + } + + /** + * Bulk allocate the specific number of containers on a range of the cluster. + * @param startNode start of the range + * @param endNode end of the range + * @param count count + * @return the number actually allocated -it will be less the count supplied + * if the node was full + */ + public int bulkAllocate(int startNode, int endNode, int count) { + int total = 0; + for (int i = startNode; i <= endNode; i++) { + total += nodeAt(i).bulkAllocate(count).size(); + } + return total; + } + + /** + * Get the list of node reports. These are not cloned; updates will persist + * in the nodemap + * @return current node report list + */ + List<NodeReport> getNodeReports() { + List<NodeReport> reports = new ArrayList<>(); + + for (MockYarnClusterNode n : nodes) { + reports.add(n.nodeReport); + } + return reports; + } + + /** + * Model cluster nodes on the simpler "slot" model than the YARN-era + * resource allocation model. Why? Easier to implement scheduling. + * Of course, if someone does want to implement the full process... + * + */ + public static class MockYarnClusterNode { + + private final int nodeIndex; + private final String hostname; + private List<String> labels = new ArrayList<>(); + private final MockNodeId nodeId; + private final MockYarnClusterContainer[] containers; + private boolean offline; + private NodeReport nodeReport; + + public MockYarnClusterNode(int index, int size) { + nodeIndex = index; + hostname = String.format(Locale.ENGLISH, "%08x", index); + nodeId = new MockNodeId(hostname, 0); + + containers = new MockYarnClusterContainer[size]; + for (int i = 0; i < size; i++) { + int cid = makeCid(index, i); + MockContainerId mci = new MockContainerId(cid); + containers[i] = new MockYarnClusterContainer(mci); + } + + nodeReport = MockFactory.INSTANCE.newNodeReport(hostname, nodeId, + NodeState.RUNNING, ""); + } + + public String getHostname() { + return hostname; + } + + public NodeId getNodeId() { + return nodeId; + } + + /** + * Look up a container. + * @param containerId + * @return + */ + public MockYarnClusterContainer lookup(int containerId) { + return containers[extractContainer(containerId)]; + } + + /** + * Go offline; release all containers. + */ + public void goOffline() { + if (!offline) { + offline = true; + reset(); + } + } + + public void goOnline() { + offline = false; + } + + /** + * Allocate a container -if one is available. + * @return the container or null for none free + * -or the cluster node is offline + */ + public MockYarnClusterContainer allocate() { + if (!offline) { + for (int i = 0; i < containers.length; i++) { + MockYarnClusterContainer c = containers[i]; + if (!c.busy) { + c.busy = true; + return c; + } + } + } + return null; + } + + /** + * Bulk allocate the specific number of containers. + * @param count count + * @return the list actually allocated -it will be less the count supplied + * if the node was full + */ + public List<MockYarnClusterContainer> bulkAllocate(int count) { + List<MockYarnClusterContainer> result = new ArrayList<>(); + for (int i = 0; i < count; i++) { + MockYarnClusterContainer allocation = allocate(); + if (allocation == null) { + break; + } + result.add(allocation); + } + return result; + } + + /** + * Release a container. + * @param cid container ID + * @return the container if the container was busy before the release + */ + public MockYarnClusterContainer release(long cid) { + MockYarnClusterContainer container = containers[extractContainer(cid)]; + boolean b = container.busy; + container.busy = false; + return b? container: null; + } + + public String httpAddress() { + return "http://$hostname/";; + } + + /** + * Reset all the containers. + */ + public void reset() { + for (MockYarnClusterContainer cont : containers) { + cont.reset(); + } + } + + public int containersInUse() { + int c = 0; + for (MockYarnClusterContainer cont : containers) { + c += cont.busy ? 1 : 0; + } + return c; + } + + public int containersFree() { + return containers.length - containersInUse(); + } + } + + /** + * Cluster container. + */ + public static class MockYarnClusterContainer { + private MockContainerId cid; + private boolean busy; + + MockYarnClusterContainer(MockContainerId cid) { + this.cid = cid; + } + + public MockContainerId getCid() { + return cid; + } + + void reset() { + busy = false; + } + } + + public static int makeCid(int hostIndex, int containerIndex) { + return (hostIndex << 8) | containerIndex & 0xff; + } + + public static final int extractHost(long cid) { + return (int)((cid >>> 8) & 0xffff); + } + + public static final int extractContainer(long cid) { + return (int)(cid & 0xff); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java new file mode 100644 index 0000000..9c5708f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.operations.CancelSingleRequest; +import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; +import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; +import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertNotNull; + +/** + * This is an evolving engine to mock YARN operations. + */ +public class MockYarnEngine { + private static final Logger LOG = + LoggerFactory.getLogger(MockYarnEngine.class); + + private MockYarnCluster cluster; + private Allocator allocator; + private List<ContainerRequestOperation> pending = new ArrayList<>(); + + private ApplicationId appId = new MockApplicationId(0, 0); + + private ApplicationAttemptId attemptId = new MockApplicationAttemptId(appId, + 1); + + @Override + public String toString() { + return "MockYarnEngine " + cluster + " + pending=" + pending.size(); + } + + public int containerCount() { + return cluster.containersInUse(); + } + + public MockYarnEngine(int clusterSize, int containersPerNode) { + cluster = new MockYarnCluster(clusterSize, containersPerNode); + allocator = new Allocator(cluster); + } + + public MockYarnCluster getCluster() { + return cluster; + } + + public Allocator getAllocator() { + return allocator; + } + + /** + * Allocate a container from a request. The containerID will be + * unique, nodeId and other fields chosen internally with + * no such guarantees; resource and priority copied over + * @param request request + * @return container + */ + public Container allocateContainer(AMRMClient.ContainerRequest request) { + MockContainer allocated = allocator.allocate(request); + if (allocated != null) { + MockContainerId id = (MockContainerId)allocated.getId(); + id.setApplicationAttemptId(attemptId); + } + return allocated; + } + + MockYarnCluster.MockYarnClusterContainer releaseContainer(ContainerId + containerId) { + return cluster.release(containerId); + } + + /** + * Process a list of operations -release containers to be released, + * allocate those for which there is space (but don't rescan the list after + * the scan). + * @param ops + * @return + */ + public List<Container> execute(List<AbstractRMOperation> ops) { + return execute(ops, new ArrayList<>()); + } + + /** + * Process a list of operations -release containers to be released, + * allocate those for which there is space (but don't rescan the list after + * the scan). Unsatisifed entries are appended to the "pending" list + * @param ops operations + * @return the list of all satisfied operations + */ + public List<Container> execute(List<AbstractRMOperation> ops, + List<ContainerId> released) { + validateRequests(ops); + List<Container> allocation = new ArrayList<>(); + for (AbstractRMOperation op : ops) { + if (op instanceof ContainerReleaseOperation) { + ContainerReleaseOperation cro = (ContainerReleaseOperation) op; + ContainerId cid = cro.getContainerId(); + assertNotNull(releaseContainer(cid)); + released.add(cid); + } else if (op instanceof CancelSingleRequest) { + // no-op + LOG.debug("cancel request {}", op); + } else if (op instanceof ContainerRequestOperation) { + ContainerRequestOperation req = (ContainerRequestOperation) op; + Container container = allocateContainer(req.getRequest()); + if (container != null) { + LOG.info("allocated container {} for {}", container, req); + allocation.add(container); + } else { + LOG.debug("Unsatisfied allocation {}", req); + pending.add(req); + } + } else { + LOG.warn("Unsupported operation {}", op); + } + } + return allocation; + } + + /** + * Try and mimic some of the logic of <code>AMRMClientImpl + * .checkLocalityRelaxationConflict</code>. + * @param ops operations list + */ + void validateRequests(List<AbstractRMOperation> ops) { + // run through the requests and verify that they are all consistent. + List<ContainerRequestOperation> outstandingRequests = new ArrayList<>(); + for (AbstractRMOperation operation : ops) { + if (operation instanceof ContainerRequestOperation) { + ContainerRequestOperation containerRequest = + (ContainerRequestOperation) operation; + ContainerRequest amRequest = containerRequest.getRequest(); + Priority priority = amRequest.getPriority(); + boolean relax = amRequest.getRelaxLocality(); + + for (ContainerRequestOperation req : outstandingRequests) { + if (req.getPriority() == priority && req.getRelaxLocality() != + relax) { + // mismatch in values + Assert.fail("operation " + operation + " has incompatible request" + + " priority from outsanding request"); + } + outstandingRequests.add(containerRequest); + + } + + } + } + } + + /** + * Get the list of node reports. These are not cloned; updates will persist + * in the nodemap. + * @return current node report list + */ + List<NodeReport> getNodeReports() { + return cluster.getNodeReports(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.java new file mode 100644 index 0000000..31f8822 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.monkey; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.slider.api.InternalKeys; +import org.apache.slider.server.appmaster.actions.ActionHalt; +import org.apache.slider.server.appmaster.actions.ActionKillContainer; +import org.apache.slider.server.appmaster.actions.AsyncAction; +import org.apache.slider.server.appmaster.actions.QueueService; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler; +import org.apache.slider.server.appmaster.monkey.ChaosKillAM; +import org.apache.slider.server.appmaster.monkey.ChaosKillContainer; +import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService; +import org.apache.slider.server.appmaster.monkey.ChaosTarget; +import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Test chaos monkey. + */ +public class TestMockMonkey extends BaseMockAppStateTest { + private static final Logger LOG = + LoggerFactory.getLogger(TestMockMonkey.class); + + /** + * This queue service is NOT started; tests need to poll the queue + * rather than expect them to execute. + */ + private QueueService queues; + private ChaosMonkeyService monkey; + + @Before + public void init() { + YarnConfiguration configuration = new YarnConfiguration(); + queues = new QueueService(); + queues.init(configuration); + monkey = new ChaosMonkeyService(METRICS.getMetrics(), queues); + monkey.init(configuration); + } + + @Test + public void testMonkeyStart() throws Throwable { + monkey.start(); + monkey.stop(); + } + + @Test + public void testMonkeyPlay() throws Throwable { + ChaosCounter counter = new ChaosCounter(); + monkey.addTarget("target", counter, InternalKeys.PROBABILITY_PERCENT_100); + assertEquals(1, monkey.getTargetCount()); + monkey.play(); + assertEquals(1, counter.count); + } + + @Test + public void testMonkeySchedule() throws Throwable { + ChaosCounter counter = new ChaosCounter(); + assertEquals(0, monkey.getTargetCount()); + monkey.addTarget("target", counter, InternalKeys.PROBABILITY_PERCENT_100); + assertEquals(1, monkey.getTargetCount()); + assertTrue(monkey.schedule(0, 1, TimeUnit.SECONDS)); + assertEquals(1, queues.scheduledActions.size()); + } + + @Test + public void testMonkeyDoesntAddProb0Actions() throws Throwable { + ChaosCounter counter = new ChaosCounter(); + monkey.addTarget("target", counter, 0); + assertEquals(0, monkey.getTargetCount()); + monkey.play(); + assertEquals(0, counter.count); + } + + @Test + public void testMonkeyScheduleProb0Actions() throws Throwable { + ChaosCounter counter = new ChaosCounter(); + monkey.addTarget("target", counter, 0); + assertFalse(monkey.schedule(0, 1, TimeUnit.SECONDS)); + assertEquals(0, queues.scheduledActions.size()); + } + + @Test + public void testMonkeyPlaySometimes() throws Throwable { + ChaosCounter counter = new ChaosCounter(); + ChaosCounter counter2 = new ChaosCounter(); + monkey.addTarget("target1", counter, InternalKeys.PROBABILITY_PERCENT_1 + * 50); + monkey.addTarget("target2", counter2, InternalKeys + .PROBABILITY_PERCENT_1 * 25); + + for (int i = 0; i < 100; i++) { + monkey.play(); + } + LOG.info("Counter1 = {} counter2 = {}", counter.count, counter2.count); + /* + * Relying on probability here to give approximate answers + */ + assertTrue(counter.count > 25); + assertTrue(counter.count < 75); + assertTrue(counter2.count < counter.count); + } + + @Test + public void testAMKiller() throws Throwable { + + ChaosKillAM chaos = new ChaosKillAM(queues, -1); + chaos.chaosAction(); + assertEquals(1, queues.scheduledActions.size()); + AsyncAction action = queues.scheduledActions.take(); + assertTrue(action instanceof ActionHalt); + } + + @Test + public void testContainerKillerEmptyApp() throws Throwable { + + + ChaosKillContainer chaos = new ChaosKillContainer(appState, + queues, + new MockRMOperationHandler()); + chaos.chaosAction(); + assertEquals(0, queues.scheduledActions.size()); + } + + @Ignore + @Test + public void testContainerKillerIgnoresAM() throws Throwable { + // TODO: AM needed in live container list? + addAppMastertoAppState(); + assertEquals(1, appState.getLiveContainers().size()); + + ChaosKillContainer chaos = new ChaosKillContainer(appState, + queues, + new MockRMOperationHandler()); + chaos.chaosAction(); + assertEquals(0, queues.scheduledActions.size()); + } + + @Test + public void testContainerKiller() throws Throwable { + MockRMOperationHandler ops = new MockRMOperationHandler(); + getRole0Status().setDesired(1); + List<RoleInstance> instances = createAndStartNodes(); + assertEquals(1, instances.size()); + RoleInstance instance = instances.get(0); + + ChaosKillContainer chaos = new ChaosKillContainer(appState, queues, ops); + chaos.chaosAction(); + assertEquals(1, queues.scheduledActions.size()); + AsyncAction action = queues.scheduledActions.take(); + ActionKillContainer killer = (ActionKillContainer) action; + assertEquals(killer.getContainerId(), instance.getContainerId()); + killer.execute(null, queues, appState); + assertEquals(1, ops.getNumReleases()); + + ContainerReleaseOperation operation = (ContainerReleaseOperation) ops + .getFirstOp(); + assertEquals(operation.getContainerId(), instance.getContainerId()); + } + + /** + * Chaos target that just implements a counter. + */ + private static class ChaosCounter implements ChaosTarget { + private int count; + + @Override + public void chaosAction() { + count++; + } + + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder( + "ChaosCounter{"); + sb.append("count=").append(count); + sb.append('}'); + return sb.toString(); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/security/TestSecurityConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/security/TestSecurityConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/security/TestSecurityConfiguration.java new file mode 100644 index 0000000..5a19a3a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/security/TestSecurityConfiguration.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.appmaster.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.slider.api.resource.Application; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.core.exceptions.SliderException; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Test security configuration. + */ +public class TestSecurityConfiguration { + + @Test + public void testValidLocalConfiguration() throws Throwable { + Configuration config = new Configuration(); + config.set(CommonConfigurationKeysPublic + .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + Map<String, String> compOps = new HashMap<>(); + compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test"); + compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, + "/some/local/path"); + Application application = new Application().configuration(new org.apache + .slider.api.resource.Configuration().properties(compOps)); + + SecurityConfiguration securityConfiguration = + new SecurityConfiguration(config, application, "testCluster"); + } + + @Test + public void testValidDistributedConfiguration() throws Throwable { + Configuration config = new Configuration(); + config.set(CommonConfigurationKeysPublic + .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + Map<String, String> compOps = new HashMap<>(); + compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test"); + compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); + Application application = new Application().configuration(new org.apache + .slider.api.resource.Configuration().properties(compOps)); + + SecurityConfiguration securityConfiguration = + new SecurityConfiguration(config, application, "testCluster"); + } + + @Test + public void testMissingPrincipalNoLoginWithDistributedConfig() throws + Throwable { + Configuration config = new Configuration(); + config.set(CommonConfigurationKeysPublic + .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + Map<String, String> compOps = new HashMap<>(); + compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); + Application application = new Application().configuration(new org.apache + .slider.api.resource.Configuration().properties(compOps)); + + try { + SecurityConfiguration securityConfiguration = + new SecurityConfiguration(config, application, "testCluster") { + @Override + protected UserGroupInformation getLoginUser() throws + IOException { + return null; + } + }; + fail("expected SliderException"); + } catch (SliderException e) { + // expected + } + } + + @Test + public void testMissingPrincipalNoLoginWithLocalConfig() throws Throwable { + Configuration config = new Configuration(); + config.set(CommonConfigurationKeysPublic + .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + Map<String, String> compOps = new HashMap<>(); + compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, + "/some/local/path"); + Application application = new Application().configuration(new org.apache + .slider.api.resource.Configuration().properties(compOps)); + + try { + SecurityConfiguration securityConfiguration = + new SecurityConfiguration(config, application, "testCluster") { + @Override + protected UserGroupInformation getLoginUser() throws IOException { + return null; + } + }; + fail("expected SliderException"); + } catch (SliderException e) { + // expected + } + } + + @Test + public void testBothKeytabMechanismsConfigured() throws Throwable { + Configuration config = new Configuration(); + config.set(CommonConfigurationKeysPublic + .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + Map<String, String> compOps = new HashMap<>(); + compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test"); + compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, + "/some/local/path"); + compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); + Application application = new Application().configuration(new org.apache + .slider.api.resource.Configuration().properties(compOps)); + + try { + SecurityConfiguration securityConfiguration = + new SecurityConfiguration(config, application, + "testCluster"); + fail("expected SliderException"); + } catch (SliderException e) { + // expected + } + } + + @Test + public void testMissingPrincipalButLoginWithDistributedConfig() throws + Throwable { + Configuration config = new Configuration(); + config.set(CommonConfigurationKeysPublic + .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + Map<String, String> compOps = new HashMap<>(); + compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); + Application application = new Application().configuration(new org.apache + .slider.api.resource.Configuration().properties(compOps)); + + SecurityConfiguration securityConfiguration = + new SecurityConfiguration(config, application, "testCluster"); + } + + @Test + public void testMissingPrincipalButLoginWithLocalConfig() throws Throwable { + Configuration config = new Configuration(); + config.set(CommonConfigurationKeysPublic + .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + Map<String, String> compOps = new HashMap<>(); + compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, + "/some/local/path"); + Application application = new Application().configuration(new org.apache + .slider.api.resource.Configuration().properties(compOps)); + + SecurityConfiguration securityConfiguration = + new SecurityConfiguration(config, application, "testCluster"); + } + + @Test + public void testKeypathLocationOnceLocalized() throws Throwable { + Configuration config = new Configuration(); + config.set(CommonConfigurationKeysPublic + .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + Map<String, String> compOps = new HashMap<>(); + compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); + Application application = new Application().configuration(new org.apache + .slider.api.resource.Configuration().properties(compOps)); + + SecurityConfiguration securityConfiguration = + new SecurityConfiguration(config, application, "testCluster"); + + assertEquals(new File(SliderKeys.KEYTAB_DIR, "some.keytab") + .getAbsolutePath(), + securityConfiguration.getKeytabFile().getAbsolutePath()); + } + + @Test + public void testAMKeytabProvided() throws Throwable { + Configuration config = new Configuration(); + Map<String, String> compOps = new HashMap<>(); + compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, " "); + Application application = new Application().configuration(new org.apache + .slider.api.resource.Configuration().properties(compOps)); + + SecurityConfiguration securityConfiguration = + new SecurityConfiguration(config, application, "testCluster"); + assertFalse(securityConfiguration.isKeytabProvided()); + + compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, ""); + assertFalse(securityConfiguration.isKeytabProvided()); + + compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); + assertTrue(securityConfiguration.isKeytabProvided()); + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
