Repository: incubator-myriad Updated Branches: refs/heads/master df4cbc0e9 -> 7aea259cf
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java index 87aee56..b6806a2 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java @@ -18,53 +18,106 @@ */ package org.apache.myriad.scheduler; +import org.apache.commons.lang.StringUtils; +import org.apache.mesos.Protos; import static org.junit.Assert.assertTrue; import org.apache.mesos.Protos.CommandInfo; +import org.apache.myriad.configuration.ServiceConfiguration; import org.apache.myriad.BaseConfigurableTest; -import org.apache.myriad.scheduler.TaskFactory.NMTaskFactoryImpl; import org.junit.Test; +import java.util.*; + /** * Class to test CommandLine generation */ public class TestServiceCommandLine extends BaseConfigurableTest { + + public static final String KEY_NM_ADDRESS = "myriad.yarn.nodemanager.address"; + public static final String KEY_NM_LOCALIZER_ADDRESS = "myriad.yarn.nodemanager.localizer.address"; + public static final String KEY_NM_WEBAPP_ADDRESS = "myriad.yarn.nodemanager.webapp.address"; + public static final String KEY_NM_SHUFFLE_PORT = "myriad.mapreduce.shuffle.port"; + + public static final String KEY_JHS_WEBAPP_ADDRESS = "myriad.mapreduce.jobhistory.webapp.address"; + public static final String KEY_JHS_ADMIN_ADDRESS = "myriad.mapreduce.jobhistory.admin.address"; + public static final String KEY_JHS_ADDRESS = "myriad.mapreduce.jobhistory.address"; + + private static final String msgFormat = System.lineSeparator() + "%s" + System.lineSeparator() + "!=" + + System.lineSeparator() + "%s"; + protected static final String CMD_FORMAT = "echo \"%1$s\" && %1$s"; static String toJHSCompare = - "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; " + - "export TASK_DIR=`basename $PWD`; sudo chmod +wx /sys/fs/cgroup/cpu/mesos/$TASK_DIR;" + - "sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver\"; sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo cp" + - " conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver"; + " sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp yarnConfiguration /usr/local/hadoop/etc/hadoop/yarn-site.xml && " + + "sudo -E -u hduser -H bin/mapred historyserver"; static String toCompare = - "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;"; + " sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp yarnConfiguration /usr/local/hadoop/etc/hadoop/yarn-site.xml && " + + "sudo -E -u hduser -H $YARN_HOME/bin/yarn nodemanager"; @Test public void testJHSCommandLineGeneration() throws Exception { - ServiceTaskFactoryImpl jhs = new ServiceTaskFactoryImpl(cfg, null); - String executorCmd = "$YARN_HOME/bin/mapred historyserver"; - ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0); + Map<String, Long> portsMap = new TreeMap<>(); + portsMap.put(KEY_JHS_ADDRESS, 0L); + portsMap.put(KEY_JHS_WEBAPP_ADDRESS, 3L); + portsMap.put(KEY_JHS_ADMIN_ADDRESS, 0L); + + ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0, portsMap); + ServiceConfiguration serviceConfiguration = cfg.getServiceConfiguration("jobhistory").get(); + ServiceCommandLineGenerator serviceCommandLineGenerator = new ServiceCommandLineGenerator(cfg); + List<Long> ports = new ArrayList<>(); + ports.add(2L); + ports.add(1L); + ports.add(3L); - CommandInfo cInfo = jhs.createCommandInfo(profile, executorCmd); - System.out.println(toJHSCompare); - System.out.println(cInfo.getValue()); + CommandInfo cInfo = serviceCommandLineGenerator.generateCommandLine(profile, + serviceConfiguration, + ports); + String testVal = String.format(CMD_FORMAT, toJHSCompare); + assertTrue(String.format(msgFormat, cInfo.getValue(), testVal), + cInfo.getValue().equals(testVal)); - assertTrue(cInfo.getValue().startsWith(toCompare)); + List<Protos.Environment.Variable> environmentList = cInfo.getEnvironment().getVariablesList(); + String yarnOpts = ""; + for (Protos.Environment.Variable variable: environmentList) { + if (variable.getName().equals(ServiceCommandLineGenerator.ENV_HADOOP_OPTS)){ + yarnOpts = variable.getValue(); + } + } + assertTrue("Environment contains " + ServiceCommandLineGenerator.ENV_HADOOP_OPTS, StringUtils.isNotEmpty(yarnOpts)); + System.out.println(yarnOpts); + assertTrue(ServiceCommandLineGenerator.ENV_HADOOP_OPTS + " must contain -D" + KEY_JHS_WEBAPP_ADDRESS + + "=0.0.0.0:3", yarnOpts.contains(KEY_JHS_WEBAPP_ADDRESS + "=0.0.0.0:3")); } @Test public void testNMCommandLineGeneration() throws Exception { Long[] ports = new Long[]{1L, 2L, 3L, 4L}; - NMPorts nmPorts = new NMPorts(ports); + List<Long> nmPorts = Arrays.asList(ports); + Map<String, Long> portsMap = new TreeMap<>(); + portsMap.put(KEY_NM_ADDRESS, 0L); + portsMap.put(KEY_NM_WEBAPP_ADDRESS, 0L); + portsMap.put(KEY_NM_LOCALIZER_ADDRESS, 0L); + portsMap.put(KEY_NM_SHUFFLE_PORT, 0L); - ServiceResourceProfile profile = new ExtendedResourceProfile(new NMProfile("nm", 10L, 15L), 3.0, 5.0); + ServiceResourceProfile profile = new ExtendedResourceProfile(new NMProfile("nm", 10L, 15L), 3.0, 5.0, portsMap); - ExecutorCommandLineGenerator clGenerator = new DownloadNMExecutorCLGenImpl(cfg, - "hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz"); - NMTaskFactoryImpl nms = new NMTaskFactoryImpl(cfg, null, clGenerator); + ExecutorCommandLineGenerator clGenerator = new NMExecutorCommandLineGenerator(cfg); - CommandInfo cInfo = nms.getCommandInfo(profile, nmPorts); - System.out.println(toCompare); - System.out.println(cInfo.getValue()); - assertTrue(cInfo.getValue().startsWith(toCompare)); + CommandInfo cInfo = clGenerator.generateCommandLine(profile, null, nmPorts); + String testVal = String.format(CMD_FORMAT, toCompare); + assertTrue(String.format(msgFormat, cInfo.getValue(), testVal), + cInfo.getValue().equals(testVal)); + List<Protos.Environment.Variable> environmentList = cInfo.getEnvironment().getVariablesList(); + String yarnOpts = ""; + for (Protos.Environment.Variable variable: environmentList) { + if (variable.getName().equals(NMExecutorCommandLineGenerator.ENV_YARN_NODEMANAGER_OPTS)){ + yarnOpts = variable.getValue(); + } + } + System.out.println(yarnOpts); + assertTrue("Environment contains " + NMExecutorCommandLineGenerator.ENV_YARN_NODEMANAGER_OPTS, StringUtils.isNotEmpty(yarnOpts)); + assertTrue(NMExecutorCommandLineGenerator.ENV_YARN_NODEMANAGER_OPTS + " must contain -D" + KEY_NM_SHUFFLE_PORT + + "=1", yarnOpts.contains(KEY_NM_SHUFFLE_PORT + "=1")); } + } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceTaskFactory.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceTaskFactory.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceTaskFactory.java new file mode 100644 index 0000000..bdd40e6 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceTaskFactory.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.scheduler; + +import org.apache.mesos.Protos; +import org.apache.myriad.BaseConfigurableTest; +import org.apache.myriad.configuration.ServiceConfiguration; +import org.apache.myriad.scheduler.offer.OfferBuilder; +import org.apache.myriad.scheduler.resource.ResourceOfferContainer; +import org.apache.myriad.state.NodeTask; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + + +/** + * Tests for ServiceTaskFactory Class + */ +public class TestServiceTaskFactory extends BaseConfigurableTest { + static Protos.FrameworkID frameworkId = Protos.FrameworkID.newBuilder().setValue("test").build(); + + @Test + public void testServiceTaskFactory() { + ServiceCommandLineGenerator clGenerator = new ServiceCommandLineGenerator(cfgWithDocker); + TaskUtils taskUtils = new TaskUtils(cfgWithDocker); + Protos.Offer offer = new OfferBuilder("test.com") + .addScalarResource("cpus", 10.0) + .addScalarResource("mem", 16000) + .addRangeResource("ports", 3400, 3410) + .build(); + Map<String, ServiceConfiguration> stringServiceConfigurationMap = cfgWithDocker.getServiceConfigurations(); + System.out.print(stringServiceConfigurationMap); + ServiceConfiguration serviceConfiguration = cfgWithDocker.getServiceConfigurations().get("jobhistory"); + ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", serviceConfiguration.getCpus(), + serviceConfiguration.getJvmMaxMemoryMB(), serviceConfiguration.getPorts()); + NodeTask nodeTask = new NodeTask(profile, null); + nodeTask.setTaskPrefix("jobhistory"); + ResourceOfferContainer roc = new ResourceOfferContainer(offer, profile, null); + System.out.print(roc.getPorts()); + ServiceTaskFactory taskFactory = new ServiceTaskFactory(cfgWithDocker, taskUtils, clGenerator); + Protos.TaskInfo taskInfo = taskFactory.createTask(roc, frameworkId, makeTaskId("jobhistory"), nodeTask); + assertTrue("taskInfo should have a container", taskInfo.hasContainer()); + assertFalse("The container should not have an executor", taskInfo.hasExecutor()); + Protos.ContainerInfo containerInfo = taskInfo.getContainer(); + assertTrue("There should be two volumes", containerInfo.getVolumesCount() == 2); + assertTrue("The first volume should be read only", containerInfo.getVolumes(0).getMode().equals(Protos.Volume.Mode.RO)); + assertTrue("The first volume should be read write", containerInfo.getVolumes(1).getMode().equals(Protos.Volume.Mode.RW)); + assertTrue("There should be a docker image", containerInfo.getDocker().hasImage()); + assertTrue("The docker image should be mesos/myraid", containerInfo.getDocker().getImage().equals("mesos/myriad")); + assertTrue("Should be using host networking", containerInfo.getDocker().getNetwork().equals(Protos.ContainerInfo.DockerInfo.Network.HOST)); + assertTrue("There should be two parameters", containerInfo.getDocker().getParametersList().size() == 2); + assertTrue("Privledged mode should be false", containerInfo.getDocker().getPrivileged() == false); + } + + private Protos.TaskID makeTaskId(String taskId) { + return Protos.TaskID.newBuilder().setValue(taskId).build(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java index d543c29..21b5ad0 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java @@ -17,15 +17,18 @@ */ package org.apache.myriad.scheduler; +import static org.junit.Assert.assertTrue; +import org.apache.mesos.Protos; +import org.apache.myriad.BaseConfigurableTest; +import org.junit.Test; + import com.google.common.collect.Range; import com.google.common.collect.Ranges; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.mesos.Protos; -import org.apache.myriad.BaseConfigurableTest; -import org.junit.Test; -import static org.junit.Assert.assertTrue; +import java.util.HashMap; + /** * Tests for TaskUtils @@ -40,7 +43,7 @@ public class TestTaskUtils extends BaseConfigurableTest { Gson gson = new GsonBuilder().registerTypeAdapter(ServiceResourceProfile.class, new ServiceResourceProfile.CustomDeserializer()) .create(); - ServiceResourceProfile parentProfile = new ServiceResourceProfile("abc", 1.0, 100.0); + ServiceResourceProfile parentProfile = new ServiceResourceProfile("abc", 1.0, 100.0, new HashMap<String, Long>()); String parentStr = gson.toJson(parentProfile); ServiceResourceProfile processedProfile = gson.fromJson(parentStr, ServiceResourceProfile.class); @@ -48,7 +51,8 @@ public class TestTaskUtils extends BaseConfigurableTest { assertTrue(processedProfile.getClass().equals(ServiceResourceProfile.class)); assertTrue(processedProfile.toString().equalsIgnoreCase(parentStr)); - ServiceResourceProfile childProfile = new ExtendedResourceProfile(new NMProfile("bcd", 5L, 15L), 2.0, 7.0); + ServiceResourceProfile childProfile = new ExtendedResourceProfile(new NMProfile("bcd", 5L, 15L), 2.0, 7.0, + cfg.getNodeManagerConfiguration().getPorts()); String childStr = gson.toJson(childProfile); ServiceResourceProfile processedChildProfile = gson.fromJson(childStr, ServiceResourceProfile.class); @@ -124,25 +128,4 @@ public class TestTaskUtils extends BaseConfigurableTest { checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus", 0.5, 1.5), "cpus", 0.0, 0.5); } - @Test - public void testContainerInfo() { - TaskUtils taskUtils = new TaskUtils(cfgWithDocker); - Protos.ContainerInfo containerInfo = taskUtils.getContainerInfo(); - assertTrue("The container should have a docker", containerInfo.hasDocker()); - assertTrue("There should be two volumes", containerInfo.getVolumesCount() == 2); - assertTrue("The first volume should be read only", containerInfo.getVolumes(0).getMode().equals(Protos.Volume.Mode.RO)); - assertTrue("The first volume should be read write", containerInfo.getVolumes(1).getMode().equals(Protos.Volume.Mode.RW)); - } - - @Test public void testDockerInfo() { - TaskUtils taskUtils = new TaskUtils(cfgWithDocker); - Protos.ContainerInfo containerInfo = taskUtils.getContainerInfo(); - assertTrue("The container should have a docker", containerInfo.hasDocker()); - assertTrue("There should be two volumes", containerInfo.getVolumesList().size() == 2); - assertTrue("There should be a docker image", containerInfo.getDocker().hasImage()); - assertTrue("The docker image should be mesos/myraid", containerInfo.getDocker().getImage().equals("mesos/myriad")); - assertTrue("Should be using host networking", containerInfo.getDocker().getNetwork().equals(Protos.ContainerInfo.DockerInfo.Network.HOST)); - assertTrue("There should be two parameters", containerInfo.getDocker().getParametersList().size() == 2); - assertTrue("Privledged mode should be false", containerInfo.getDocker().getPrivileged() == false); - } } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManagerTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManagerTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManagerTest.java index e1f4eb5..a4f4268 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManagerTest.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManagerTest.java @@ -4,6 +4,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.util.Set; +import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; @@ -62,7 +63,8 @@ public class YarnNodeCapacityManagerTest extends BaseConfigurableTest { private Set<NodeTask> getNodeTasks() { Constraint cZero = new LikeConstraint("0.0.0.1", "host-[0-9]*.example.com"); - ServiceResourceProfile zProfile = new ServiceResourceProfile("zProfile", 0.0, 0.0, 0.0, 0.0); + TreeMap<String, Long> ports = new TreeMap<>(); + ServiceResourceProfile zProfile = new ServiceResourceProfile("zProfile", 0.0, 0.0, ports); ntZero = new NodeTask(zProfile, cZero); ntZero.setTaskPrefix("nm"); http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/offer/OfferBuilder.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/offer/OfferBuilder.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/offer/OfferBuilder.java new file mode 100644 index 0000000..6772409 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/offer/OfferBuilder.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.myriad.scheduler.offer; + +import org.apache.mesos.Protos; + +import java.util.Collections; + + +/** + * Simple builder to make Offers for tests + */ +public class OfferBuilder { + Protos.Offer.Builder offer = Protos.Offer.newBuilder(); + + public OfferBuilder(String offerId, String hostname, String slaveId) { + offer.setHostname(hostname); + offer.setId(Protos.OfferID.newBuilder().setValue(offerId)); + offer.setSlaveId(Protos.SlaveID.newBuilder().setValue(slaveId)); + offer.setFrameworkId(Protos.FrameworkID.newBuilder().setValue("Myriad")); + } + public OfferBuilder(String offerId, String hostname) { + offer.setHostname(hostname); + offer.setId(Protos.OfferID.newBuilder().setValue(offerId)); + offer.setSlaveId(Protos.SlaveID.newBuilder().setValue("agent")); + offer.setFrameworkId(Protos.FrameworkID.newBuilder().setValue("Myriad")); + } + public OfferBuilder(String hostname) { + offer.setHostname(hostname); + offer.setId(Protos.OfferID.newBuilder().setValue("test")); + offer.setSlaveId(Protos.SlaveID.newBuilder().setValue("agent")); + offer.setFrameworkId(Protos.FrameworkID.newBuilder().setValue("Myriad")); + } + + public OfferBuilder addScalarResource(String name, double value) { + offer.addAllResources(Collections.singleton(createScalarResource(name, value))); + return this; + } + + public OfferBuilder addScalarResource(String name, String role, double value) { + offer.addAllResources(Collections.singleton(createScalarResource(name, role, value))); + return this; + } + + public OfferBuilder addRangeResource(String name, long begin, long end) { + offer.addAllResources(Collections.singleton(createRangeResource(name, begin, end))); + return this; + } + + public OfferBuilder addRangeResource(String name, String role, long begin, long end) { + offer.addAllResources(Collections.singleton(createRangeResource(name, role, begin, end))); + return this; + } + + public Protos.Offer build() { + return offer.build(); + } + + private static Protos.Resource createScalarResource(String name, String role, double value) { + return Protos.Resource.newBuilder() + .setScalar(Protos.Value.Scalar.newBuilder().setValue(value)) + .setType(Protos.Value.Type.SCALAR) + .setRole(role) + .setName(name) + .build(); + } + + private static Protos.Resource createScalarResource(String name, double value) { + return Protos.Resource.newBuilder() + .setScalar(Protos.Value.Scalar.newBuilder().setValue(value)) + .setType(Protos.Value.Type.SCALAR) + .setName(name) + .build(); + } + + private static Protos.Resource createRangeResource(String name, long begin, long end) { + return Protos.Resource.newBuilder() + .setName(name) + .setType(Protos.Value.Type.RANGES) + .setRanges(Protos.Value.Ranges.newBuilder() + .addRange(Protos.Value.Range.newBuilder() + .setBegin(begin) + .setEnd(end) + .build()) + .build()) + .build(); + } + + private static Protos.Resource createRangeResource(String name, String role, long begin, long end) { + return Protos.Resource.newBuilder() + .setName(name) + .setType(Protos.Value.Type.RANGES) + .setRole(role) + .setRanges(Protos.Value.Ranges.newBuilder() + .addRange(Protos.Value.Range.newBuilder() + .setBegin(begin) + .setEnd(end) + .build()) + .build()) + .build(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/resource/TestResourceOfferContainer.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/resource/TestResourceOfferContainer.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/resource/TestResourceOfferContainer.java new file mode 100644 index 0000000..d54cc7a --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/resource/TestResourceOfferContainer.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.scheduler.resource; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.apache.mesos.Protos; +import org.apache.myriad.configuration.MyriadConfiguration; +import org.apache.myriad.scheduler.ExtendedResourceProfile; +import org.apache.myriad.scheduler.NMProfile; +import org.apache.myriad.scheduler.ServiceResourceProfile; +import org.apache.myriad.scheduler.offer.OfferBuilder; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests for ResourceOfferContainerClass + */ +public class TestResourceOfferContainer { + static MyriadConfiguration cfg; + static double epsilon = .0001; + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), + MyriadConfiguration.class); + } + + @Test + public void testResouceOfferContainerForNMWithOutRole() { + Protos.Offer offer = new OfferBuilder("test.com") + .addScalarResource("cpus", 4.0) + .addScalarResource("mem", 8000) + .addRangeResource("ports", 3500, 3600) + .build(); + Map<String, Long> ports = new HashMap<>(4); + ports.put("test1.address", 0L); + ports.put("test2.address", 0L); + ports.put("test3.address", 0L); + ports.put("test4.port", 3501L); + ServiceResourceProfile profile1 = new ExtendedResourceProfile(new NMProfile("small", 2L, 6000L), .2, 1024.0, ports); + ResourceOfferContainer roc = new ResourceOfferContainer(offer, profile1, ""); + System.out.print(roc.getCpus()); + System.out.print(roc.getMem()); + System.out.print(roc.getPorts()); + assertTrue(roc.getHostName().equals("test.com")); + assertTrue("Should be satisfied if offer contains request", roc.satisfies(profile1)); + ServiceResourceProfile profile2 = new ExtendedResourceProfile(new NMProfile("tooMuchCpu", 7L, 8000L), .2, 1024.0, ports); + roc = new ResourceOfferContainer(offer, profile2, ""); + assertFalse("Should be unsatisfied if too much cpu requested", roc.satisfies(profile2)); + ServiceResourceProfile profile3 = new ExtendedResourceProfile(new NMProfile("tooMuchMem", 3L, 50000L), .2, 1024.0, ports); + roc = new ResourceOfferContainer(offer, profile3, ""); + assertFalse("Should be unsatisfied if too much memory requested", roc.satisfies(profile3)); + ports.put("test.bad.address", 1500L); + ServiceResourceProfile profile4 = new ExtendedResourceProfile(new NMProfile("portOutOfRange", 3L, 50000L), .2, 1024.0, ports); + roc = new ResourceOfferContainer(offer, profile4, ""); + assertFalse("Should be unsatisfied if port not in range", roc.satisfies(profile4)); + List<Protos.Resource> resourcesCpu = roc.consumeCpus(3.0); + assertTrue("Should get a list of resources of size 1", resourcesCpu.size() == 1.0); + assertTrue("Cpus should be decreased", roc.getCpus() == 1.0); + List<Protos.Resource> resourcesMem = roc.consumeMem(7000.0); + assertTrue("Should get a list of resources of size 1", resourcesMem.size() == 1); + assertTrue("Mem should be decreased", roc.getMem() == 1000.0); + } + + @Test + public void testResouceOfferContainerForNMWithRole() { + Protos.Offer offer = new OfferBuilder("test.com") + .addScalarResource("cpus", 2.0) + .addScalarResource("mem", 8000) + .addScalarResource("cpus", "test", 4.0) + .addScalarResource("mem", "test", 32000.0) + .addRangeResource("ports", 3500, 3600) + .addRangeResource("ports", "test", 1500, 1600) + .build(); + Map<String, Long> ports = new HashMap<>(4); + ports.put("test1.address", 0L); + ports.put("test2.address", 0L); + ports.put("test3.address", 1500L); + ports.put("test4.port", 3502L); + ServiceResourceProfile profile1 = new ExtendedResourceProfile(new NMProfile("small", 2L, 8000L), .2, 1024.0, ports); + ResourceOfferContainer roc = new ResourceOfferContainer(offer, profile1, "test"); + assertTrue(roc.getHostName().equals("test.com")); + assertTrue("Should be satisfied if offer contains request", roc.satisfies(profile1)); + ServiceResourceProfile profile2 = new ExtendedResourceProfile(new NMProfile("tooMuchCpu", 7L, 8000L), .2, 1024.0, ports); + roc = new ResourceOfferContainer(offer, profile2, "test"); + assertFalse("Should be unsatisfied if too much cpu requested", roc.satisfies(profile2)); + ServiceResourceProfile profile3 = new ExtendedResourceProfile(new NMProfile("tooMuchMem", 3L, 50000L), .2, 1024.0, ports); + roc = new ResourceOfferContainer(offer, profile3, "test"); + assertFalse("Should be unsatisfied if too much memory requested", roc.satisfies(profile3)); + ports.put("test.bad.address", 32000L); + ServiceResourceProfile profile4 = new ExtendedResourceProfile(new NMProfile("portOutOfRange", 3L, 50000L), .2, 1024.0, ports); + roc = new ResourceOfferContainer(offer, profile4, "test"); + assertFalse("Should be unsatisfied if port not in range", roc.satisfies(profile4)); + List<Protos.Resource> resources = roc.consumeCpus(4.5); + assertTrue("Resource List should be of size to when requesting 4.1 cpus", (resources.size() == 2)); + assertTrue("Cpus should be decreased", roc.getCpus() <= 1.5); + List<Protos.Resource> resources1 = roc.consumeCpus(1.5); + assertTrue("Resource List should be of size 1", resources1.size() == 1); + assertTrue("All cpu resources should be consumed", roc.getCpus() <= 0.0); + } + + @Test + public void testResourceOfferContainerForAuxServiceWithOutRole() { + Protos.Offer offer = new OfferBuilder("test.com") + .addScalarResource("cpus", 2.0) + .addScalarResource("mem", 8000) + .addRangeResource("ports", 3500, 3600) + .build(); + Map<String, Long> ports = new HashMap<>(4); + ports.put("test1.address", 0L); + ports.put("test2.address", 0L); + ports.put("test3.address", 0L); + ports.put("test4.port", 3501L); + ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 2.0, 8000.0, ports); + ResourceOfferContainer roc = new ResourceOfferContainer(offer, profile, null); + assertTrue(roc.getHostName().equals("test.com")); + assertTrue("Should be satisfied if offer contains request", roc.satisfies(profile)); + } + + @Test + public void testResourceOfferContainerForAuxServiceWithRole() { + Protos.Offer offer = new OfferBuilder("test.com") + .addScalarResource("cpus", 2.0) + .addScalarResource("mem", 8000) + .addScalarResource("cpus", "test", 4.0) + .addScalarResource("mem", "test", 32000.0) + .addRangeResource("ports", 3500, 3600) + .addRangeResource("ports", "test", 1500, 1600) + .build(); + Map<String, Long> ports = new HashMap<>(4); + ports.put("test1.address", 0L); + ports.put("test2.address", 0L); + ports.put("test3.address", 1500L); + ports.put("test4.port", 3501L); + ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 2.0, 8000.0, ports); + ResourceOfferContainer roc = new ResourceOfferContainer(offer, profile, null); + assertTrue(roc.getHostName().equals("test.com")); + assertTrue("Should be satisfied if offer contains request", roc.satisfies(profile)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/state/ClusterTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/state/ClusterTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/state/ClusterTest.java index 69954b7..09145a2 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/state/ClusterTest.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/state/ClusterTest.java @@ -7,6 +7,8 @@ import org.apache.myriad.scheduler.constraints.LikeConstraint; import org.junit.Before; import org.junit.Test; +import java.util.TreeMap; + /** * Unit tests for Cluster */ @@ -16,15 +18,16 @@ public class ClusterTest { @Before public void setUp() throws Exception { + TreeMap<String, Long> ports = new TreeMap<>(); cluster = new Cluster(); cluster.setClusterName("test-cluster"); cluster.setMinQuota(5.0); cluster.setResourceManagerHost("localhost"); cluster.setResourceManagerPort("8192"); - task1 = new NodeTask(new ServiceResourceProfile("profile1", 0.1, 1024.0), new LikeConstraint("hostname1", "host-[0-9]*.example1.com")); - task2 = new NodeTask(new ServiceResourceProfile("profile2", 0.2, 1024.0), new LikeConstraint("hostname2", "host-[0-9]*.example2.com")); - task3 = new NodeTask(new ServiceResourceProfile("profile3", 0.3, 1024.0), new LikeConstraint("hostname3", "host-[0-9]*.example3.com")); + task1 = new NodeTask(new ServiceResourceProfile("profile1", 0.1, 1024.0, ports), new LikeConstraint("hostname1", "host-[0-9]*.example1.com")); + task2 = new NodeTask(new ServiceResourceProfile("profile2", 0.2, 1024.0, ports), new LikeConstraint("hostname2", "host-[0-9]*.example2.com")); + task3 = new NodeTask(new ServiceResourceProfile("profile3", 0.3, 1024.0, ports), new LikeConstraint("hostname3", "host-[0-9]*.example3.com")); } private void resetCluster() throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/state/NodeTaskTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/state/NodeTaskTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/state/NodeTaskTest.java index d3cf2fe..3f486c0 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/state/NodeTaskTest.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/state/NodeTaskTest.java @@ -6,6 +6,8 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.util.TreeMap; + /** * Unit test cases for NodeTask */ @@ -13,12 +15,13 @@ public class NodeTaskTest { NodeTask task; @Before - public void setUp() throws Exception { - task = new NodeTask(new ServiceResourceProfile("profile", 0.1, 1024.0), new LikeConstraint("hostname", "host-[0-9]*.example.com")); + public void setUp() throws Exception { + TreeMap<String, Long> ports = new TreeMap<>(); + task = new NodeTask(new ServiceResourceProfile("profile", 0.1, 1024.0, ports), new LikeConstraint("hostname", "host-[0-9]*.example.com")); task.setHostname("localhost"); task.setTaskPrefix("prefix"); - task.setProfile(new ServiceResourceProfile("ServiceResourceProfile", 0.1, 1024.0, 0.1, 512.0)); + task.setProfile(new ServiceResourceProfile("ServiceResourceProfile", 0.1, 1024.0, ports)); } @Test http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/state/SchedulerStateTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/state/SchedulerStateTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/state/SchedulerStateTest.java index 59e2d41..cd4a12e 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/state/SchedulerStateTest.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/state/SchedulerStateTest.java @@ -5,6 +5,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import java.util.Set; +import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore; @@ -38,8 +39,8 @@ public class SchedulerStateTest { @Test public void testAddAndRemoveTask() throws Exception { - NodeTask task1 = new NodeTask(new ServiceResourceProfile("profile1", 0.1, 1024.0), new LikeConstraint("hostname", "host-[0-9]*.example.com")); - NodeTask task2 = new NodeTask(new ServiceResourceProfile("profile2", 0.1, 1024.0), new LikeConstraint("hostname", "host-[0-9]*.example.com")); + NodeTask task1 = new NodeTask(new ServiceResourceProfile("profile1", 0.1, 1024.0, new TreeMap<String, Long>()), new LikeConstraint("hostname", "host-[0-9]*.example.com")); + NodeTask task2 = new NodeTask(new ServiceResourceProfile("profile2", 0.1, 1024.0, new TreeMap<String, Long>()), new LikeConstraint("hostname", "host-[0-9]*.example.com")); TaskID id1 = TaskID.newBuilder().setValue("mock-task-1").build(); TaskID id2 = TaskID.newBuilder().setValue("mock-task-2").build(); http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/state/utils/ByteBufferSupportTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/state/utils/ByteBufferSupportTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/state/utils/ByteBufferSupportTest.java index 6794ec2..9d4dac5 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/state/utils/ByteBufferSupportTest.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/state/utils/ByteBufferSupportTest.java @@ -5,7 +5,7 @@ import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; -import java.util.List; +import java.util.*; import org.apache.commons.lang3.ArrayUtils; import org.apache.mesos.Protos; @@ -33,7 +33,8 @@ public class ByteBufferSupportTest { @Before public void setUp() throws Exception { - task = new NodeTask(new ServiceResourceProfile("profile", 0.1, 1024.0), new LikeConstraint("hostname", "host-[0-9]*.example.com")); + task = new NodeTask(new ServiceResourceProfile("profile", 0.1, 1024.0, new TreeMap<String, Long>()), + new LikeConstraint("hostname", "host-[0-9]*.example.com")); task.setHostname("localhost"); task.setTaskPrefix("prefix"); task.setExecutorInfo(getExecutorInfo()); http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml index 4bdb6e2..cd72c18 100644 --- a/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml +++ b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml @@ -63,6 +63,7 @@ services: myriad.mapreduce.jobhistory.webapp.address: 0 envSettings: -Dcluster.name.prefix=/mycluster taskName: jobhistory + command: bin/mapred historyserver timelineserver: jvmMaxMemoryMB: 1024 cpus: 1 http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/resources/myriad-config-test-default.yml ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/resources/myriad-config-test-default.yml b/myriad-scheduler/src/test/resources/myriad-config-test-default.yml index f6f6b58..d07a318 100644 --- a/myriad-scheduler/src/test/resources/myriad-config-test-default.yml +++ b/myriad-scheduler/src/test/resources/myriad-config-test-default.yml @@ -60,18 +60,20 @@ yarnEnvironment: mesosAuthenticationPrincipal: mesosAuthenticationSecretFilename: services: - jobhistory: - maxInstances: 1 - jvmMaxMemoryMB: 1024 - cpus: 1 - ports: - myriad.mapreduce.jobhistory.admin.address: 0 - myriad.mapreduce.jobhistory.address: 0 - myriad.mapreduce.jobhistory.webapp.address: 0 - envSettings: -Dcluster.name.prefix=/mycluster - taskName: jobhistory - timelineserver: - jvmMaxMemoryMB: 1024 - cpus: 1 - envSettings: -Dcluster.name.prefix=/mycluster2 - taskName: timelineserver + jobhistory: + maxInstances: 1 + jvmMaxMemoryMB: 1024 + cpus: 1 + ports: + myriad.mapreduce.jobhistory.admin.address: 0 + myriad.mapreduce.jobhistory.address: 0 + myriad.mapreduce.jobhistory.webapp.address: 1 + command: bin/mapred historyserver + envSettings: -Dcluster.name.prefix=/mycluster + taskName: jobhistory + timelineserver: + jvmMaxMemoryMB: 1024 + cpus: 1 + envSettings: -Dcluster.name.prefix=/mycluster2 + taskName: timelineserver + command: bin/mapred historyserver
