MYRIAD-200 Increase JUnit Test Coverage
JIRA: [MYRIAD-200] https://issues.apache.org/jira/browse/MYRIAD-200 Pull Request: Closes #78 Author: hokiegeek2 <hokiege...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/4a6e50c4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/4a6e50c4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/4a6e50c4 Branch: refs/heads/master Commit: 4a6e50c41ce3098a393c96ff8f55f9d4eb78b390 Parents: 6354ce6 Author: hokiegeek2 <hokiege...@gmail.com> Authored: Thu Jun 9 15:03:24 2016 -0400 Committer: darinj <dar...@apache.org> Committed: Tue Jul 12 14:48:32 2016 -0400 ---------------------------------------------------------------------- .../myriad/executor/MyriadExecutorDefaults.java | 6 +- .../apache/myriad/executor/NMTaskConfig.java | 2 +- .../recovery/MyriadFileSystemRMStateStore.java | 5 +- .../src/main/java/org/apache/myriad/Main.java | 30 +- .../java/org/apache/myriad/MyriadModule.java | 22 +- .../apache/myriad/health/HealthCheckUtils.java | 39 ++- .../scheduler/ExtendedResourceProfile.java | 47 ++- .../org/apache/myriad/scheduler/NMProfile.java | 47 ++- .../myriad/scheduler/NMProfileManager.java | 25 +- .../scheduler/ServiceResourceProfile.java | 96 +++++- .../scheduler/event/OfferRescindedEvent.java | 6 +- .../handlers/OfferRescindedEventHandler.java | 5 +- .../event/handlers/SlaveLostEventHandler.java | 1 + .../scheduler/fgs/OfferLifecycleManager.java | 9 + .../java/org/apache/myriad/state/Cluster.java | 4 + .../java/org/apache/myriad/state/NodeTask.java | 15 +- .../myriad/state/utils/ByteBufferSupport.java | 27 +- .../apache/myriad/webapp/MyriadWebServer.java | 21 +- .../MyriadFileSystemRMStateStoreTest.java | 69 ++++ .../org/apache/myriad/BaseConfigurableTest.java | 35 ++ .../org/apache/myriad/MultiBindingsTest.java | 33 +- .../org/apache/myriad/TestObjectFactory.java | 90 +++++ .../myriad/api/ArtifactsResourceTest.java | 51 +++ .../myriad/api/SchedulerStateResourceTest.java | 58 ++++ .../configuration/MyriadConfigurationTest.java | 95 +++--- .../myriad/health/HealthCheckUtilsTest.java | 27 ++ .../health/MesosDriverHealthCheckTest.java | 69 ++++ .../myriad/scheduler/MockSchedulerDriver.java | 124 +++++++ .../myriad/scheduler/MyriadDriverTest.java | 43 +++ .../myriad/scheduler/MyriadOperationsTest.java | 112 ++++++ .../myriad/scheduler/NMProfileManagerTest.java | 63 ++++ .../scheduler/ServiceResourceProfileTest.java | 31 ++ .../scheduler/TaskConstraintsManagerTest.java | 32 ++ .../scheduler/TestServiceCommandLine.java | 27 +- .../apache/myriad/scheduler/TestTaskUtils.java | 41 +-- .../fgs/OfferLifeCycleManagerTest.java | 54 +++ .../myriad/scheduler/fgs/OfferUtilsTest.java | 74 ++++ .../org/apache/myriad/state/ClusterTest.java | 62 ++++ .../org/apache/myriad/state/MockDispatcher.java | 32 ++ .../org/apache/myriad/state/MockFuture.java | 44 +++ .../java/org/apache/myriad/state/MockRMApp.java | 202 +++++++++++ .../org/apache/myriad/state/MockRMContext.java | 339 +++++++++++++++++++ .../java/org/apache/myriad/state/MockState.java | 45 +++ .../org/apache/myriad/state/MockVariable.java | 35 ++ .../apache/myriad/state/MyriadStateTest.java | 21 ++ .../org/apache/myriad/state/NodeTaskTest.java | 40 +++ .../apache/myriad/state/SchedulerStateTest.java | 65 ++++ .../state/utils/ByteBufferSupportTest.java | 184 ++++++++++ .../webapp/HttpConnectorProviderTest.java | 22 ++ .../myriad/webapp/MyriadWebServerTest.java | 29 ++ .../resources/myriad-config-test-default.yml | 7 +- 51 files changed, 2455 insertions(+), 207 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java ---------------------------------------------------------------------- diff --git a/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java b/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java index bda7ff0..c7e4515 100644 --- a/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java +++ b/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java @@ -69,9 +69,7 @@ public class MyriadExecutorDefaults { public static final double DEFAULT_JVM_MAX_MEMORY_MB = 256; /** - * Default cpus for executor JVM. + * Default CPU cores for executor JVM. */ public static final double DEFAULT_CPUS = 0.2; - - -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-commons/src/main/java/org/apache/myriad/executor/NMTaskConfig.java ---------------------------------------------------------------------- diff --git a/myriad-commons/src/main/java/org/apache/myriad/executor/NMTaskConfig.java b/myriad-commons/src/main/java/org/apache/myriad/executor/NMTaskConfig.java index 21d2420..1b250ec 100644 --- a/myriad-commons/src/main/java/org/apache/myriad/executor/NMTaskConfig.java +++ b/myriad-commons/src/main/java/org/apache/myriad/executor/NMTaskConfig.java @@ -21,7 +21,7 @@ package org.apache.myriad.executor; import java.util.Map; /** - * Node Manger Task Configuraiton + * Node Manger Task Configuration */ public class NMTaskConfig { private String yarnHome; http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java index 923e29d..6257ffc 100644 --- a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java +++ b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java @@ -32,6 +32,8 @@ import org.apache.myriad.state.utils.StoreContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; + /** * StateStore that stores Myriad state in addition to RM state to DFS. */ @@ -70,6 +72,7 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore impleme return null; } + @VisibleForTesting @Override protected synchronized void startInternal() throws Exception { super.startInternal(); @@ -120,7 +123,7 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore impleme protected void reflectedUpdateFile(Path outputPath, byte[] data) throws InvocationTargetException, IllegalAccessException { - Class [] parameters = updateFileMethod.getParameterTypes(); + Class<?> [] parameters = updateFileMethod.getParameterTypes(); if (parameters.length == 2 && parameters[0].equals(Path.class) && parameters[1].isArray()) { updateFileMethod.invoke(this, outputPath, data); } else if (parameters.length == 3 && parameters[0].equals(Path.class) && parameters[1].isArray() && parameters[2].isPrimitive()) { http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/Main.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/Main.java b/myriad-scheduler/src/main/java/org/apache/myriad/Main.java index e825256..14ab806 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/Main.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/Main.java @@ -18,19 +18,13 @@ */ package org.apache.myriad; -import com.codahale.metrics.JmxReporter; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.health.HealthCheckRegistry; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.google.inject.Guice; -import com.google.inject.Injector; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; + import org.apache.commons.collections.MapUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -61,6 +55,12 @@ import org.apache.myriad.webapp.WebAppGuiceModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.codahale.metrics.JmxReporter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.health.HealthCheckRegistry; +import com.google.inject.Guice; +import com.google.inject.Injector; + /** * Main entry point for myriad scheduler */ @@ -76,18 +76,14 @@ public class Main { private static Injector injector; public static void initialize(Configuration hadoopConf, AbstractYarnScheduler yarnScheduler, RMContext rmContext, - InterceptorRegistry registry) throws Exception { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - MyriadConfiguration cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource( - "myriad-config-default.yml"), MyriadConfiguration.class); - - MyriadModule myriadModule = new MyriadModule(cfg, hadoopConf, yarnScheduler, rmContext, registry); + InterceptorRegistry registry) throws Exception { + MyriadModule myriadModule = new MyriadModule("myriad-config-default.yml", hadoopConf, yarnScheduler, rmContext, registry); MesosModule mesosModule = new MesosModule(); injector = Guice.createInjector(myriadModule, mesosModule, new WebAppGuiceModule()); - new Main().run(cfg); + new Main().run(injector.getInstance(MyriadConfiguration.class)); } - + // TODO (Kannan Rajah) Hack to get injector in unit test. public static Injector getInjector() { return injector; @@ -158,10 +154,8 @@ public class Main { Long cpu = Long.parseLong(profileResourceMap.get("cpu")); Long mem = Long.parseLong(profileResourceMap.get("mem")); - ServiceResourceProfile serviceProfile = new ExtendedResourceProfile(new NMProfile(profile.getKey(), cpu, mem), + ServiceResourceProfile serviceProfile = new ExtendedResourceProfile(new NMProfile(profile.getKey(), cpu, mem), taskUtils.getExecutorCpus(), taskUtils.getExecutorMemory(), taskUtils.getNodeManagerCpus(), taskUtils.getNodeManagerMemory()); - serviceProfile.setExecutorCpu(taskUtils.getExecutorCpus()); - serviceProfile.setExecutorMemory(taskUtils.getExecutorMemory()); profileManager.add(serviceProfile); } else { http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/MyriadModule.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/MyriadModule.java b/myriad-scheduler/src/main/java/org/apache/myriad/MyriadModule.java index 92add9a..8748dcb 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/MyriadModule.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/MyriadModule.java @@ -18,11 +18,15 @@ */ package org.apache.myriad; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.Singleton; import com.google.inject.multibindings.MapBinder; + +import java.io.IOException; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -68,15 +72,15 @@ public class MyriadModule extends AbstractModule { private final RMContext rmContext; private InterceptorRegistry interceptorRegistry; - public MyriadModule(MyriadConfiguration cfg, Configuration hadoopConf, AbstractYarnScheduler yarnScheduler, RMContext rmContext, + public MyriadModule(String configFile, Configuration hadoopConf, AbstractYarnScheduler yarnScheduler, RMContext rmContext, InterceptorRegistry interceptorRegistry) { - this.cfg = cfg; + this.cfg = this.generateMyriadConfiguration(configFile); this.hadoopConf = hadoopConf; this.yarnScheduler = yarnScheduler; this.rmContext = rmContext; this.interceptorRegistry = interceptorRegistry; } - + @Override protected void configure() { LOGGER.debug("Configuring guice"); @@ -168,4 +172,16 @@ public class MyriadModule extends AbstractModule { } return cliGenerator; } + + protected MyriadConfiguration generateMyriadConfiguration(String configFile) { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + try { + return mapper.readValue(Thread.currentThread().getContextClassLoader().getResource( + configFile), MyriadConfiguration.class); + } catch (IOException e) { + LOGGER.error("The configFile {} could not be found", configFile); + throw new IllegalArgumentException("The configFile cannot be found", e); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/health/HealthCheckUtils.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/health/HealthCheckUtils.java b/myriad-scheduler/src/main/java/org/apache/myriad/health/HealthCheckUtils.java index 8abbb57..75ac1f6 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/health/HealthCheckUtils.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/health/HealthCheckUtils.java @@ -20,6 +20,8 @@ package org.apache.myriad.health; import java.io.IOException; import java.net.Socket; + +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,15 +32,34 @@ public class HealthCheckUtils { private static final Logger LOGGER = LoggerFactory.getLogger(HealthCheckUtils.class); public static boolean checkHostPort(String connectionString) { - String[] split = connectionString.split(":"); - String serverAddress = split[0]; - Integer serverPort = Integer.valueOf(split[1]); - try (Socket s = new Socket(serverAddress, serverPort)) { + String[] hostPort = generateHostPortArray(connectionString); + + try { + createSocket(hostPort); return true; - } catch (IOException ex) { - LOGGER.error("parsing host port", ex); - } - + } catch (IOException e) { + LOGGER.error("error in connecting to " + hostPort[0] + ":" + hostPort[1], e); + } + return false; } -} + + private static void createSocket(String[] hostPort) throws IOException { + String address = hostPort[0]; + Integer port = Integer.valueOf(hostPort[1]); + + Socket s = new Socket(address, port); + s.close(); + } + + private static String[] generateHostPortArray(String connectionString) { + String[] split = connectionString.split(":"); + if (split.length != 2) { + throw new IllegalArgumentException("The Connection String " + connectionString + " is invalid. It must be in <host>:<port> format"); + } else if (!StringUtils.isNumeric(split[1])) { + throw new IllegalArgumentException("The Connection String " + connectionString + " is invalid. The port must be an integer"); + } else { + return split; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ExtendedResourceProfile.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ExtendedResourceProfile.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ExtendedResourceProfile.java index 8119360..6232258 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ExtendedResourceProfile.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ExtendedResourceProfile.java @@ -33,8 +33,21 @@ public class ExtendedResourceProfile extends ServiceResourceProfile { * @param cpu * @param mem will throw NullPoiterException if childProfile is null */ + public ExtendedResourceProfile(NMProfile childProfile, Double cpu, Double mem, Double execCpu, Double execMemory) { + super(childProfile.getName(), cpu, mem, execCpu, execMemory); + + this.childProfile = childProfile; + this.className = ExtendedResourceProfile.class.getName(); + } + + /** + * @param childProfile - should be null + * @param cpu + * @param mem will throw NullPoiterException if childProfile is null + */ public ExtendedResourceProfile(NMProfile childProfile, Double cpu, Double mem) { super(childProfile.getName(), cpu, mem); + this.childProfile = childProfile; this.className = ExtendedResourceProfile.class.getName(); } @@ -77,4 +90,36 @@ public class ExtendedResourceProfile extends ServiceResourceProfile { Gson gson = new Gson(); return gson.toJson(this); } -} + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((childProfile == null) ? 0 : childProfile.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!super.equals(obj)) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + + ExtendedResourceProfile other = (ExtendedResourceProfile) obj; + + if (childProfile == null) { + if (other.childProfile != null) { + return false; + } + } else if (!childProfile.equals(other.childProfile)) { + return false; + } + return true; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMProfile.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMProfile.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMProfile.java index 3de82a5..2215835 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMProfile.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMProfile.java @@ -60,4 +60,49 @@ public class NMProfile { return gson.toJson(this); } -} + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((cpus == null) ? 0 : cpus.hashCode()); + result = prime * result + ((memory == null) ? 0 : memory.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + NMProfile other = (NMProfile) obj; + if (cpus == null) { + if (other.cpus != null) { + return false; + } + } else if (!cpus.equals(other.cpus)) { + return false; + } + if (memory == null) { + if (other.memory != null) { + return false; + } + } else if (!memory.equals(other.memory)) { + return false; + } + if (name == null) { + if (other.name != null) { + return false; + } + } else if (!name.equals(other.name)) { + return false; + } + return true; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMProfileManager.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMProfileManager.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMProfileManager.java index 4256f40..285de5d 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMProfileManager.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMProfileManager.java @@ -18,14 +18,16 @@ */ package org.apache.myriad.scheduler; -import com.google.gson.Gson; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Node Manager Profile Manager + * NMProfile Manager */ public class NMProfileManager { private static final Logger LOGGER = LoggerFactory.getLogger(NMProfileManager.class); @@ -46,8 +48,21 @@ public class NMProfileManager { return this.profiles.containsKey(name); } + public int numberOfProfiles() { + return profiles.size(); + } + + @Override public String toString() { - Gson gson = new Gson(); - return gson.toJson(this); + ToStringBuilder builder = new ToStringBuilder(this, ToStringStyle.JSON_STYLE); + + for (Map.Entry<String, NMProfile> profile : profiles.entrySet()) { + NMProfile value = profile.getValue(); + builder.append("name", value.getName()); + builder.append("cpus", value.getCpus()); + builder.append("memory", value.getMemory()); + } + + return builder.toString(); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceResourceProfile.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceResourceProfile.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceResourceProfile.java index 4c033c9..146a80c 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceResourceProfile.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceResourceProfile.java @@ -28,7 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Resource Profile for any service + * Resource Profile for any service */ public class ServiceResourceProfile { @@ -48,15 +48,21 @@ public class ServiceResourceProfile { protected Double executorMemory = 0.0; - protected String className; + protected String className = ServiceResourceProfile.class.getName(); - public ServiceResourceProfile(String name, Double cpu, Double mem) { + public ServiceResourceProfile(String name, Double cpus, Double mem) { this.name = name; - this.cpus = cpu; + this.cpus = cpus; this.memory = mem; - this.className = ServiceResourceProfile.class.getName(); } + public ServiceResourceProfile(String name, Double cpus, Double mem, Double execCpus, Double execMemory) { + this.name = name; + this.cpus = cpus; + this.memory = mem; + this.executorCpu = execCpus; + this.executorMemory = execMemory; + } public String getName() { return name; @@ -82,19 +88,10 @@ public class ServiceResourceProfile { return executorCpu; } - public void setExecutorCpu(Double executorCpu) { - this.executorCpu = executorCpu; - } - public Double getExecutorMemory() { return executorMemory; } - public void setExecutorMemory(Double executorMemory) { - this.executorMemory = executorMemory; - } - - @Override public String toString() { Gson gson = new Gson(); @@ -125,6 +122,75 @@ public class ServiceResourceProfile { } return null; } + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((className == null) ? 0 : className.hashCode()); + result = prime * result + ((cpus == null) ? 0 : cpus.hashCode()); + result = prime * result + ((executorCpu == null) ? 0 : executorCpu.hashCode()); + result = prime * result + ((executorMemory == null) ? 0 : executorMemory.hashCode()); + result = prime * result + ((memory == null) ? 0 : memory.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + ServiceResourceProfile other = (ServiceResourceProfile) obj; + if (className == null) { + if (other.className != null) { + return false; + } + } else if (!className.equals(other.className)) { + return false; + } + if (cpus == null) { + if (other.cpus != null) { + return false; + } + } else if (!cpus.equals(other.cpus)) { + return false; + } + if (executorCpu == null) { + if (other.executorCpu != null) { + return false; + } + } else if (!executorCpu.equals(other.executorCpu)) { + return false; + } + if (executorMemory == null) { + if (other.executorMemory != null) { + return false; + } + } else if (!executorMemory.equals(other.executorMemory)) { + return false; + } + if (memory == null) { + if (other.memory != null) { + return false; + } + } else if (!memory.equals(other.memory)) { + return false; + } + if (name == null) { + if (other.name != null) { + return false; + } + } else if (!name.equals(other.name)) { + return false; + } + return true; } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/OfferRescindedEvent.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/OfferRescindedEvent.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/OfferRescindedEvent.java index 180655c..126cfb8 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/OfferRescindedEvent.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/OfferRescindedEvent.java @@ -44,4 +44,8 @@ public class OfferRescindedEvent { this.offerId = offerId; } -} + @Override + public String toString() { + return "OfferRescindedEvent [driver=" + driver + ", offerId=" + offerId + "]"; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/handlers/OfferRescindedEventHandler.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/handlers/OfferRescindedEventHandler.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/handlers/OfferRescindedEventHandler.java index 85e8043..e49d2b9 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/handlers/OfferRescindedEventHandler.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/handlers/OfferRescindedEventHandler.java @@ -31,7 +31,6 @@ public class OfferRescindedEventHandler implements EventHandler<OfferRescindedEv @Override public void onEvent(OfferRescindedEvent event, long sequence, boolean endOfBatch) throws Exception { - LOGGER.info("OfferRescinded event: {}", event); + LOGGER.info("OfferRescindedEvent for offer id: {} for scheuler {}", event.getOfferId().toString(), event.getDriver().getClass().getName()); } - -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/handlers/SlaveLostEventHandler.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/handlers/SlaveLostEventHandler.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/handlers/SlaveLostEventHandler.java index b1f37bb..9e17f6c 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/handlers/SlaveLostEventHandler.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/event/handlers/SlaveLostEventHandler.java @@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory; public class SlaveLostEventHandler implements EventHandler<SlaveLostEvent> { private static final Logger LOGGER = LoggerFactory.getLogger(SlaveLostEventHandler.class); + //TODO (hokiegeek2) Research how else Myriad should respond to this event @Override public void onEvent(SlaveLostEvent event, long sequence, boolean endOfBatch) throws Exception { SlaveID slaveId = event.getSlaveId(); http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/fgs/OfferLifecycleManager.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/fgs/OfferLifecycleManager.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/fgs/OfferLifecycleManager.java index 135158f..e4cec83 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/fgs/OfferLifecycleManager.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/fgs/OfferLifecycleManager.java @@ -21,13 +21,17 @@ package org.apache.myriad.scheduler.fgs; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; + import javax.inject.Inject; + import org.apache.mesos.Protos; import org.apache.mesos.Protos.Offer; import org.apache.myriad.scheduler.MyriadDriver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; + /** * Manages the Mesos offers tracked by Myriad. */ @@ -110,4 +114,9 @@ public class OfferLifecycleManager { LOGGER.info("Declined {} outstanding offers for host {}", numOutStandingOffers, hostname); } } + + @VisibleForTesting + public ConsumedOffer getConsumedOffer(String hostname) { + return consumedOfferMap.get(hostname); + } } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/state/Cluster.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/state/Cluster.java b/myriad-scheduler/src/main/java/org/apache/myriad/state/Cluster.java index a5ccca7..2586720 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/state/Cluster.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/state/Cluster.java @@ -66,6 +66,10 @@ public class Cluster { public void removeNode(NodeTask task) { this.nodes.remove(task); } + + public void removeAllNodes() { + this.nodes.clear(); + } public String getResourceManagerHost() { return resourceManagerHost; http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/state/NodeTask.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/state/NodeTask.java b/myriad-scheduler/src/main/java/org/apache/myriad/state/NodeTask.java index 3a4ee56..5acd7cb 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/state/NodeTask.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/state/NodeTask.java @@ -21,6 +21,8 @@ package org.apache.myriad.state; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.inject.Inject; import java.util.List; + +import org.apache.commons.lang.StringUtils; import org.apache.mesos.Protos; import org.apache.mesos.Protos.Attribute; import org.apache.myriad.scheduler.ServiceResourceProfile; @@ -32,7 +34,7 @@ import org.apache.myriad.scheduler.constraints.Constraint; */ public class NodeTask { @JsonProperty - private String hostname; + private String hostname = StringUtils.EMPTY; @JsonProperty private Protos.SlaveID slaveId; @JsonProperty @@ -40,21 +42,22 @@ public class NodeTask { @JsonProperty private String taskPrefix; @JsonProperty - private ServiceResourceProfile serviceresourceProfile; + private ServiceResourceProfile profile; @Inject TaskUtils taskUtils; + /** * Mesos executor for this node. */ private Protos.ExecutorInfo executorInfo; private Constraint constraint; + private List<Attribute> slaveAttributes; public NodeTask(ServiceResourceProfile profile, Constraint constraint) { - this.serviceresourceProfile = profile; - this.hostname = ""; + this.profile = profile; this.constraint = constraint; } @@ -111,10 +114,10 @@ public class NodeTask { } public ServiceResourceProfile getProfile() { - return serviceresourceProfile; + return profile; } public void setProfile(ServiceResourceProfile serviceresourceProfile) { - this.serviceresourceProfile = serviceresourceProfile; + this.profile = serviceresourceProfile; } } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/state/utils/ByteBufferSupport.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/state/utils/ByteBufferSupport.java b/myriad-scheduler/src/main/java/org/apache/myriad/state/utils/ByteBufferSupport.java index b473b9e..9b0e643 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/state/utils/ByteBufferSupport.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/state/utils/ByteBufferSupport.java @@ -28,6 +28,9 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.mesos.Protos; import org.apache.myriad.scheduler.ServiceResourceProfile; @@ -50,7 +53,7 @@ public class ByteBufferSupport { public static void addByteBuffers(List<ByteBuffer> list, ByteArrayOutputStream bytes) throws IOException { // If list, add the list size, then the size of each buffer followed by the buffer. - if (list != null) { + if (CollectionUtils.isNotEmpty(list)) { bytes.write(toIntBytes(list.size())); for (ByteBuffer bb : list) { addByteBuffer(bb, bytes); @@ -61,7 +64,7 @@ public class ByteBufferSupport { } public static void addByteBuffer(ByteBuffer bb, ByteArrayOutputStream bytes) throws IOException { - if (bb != null && bytes != null) { + if (byteBufferNotEmpty(bb)) { bytes.write(toIntBytes(bb.array().length)); bytes.write(bb.array()); } @@ -199,16 +202,21 @@ public class ByteBufferSupport { */ public static NodeTask toNodeTask(ByteBuffer bb) { NodeTask nt = null; - if (bb != null && bb.array().length > 0) { + if (byteBufferNotEmpty(bb)) { nt = new NodeTask(getServiceResourceProfile(bb), getConstraint(bb)); nt.setHostname(toString(bb)); nt.setSlaveId(toSlaveId(bb)); nt.setTaskStatus(toTaskStatus(bb)); nt.setExecutorInfo(toExecutorInfo(bb)); + nt.setTaskPrefix(toString(bb)); } return nt; } + private static boolean byteBufferNotEmpty(ByteBuffer bb) { + return bb != null && bb.array().length > 0; + } + public static byte[] getTaskBytes(NodeTask nt) { if (nt.getTaskStatus() != null) { return nt.getTaskStatus().toByteArray(); @@ -234,7 +242,7 @@ public class ByteBufferSupport { } public static void putBytes(ByteBuffer bb, byte bytes[]) { - if (bytes != null && bytes.length > 0) { + if (ArrayUtils.isNotEmpty(bytes)) { bb.putInt(bytes.length); bb.put(bytes); } else { @@ -250,7 +258,7 @@ public class ByteBufferSupport { /** * This assumes the next position is the size as an int, and the following is a string - * iff the size is not zero. + * if the size is not zero. * * @param bb ByteBuffer to extract string from * @return string from the next position, or "" if the size is zero @@ -259,7 +267,7 @@ public class ByteBufferSupport { byte[] bytes = new byte[bb.getInt()]; String s = ""; try { - if (bytes.length > 0) { + if (ArrayUtils.isNotEmpty(bytes)) { bb.get(bytes); s = new String(bytes, UTF8); } @@ -279,7 +287,7 @@ public class ByteBufferSupport { public static ServiceResourceProfile getServiceResourceProfile(ByteBuffer bb) { String p = toString(bb); - if (!StringUtils.isEmpty(p)) { + if (StringUtils.isNotEmpty(p)) { return gsonCustom.fromJson(p, ServiceResourceProfile.class); } else { return null; @@ -295,7 +303,7 @@ public class ByteBufferSupport { case LIKE: - if (!StringUtils.isEmpty(p)) { + if (StringUtils.isNotEmpty(p)) { return gson.fromJson(p, LikeConstraint.class); } } @@ -363,5 +371,4 @@ public class ByteBufferSupport { public static ByteBuffer createBuffer(ByteBuffer bb) { return fillBuffer(getBytes(bb, bb.getInt())); } - -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadWebServer.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadWebServer.java b/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadWebServer.java index 126febc..60ef87d 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadWebServer.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadWebServer.java @@ -37,6 +37,11 @@ public class MyriadWebServer { private final Connector connector; private final GuiceFilter filter; + /** + * Status codes for MyriadWebServer + */ + public enum Status {STARTED, RUNNING, STOPPED, FAILED, UNKNOWN} + @Inject public MyriadWebServer(Server jetty, Connector connector, GuiceFilter filter) { this.jetty = jetty; @@ -71,8 +76,22 @@ public class MyriadWebServer { this.jetty.start(); } + public Status getStatus() { + if (jetty.isFailed()) { + return Status.FAILED; + } else if (jetty.isStarted()) { + return Status.STARTED; + } else if (jetty.isRunning()) { + return Status.RUNNING; + } else if (jetty.isStopped()) { + return Status.STOPPED; + } else { + return Status.UNKNOWN; + } + } + public void stop() throws Exception { this.jetty.stop(); this.connector.close(); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStoreTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStoreTest.java b/myriad-scheduler/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStoreTest.java new file mode 100644 index 0000000..a0a9ed1 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStoreTest.java @@ -0,0 +1,69 @@ +package org.apache.hadoop.yarn.server.resourcemanager.recovery; + +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.Service.STATE; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.myriad.state.MockDispatcher; +import org.apache.myriad.state.MockRMApp; +import org.junit.Test; + +/** + * Unit tests for MyriadFileSystemRMStateStore + */ +public class MyriadFileSystemRMStateStoreTest { + + @Test + public void testInit() throws Exception { + Configuration conf = getConfiguration(); + MyriadFileSystemRMStateStore store = new MyriadFileSystemRMStateStore(); + assertTrue(store.isInState(STATE.NOTINITED)); + store.init(conf); + assertTrue(store.isInState(STATE.INITED)); + store.startInternal(); + store.close(); + } + + @Test + public void testStartStop() throws Exception { + MyriadFileSystemRMStateStore store = getInitializedStore(); + store.start(); + assertTrue(store.isInState(STATE.STARTED)); + store.stop(); + assertTrue(store.isInState(STATE.STOPPED)); + store.close(); + } + + @Test + public void testStoreAndRemoveApplication() throws Exception { + MyriadFileSystemRMStateStore store = getInitializedStore(); + store.start(); + RMApp appOne = new MockRMApp(0, 0, RMAppState.NEW); + RMApp appTwo = new MockRMApp(0, 0, RMAppState.NEW); + + store.storeNewApplication(appOne); + store.storeNewApplication(appTwo); + store.removeApplication(appOne); + store.removeApplication(appTwo); + store.close(); + } + + private MyriadFileSystemRMStateStore getInitializedStore() throws Exception { + Configuration conf = getConfiguration(); + MyriadFileSystemRMStateStore store = new MyriadFileSystemRMStateStore(); + store.init(conf); + store.startInternal(); + store.loadState(); + store.loadMyriadState(); + store.setRMDispatcher(new MockDispatcher()); + return store; + } + + private Configuration getConfiguration() { + Configuration conf = new Configuration(); + conf.set("yarn.resourcemanager.fs.state-store.uri", "file:///tmp/"); + return conf; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/BaseConfigurableTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/BaseConfigurableTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/BaseConfigurableTest.java new file mode 100644 index 0000000..45443fe --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/BaseConfigurableTest.java @@ -0,0 +1,35 @@ +package org.apache.myriad; + +import org.apache.myriad.configuration.MyriadConfiguration; +import org.junit.Before; +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +/** + * Base class for all JUnit tests that require a MyriadConfiguration object. This class encapsulates the + * logic instantiate and configure a MyriadConfiguration object using all yml config files. + * + */ +public class BaseConfigurableTest { + protected MyriadConfiguration cfg; + protected MyriadConfiguration cfgWithRole; + protected MyriadConfiguration cfgWithDocker; + + @Before + public void setUp() throws Exception { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), + MyriadConfiguration.class); + cfgWithRole = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default-with-framework-role.yml"), + MyriadConfiguration.class); + cfgWithDocker = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default-with-docker-info.yml"), + MyriadConfiguration.class); + } + + @Test + public void testMyriadConfiguration() throws Exception { + cfg.getFrameworkName(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java index b36f32e..e8cb8b0 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/MultiBindingsTest.java @@ -18,15 +18,20 @@ package org.apache.myriad; -import com.google.inject.Guice; -import com.google.inject.Injector; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; import java.util.Map; + import org.apache.myriad.scheduler.TaskFactory; -import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; + +import com.google.common.collect.Lists; +import com.google.inject.Guice; +import com.google.inject.Injector; /** * Test for Multibindings @@ -34,22 +39,17 @@ import static org.junit.Assert.assertNotNull; public class MultiBindingsTest { private static Injector injector; + + private List<String> keyNames = Lists.newArrayList("nm", "jobhistory", "timelineserver"); @BeforeClass public static void setUpBeforeClass() throws Exception { MyriadTestModule myriadModule = new MyriadTestModule(); injector = Guice.createInjector(myriadModule); - - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { } @Test public void multiBindingsTest() { - - MultiBindingsUsage myinstance = injector.getInstance(MultiBindingsUsage.class); Map<String, TaskFactory> taskMap = myinstance.getMap(); @@ -58,12 +58,7 @@ public class MultiBindingsTest { taskMap = myinstance.getMap(); for (Map.Entry<String, TaskFactory> entry : taskMap.entrySet()) { - String keyName = entry.getKey(); - TaskFactory taskFactory = entry.getValue(); - System.out.println(taskFactory); + assertTrue(keyNames.contains(entry.getKey())); } - - } - -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/TestObjectFactory.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/TestObjectFactory.java b/myriad-scheduler/src/test/java/org/apache/myriad/TestObjectFactory.java new file mode 100644 index 0000000..c0cf187 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/TestObjectFactory.java @@ -0,0 +1,90 @@ +package org.apache.myriad; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.mesos.Protos; +import org.apache.mesos.Protos.FrameworkID; +import org.apache.mesos.Protos.Offer; +import org.apache.mesos.Protos.OfferID; +import org.apache.mesos.Protos.SlaveID; +import org.apache.myriad.configuration.MyriadConfiguration; +import org.apache.myriad.scheduler.MockSchedulerDriver; +import org.apache.myriad.scheduler.MyriadDriver; +import org.apache.myriad.scheduler.MyriadDriverManager; +import org.apache.myriad.scheduler.yarn.MyriadCapacityScheduler; +import org.apache.myriad.scheduler.yarn.interceptor.CompositeInterceptor; +import org.apache.myriad.scheduler.yarn.interceptor.InterceptorRegistry; +import org.apache.myriad.state.MockDispatcher; +import org.apache.myriad.state.SchedulerState; +import org.apache.myriad.webapp.HttpConnectorProvider; +import org.apache.myriad.webapp.MyriadWebServer; +import org.mortbay.jetty.Connector; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.servlet.DefaultServlet; +import org.mortbay.jetty.servlet.ServletHandler; +import org.mortbay.jetty.servlet.ServletHolder; + +import com.google.inject.servlet.GuiceFilter; + +/** + * Factory for common objects utilized over 1..n Junit tests + */ +public class TestObjectFactory { + public static SchedulerState getSchedulerState(MyriadConfiguration cfg) { + SchedulerState state = new SchedulerState(new MyriadFileSystemRMStateStore()); + state.setFrameworkId(FrameworkID.newBuilder().setValue("mock-framework").build()); + return state; + } + + public static MyriadDriverManager getMyriadDriverManager() { + return new MyriadDriverManager(new MyriadDriver(new MockSchedulerDriver())); + } + + public static InterceptorRegistry getInterceptorRegistry() { + return new CompositeInterceptor(); + } + + public static AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> getYarnScheduler() { + MyriadCapacityScheduler scheduler = new MyriadCapacityScheduler(); + return scheduler; + } + + public static Server getJettyServer() { + Server server = new Server(); + ServletHandler context = new ServletHandler(); + ServletHolder holder = new ServletHolder(DefaultServlet.class); + holder.setInitParameter("resourceBase", System.getProperty("user.dir")); + holder.setInitParameter("dirAllowed", "true"); + context.setServer(server); + context.addServlet(holder); + server.setHandler(context); + + return server; + } + + public static MyriadWebServer getMyriadWebServer(MyriadConfiguration cfg) { + Server server = TestObjectFactory.getJettyServer(); + HttpConnectorProvider provider = new HttpConnectorProvider(cfg); + Connector connector = provider.get(); + return new MyriadWebServer(server, connector, new GuiceFilter()); + } + + public static MyriadFileSystemRMStateStore getStateStore(Configuration conf) throws Exception { + conf.set("yarn.resourcemanager.fs.state-store.uri", "file:///tmp/"); + MyriadFileSystemRMStateStore store = new MyriadFileSystemRMStateStore(); + store.init(conf); + store.start(); + store.loadState(); + store.setRMDispatcher(new MockDispatcher()); + return store; + } + + public static Offer getOffer(String host, String slaveId, String frameworkId, String offerId) { + Protos.SlaveID sid = SlaveID.newBuilder().setValue(slaveId).build(); + Protos.FrameworkID fid = FrameworkID.newBuilder().setValue(frameworkId).build(); + return Protos.Offer.newBuilder().setHostname(host).setId(OfferID.newBuilder().setValue(offerId)).setSlaveId(sid).setFrameworkId(fid).build(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/api/ArtifactsResourceTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/api/ArtifactsResourceTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/api/ArtifactsResourceTest.java new file mode 100644 index 0000000..5d7bb75 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/api/ArtifactsResourceTest.java @@ -0,0 +1,51 @@ +package org.apache.myriad.api; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import javax.ws.rs.core.Response; + +import org.apache.myriad.BaseConfigurableTest; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests for ArtifactsResource + */ +public class ArtifactsResourceTest extends BaseConfigurableTest { + ArtifactsResource resource; + File configFile; + File binaryFile; + + @Before + public void setUp() throws Exception { + super.setUp(); + configFile = new File("/tmp/myriadEtc"); + binaryFile = new File("/tmp/myriadBinary"); + assertTrue(configFile.createNewFile()); + assertTrue(binaryFile.createNewFile()); + resource = new ArtifactsResource(cfg); + } + + @Test + public void testGetConfig() throws Exception { + Response res = resource.getConfig(); + assertEquals(configFile, res.getEntity()); + assertEquals(200, res.getStatus()); + } + + @Test + public void testGetBinary() throws Exception { + Response res = resource.getBinary(); + assertEquals(binaryFile, res.getEntity()); + assertEquals(200, res.getStatus()); + } + + @After + public void tearDown() throws Exception { + assertTrue(new File("/tmp/myriadEtc").delete()); + assertTrue(new File("/tmp/myriadBinary").delete()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/api/SchedulerStateResourceTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/api/SchedulerStateResourceTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/api/SchedulerStateResourceTest.java new file mode 100644 index 0000000..e57b128 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/api/SchedulerStateResourceTest.java @@ -0,0 +1,58 @@ +package org.apache.myriad.api; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore; +import org.apache.mesos.Protos; +import org.apache.mesos.Protos.FrameworkID; +import org.apache.mesos.Protos.TaskID; +import org.apache.myriad.BaseConfigurableTest; +import org.apache.myriad.api.model.GetSchedulerStateResponse; +import org.apache.myriad.scheduler.ServiceResourceProfile; +import org.apache.myriad.scheduler.constraints.LikeConstraint; +import org.apache.myriad.state.NodeTask; +import org.apache.myriad.state.SchedulerState; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests for SchedulerStateResource + */ +public class SchedulerStateResourceTest extends BaseConfigurableTest { + SchedulerStateResource resource; + TaskID idOne, idTwo, idThree; + + @Before + public void setUp() throws Exception { + super.setUp(); + resource = new SchedulerStateResource(cfg, getSchedulerState()); + } + + private SchedulerState getSchedulerState() throws Exception { + SchedulerState state = new SchedulerState(new MyriadFileSystemRMStateStore()); + idOne = Protos.TaskID.newBuilder().setValue("nt-1").build(); + idTwo = Protos.TaskID.newBuilder().setValue("nt-2").build(); + idThree = Protos.TaskID.newBuilder().setValue("nt-3").build(); + + state.addTask(idOne, new NodeTask(new ServiceResourceProfile("profile1", 0.2, 1024.0), new LikeConstraint("localhost", "host-[0-9]*.example.com"))); + state.addTask(idTwo, new NodeTask(new ServiceResourceProfile("profile2", 0.4, 2048.0), new LikeConstraint("localhost", "host-[0-9]*.example.com"))); + state.addTask(idThree, new NodeTask(new ServiceResourceProfile("profile3", 0.6, 3072.0), new LikeConstraint("localhost", "host-[0-9]*.example.com"))); + + state.setFrameworkId(FrameworkID.newBuilder().setValue("mock-framework").build()); + state.makeTaskActive(idOne); + state.makeTaskPending(idTwo); + state.makeTaskStaging(idThree); + + return state; + } + + @Test + public void test() throws Exception { + GetSchedulerStateResponse response = resource.getState(); + assertNotNull(response); + assertEquals(1, response.getActiveTasks().size()); + assertEquals(1, response.getPendingTasks().size()); + assertEquals(1, response.getStagingTasks().size()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java index 22df23d..562d128 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/configuration/MyriadConfigurationTest.java @@ -21,71 +21,51 @@ package org.apache.myriad.configuration; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; +import java.util.Iterator; import java.util.Map; +import java.util.Set; -import org.junit.BeforeClass; +import org.apache.myriad.BaseConfigurableTest; import org.junit.Test; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.google.common.collect.Sets; /** - * AuxServices/tasks test + * Unit tests for MyriadConfiguration */ -public class MyriadConfigurationTest { +public class MyriadConfigurationTest extends BaseConfigurableTest { - static MyriadConfiguration cfg; + public void testMyriadContainerConfiguration() throws Exception { + MyriadContainerConfiguration conf = cfgWithDocker.getContainerInfo().get(); + assertTrue(conf.getDockerInfo().isPresent()); - @BeforeClass - public static void setUpBeforeClass() throws Exception { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), - MyriadConfiguration.class); + MyriadDockerConfiguration dConf = conf.getDockerInfo().get(); + assertEquals(false, dConf.getForcePullImage()); + assertEquals("mesos/myriad", dConf.getImage()); - } - - @Test - public void serviceConfigurationTest() throws Exception { - Map<String, ServiceConfiguration> auxConfigs = cfg.getServiceConfigurations(); - - assertEquals(auxConfigs.size(), 2); - - for (Map.Entry<String, ServiceConfiguration> entry : auxConfigs.entrySet()) { - String taskName = entry.getKey(); - ServiceConfiguration config = entry.getValue(); - String outTaskname = config.getTaskName(); - assertEquals(taskName, outTaskname); + assertNotNull(conf.getVolumes()); + + Set<String> keys = Sets.newHashSet("hostPath", "containerPath", "mode"); + Set<String> modes = Sets.newHashSet("RO", "RW"); + Iterator<Map<String, String>> iter = conf.getVolumes().iterator(); + + while (iter.hasNext()) { + Map<String, String> mcConf = iter.next(); + assertEquals(keys, mcConf.keySet()); + assertTrue(modes.contains(mcConf.get("mode"))); } } - + @Test - public void coreConfigurationTest() throws Exception { - assertEquals("MyriadTest", cfg.getFrameworkName()); - - //authorization parameters + public void testRoles() throws Exception { + assertEquals("test", cfgWithRole.getFrameworkRole()); assertEquals("*", cfg.getFrameworkRole()); - assertEquals("hduser", cfg.getFrameworkUser().get()); - assertEquals("root", cfg.getFrameworkSuperUser().get()); - - //ports and directory paths - assertEquals("10.0.2.15:5050", cfg.getMesosMaster()); - assertEquals("/usr/local/lib/libmesos.so", cfg.getNativeLibrary()); - assertEquals(new Integer(8192), cfg.getRestApiPort()); - assertEquals("10.0.2.15:2181", cfg.getZkServers()); - - //timeouts - assertEquals(new Double(44200000), cfg.getFrameworkFailoverTimeout()); - assertEquals(new Integer(25000), cfg.getZkTimeout()); - - //checkpoints - assertEquals(false, cfg.isCheckpoint()); - assertEquals(true, cfg.isHAEnabled()); - assertEquals(false, cfg.isRebalancerEnabled()); } - + @Test - public void executorConfigurationTest() throws Exception { + public void testExecutorConfiguration() throws Exception { MyriadExecutorConfiguration conf = cfg.getMyriadExecutorConfiguration(); assertEquals(new Double(256), conf.getJvmMaxMemoryMB()); @@ -94,7 +74,18 @@ public class MyriadConfigurationTest { } @Test - public void nodeManagerConfigurationTest() throws Exception { + public void testServiceConfigurations() throws Exception { + Map<String, ServiceConfiguration> confs = cfg.getServiceConfigurations(); + Set<String> configKeys = Sets.newHashSet("jobhistory", "timelineserver"); + + assertEquals(configKeys, confs.keySet()); + ServiceConfiguration sConfig = confs.get("jobhistory"); + assertEquals(new Double(1.0), sConfig.getCpus()); + assertEquals("jobhistory", sConfig.getTaskName()); + } + + @Test + public void testNodeManagerConfiguration() throws Exception { NodeManagerConfiguration config = cfg.getNodeManagerConfiguration(); assertFalse(config.getCgroups()); @@ -103,7 +94,7 @@ public class MyriadConfigurationTest { } @Test - public void profilesConfigurationTest() throws Exception { + public void testProfilesConfiguration() throws Exception { Map<String, Map<String, String>> profiles = cfg.getProfiles(); for (Map.Entry<String, Map<String, String>> profile : profiles.entrySet()) { @@ -112,9 +103,9 @@ public class MyriadConfigurationTest { } private boolean validateProfile(Map.Entry<String, Map<String, String>> entry) { - String key = entry.getKey(); + String key = entry.getKey(); Map<String, String> value = entry.getValue(); - + switch (key) { case "small" : { return value.get("cpu").equals("1") && value.get("mem").equals("1100"); http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/health/HealthCheckUtilsTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/health/HealthCheckUtilsTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/health/HealthCheckUtilsTest.java new file mode 100644 index 0000000..e403f90 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/health/HealthCheckUtilsTest.java @@ -0,0 +1,27 @@ +package org.apache.myriad.health; + +import java.net.ServerSocket; + +import org.junit.Test; + +/** + * Unit tests for HealthCheckUtils class + */ +public class HealthCheckUtilsTest { + @Test(expected = IllegalArgumentException.class) + public void testInvalidHost() throws Exception { + HealthCheckUtils.checkHostPort("localhost-8000"); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidPort() throws Exception { + HealthCheckUtils.checkHostPort("localhost:ab12"); + } + + @Test + public void testValidHostPortString() throws Exception { + ServerSocket socket = new ServerSocket(8000); + HealthCheckUtils.checkHostPort("localhost:8000"); + socket.close(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/health/MesosDriverHealthCheckTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/health/MesosDriverHealthCheckTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/health/MesosDriverHealthCheckTest.java new file mode 100644 index 0000000..cebf2c7 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/health/MesosDriverHealthCheckTest.java @@ -0,0 +1,69 @@ +package org.apache.myriad.health; + +import static org.junit.Assert.assertEquals; + +import org.apache.myriad.scheduler.MockSchedulerDriver; +import org.apache.myriad.scheduler.MyriadDriver; +import org.apache.myriad.scheduler.MyriadDriverManager; +import org.junit.Test; + +import com.codahale.metrics.health.HealthCheck.Result; + +/** + * Unit tests for MesosDriverHealthCheck + */ +public class MesosDriverHealthCheckTest { + + private static class HealthCheckTestTuple { + MyriadDriverManager manager; + MesosDriverHealthCheck checker; + + private HealthCheckTestTuple(MyriadDriverManager manager) { + this.manager = manager; + this.checker = new MesosDriverHealthCheck(manager); + } + + public MyriadDriverManager getManager() { + return manager; + } + + public MesosDriverHealthCheck getChecker() { + return checker; + } + } + + private HealthCheckTestTuple getTestStack() { + MyriadDriver driver = new MyriadDriver(new MockSchedulerDriver()); + return new HealthCheckTestTuple(new MyriadDriverManager(driver)); + } + + @Test + public void testCheckHealthyResult() throws Exception { + HealthCheckTestTuple tuple = getTestStack(); + MyriadDriverManager manager = tuple.getManager(); + MesosDriverHealthCheck checker = tuple.getChecker(); + manager.startDriver(); + assertEquals(Result.healthy(), checker.check()); + manager.stopDriver(false); + } + + @Test + public void testCheckStoppedDriverUnhealthyResult() throws Exception { + HealthCheckTestTuple tuple = getTestStack(); + MyriadDriverManager manager = tuple.getManager(); + MesosDriverHealthCheck checker = tuple.getChecker(); + manager.startDriver(); + manager.stopDriver(false); + assertEquals(Result.unhealthy("Driver status: DRIVER_STOPPED"), checker.check()); + } + + @Test + public void testCheckAbortedDriverUnhealthyResult() throws Exception { + HealthCheckTestTuple tuple = getTestStack(); + MyriadDriverManager manager = tuple.getManager(); + MesosDriverHealthCheck checker = tuple.getChecker(); + manager.startDriver(); + manager.abortDriver(); + assertEquals(Result.unhealthy("Driver status: DRIVER_ABORTED"), checker.check()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MockSchedulerDriver.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MockSchedulerDriver.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MockSchedulerDriver.java new file mode 100644 index 0000000..2a60e58 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MockSchedulerDriver.java @@ -0,0 +1,124 @@ +package org.apache.myriad.scheduler; + +import java.util.Collection; + +import org.apache.mesos.Protos; +import org.apache.mesos.Protos.ExecutorID; +import org.apache.mesos.Protos.Filters; +import org.apache.mesos.Protos.Offer.Operation; +import org.apache.mesos.Protos.OfferID; +import org.apache.mesos.Protos.Request; +import org.apache.mesos.Protos.SlaveID; +import org.apache.mesos.Protos.Status; +import org.apache.mesos.Protos.TaskID; +import org.apache.mesos.Protos.TaskInfo; +import org.apache.mesos.Protos.TaskStatus; +import org.apache.mesos.SchedulerDriver; + +/** + * Mock SchedulerDriver implementation for JUnit tests + */ +public class MockSchedulerDriver implements SchedulerDriver { + + @Override + public Status start() { + return Protos.Status.DRIVER_RUNNING; + } + + @Override + public Status stop(boolean failover) { + return Protos.Status.DRIVER_STOPPED; + } + + @Override + public Status stop() { + return Protos.Status.DRIVER_STOPPED; + } + + @Override + public Status abort() { + return Protos.Status.DRIVER_ABORTED; + } + + @Override + public Status join() { + return Protos.Status.DRIVER_RUNNING; + } + + @Override + public Status run() { + return Protos.Status.DRIVER_RUNNING; + } + + @Override + public Status requestResources(Collection<Request> requests) { + return null; + } + + @Override + public Status launchTasks(Collection<OfferID> offerIds, Collection<TaskInfo> tasks, Filters filters) { + return null; + } + + @Override + public Status launchTasks(Collection<OfferID> offerIds, Collection<TaskInfo> tasks) { + return null; + } + + @Override + @SuppressWarnings("deprecation") + public Status launchTasks(OfferID offerId, Collection<TaskInfo> tasks, Filters filters) { + return null; + } + + @Override + @SuppressWarnings("deprecation") + public Status launchTasks(OfferID offerId, Collection<TaskInfo> tasks) { + return null; + } + + @Override + public Status killTask(TaskID taskId) { + return null; + } + + @Override + public Status acceptOffers(Collection<OfferID> offerIds, Collection<Operation> operations, Filters filters) { + return null; + } + + @Override + public Status declineOffer(OfferID offerId, Filters filters) { + return null; + } + + @Override + public Status declineOffer(OfferID offerId) { + return null; + } + + @Override + public Status reviveOffers() { + return null; + } + + @Override + public Status suppressOffers() { + return null; + } + + @Override + public Status acknowledgeStatusUpdate(TaskStatus status) { + return null; + } + + @Override + public Status sendFrameworkMessage(ExecutorID executorId, SlaveID slaveId, byte[] data) { + return null; + } + + @Override + public Status reconcileTasks(Collection<TaskStatus> statuses) { + return null; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MyriadDriverTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MyriadDriverTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MyriadDriverTest.java new file mode 100644 index 0000000..d384150 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MyriadDriverTest.java @@ -0,0 +1,43 @@ +package org.apache.myriad.scheduler; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.mesos.Protos; +import org.apache.mesos.Protos.Status; +import org.apache.mesos.SchedulerDriver; +import org.junit.Test; + +/** + * Unit test for MyriadDriver class + */ +public class MyriadDriverTest { + @Test + public void testStart() throws Exception { + MyriadDriver driver = new MyriadDriver(new MockSchedulerDriver()); + Status status = driver.start(); + assertEquals(Protos.Status.DRIVER_RUNNING_VALUE, status.getNumber()); + } + + @Test + public void testAbort() throws Exception { + MyriadDriver driver = new MyriadDriver(new MockSchedulerDriver()); + Status status = driver.abort(); + assertEquals(Protos.Status.DRIVER_ABORTED_VALUE, status.getNumber()); + } + + @Test + public void testStop() throws Exception { + MyriadDriver driver = new MyriadDriver(new MockSchedulerDriver()); + Status status = driver.stop(true); + assertEquals(Protos.Status.DRIVER_STOPPED_VALUE, status.getNumber()); + } + + @Test + public void testGetDriver() throws Exception { + MyriadDriver driver = new MyriadDriver(new MockSchedulerDriver()); + SchedulerDriver sDriver = driver.getDriver(); + + assertTrue(sDriver instanceof MockSchedulerDriver); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MyriadOperationsTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MyriadOperationsTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MyriadOperationsTest.java new file mode 100644 index 0000000..6c039dd --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/MyriadOperationsTest.java @@ -0,0 +1,112 @@ +package org.apache.myriad.scheduler; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; +import org.apache.mesos.Protos.FrameworkID; +import org.apache.myriad.BaseConfigurableTest; +import org.apache.myriad.TestObjectFactory; +import org.apache.myriad.configuration.MyriadBadConfigurationException; +import org.apache.myriad.policy.LeastAMNodesFirstPolicy; +import org.apache.myriad.scheduler.constraints.Constraint; +import org.apache.myriad.scheduler.constraints.LikeConstraint; +import org.apache.myriad.scheduler.yarn.interceptor.CompositeInterceptor; +import org.apache.myriad.state.MockDispatcher; +import org.apache.myriad.state.MockRMContext; +import org.apache.myriad.state.SchedulerState; +import org.apache.myriad.webapp.MyriadWebServer; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests for MyriadOperations class + */ +public class MyriadOperationsTest extends BaseConfigurableTest { + MyriadOperations ops; + ServiceResourceProfile small; + Constraint constraint = new LikeConstraint("localhost", "host-[0-9]*.example.com"); + SchedulerState sState; + + @Before + public void setUp() throws Exception { + super.setUp(); + AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> scheduler = TestObjectFactory.getYarnScheduler(); + //sState = new SchedulerState(new MyriadFileSystemRMStateStore()); + sState = TestObjectFactory.getSchedulerState(cfg); + sState.setFrameworkId(FrameworkID.newBuilder().setValue("mock-framework").build()); + + MyriadDriverManager manager = TestObjectFactory.getMyriadDriverManager(); + MyriadWebServer webServer = TestObjectFactory.getMyriadWebServer(cfg); + CompositeInterceptor registry = new CompositeInterceptor(); + LeastAMNodesFirstPolicy policy = new LeastAMNodesFirstPolicy(registry, scheduler, sState); + + manager.startDriver(); + + ops = new MyriadOperations(cfg, sState, policy, manager, webServer, generateRMContext(scheduler)); + generateProfiles(); + } + + private void generateProfiles() { + small = new ServiceResourceProfile("small", 0.1, 512.0); + } + + private RMContext generateRMContext(AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> scheduler) throws Exception { + Configuration conf = new Configuration(); + MockRMContext context = null; + Dispatcher dispatcher = new MockDispatcher(); + + RMApplicationHistoryWriter rmApplicationHistoryWriter = new RMApplicationHistoryWriter(); + AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor(dispatcher); + AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor(dispatcher); + RMDelegationTokenSecretManager delegationTokenSecretManager = new RMDelegationTokenSecretManager(1, 1, 1, 1, context); + + context = new MockRMContext(); + context.setStateStore(TestObjectFactory.getStateStore(conf)); + context.setAmLivelinessMonitor(amLivelinessMonitor); + context.setAmFinishingMonitor(amFinishingMonitor); + context.setRMApplicationHistoryWriter(rmApplicationHistoryWriter); + context.setRMDelegationTokenSecretManager(delegationTokenSecretManager); + return context; + } + + @Test + public void testFlexUpAndFlexDownCluster() throws Exception { + assertEquals(0, sState.getPendingTaskIds().size()); + ops.flexUpCluster(small, 1, constraint); + assertEquals(1, sState.getPendingTaskIds().size()); + ops.flexDownCluster(small, constraint, 1); + assertEquals(0, sState.getPendingTaskIds().size()); + } + + @Test + public void testFlexUpAndFlexDownService() throws Exception { + ops.flexUpAService(1, "jobhistory"); + assertEquals(1, sState.getPendingTasksByType("jobhistory").size()); + ops.flexDownAService(1, "jobhistory"); + assertEquals(0, sState.getPendingTasksByType("jobhistory").size()); + } + + @Test(expected = MyriadBadConfigurationException.class) + public void testFlexUpAServiceOverMaxInstances() throws Exception { + ops.flexUpAService(2, "jobhistory"); + } + + @Test + public void testGetFlexibleInstances() throws Exception { + ops.flexUpAService(1, "jobhistory"); + assertEquals(1, ops.getFlexibleInstances("jobhistory").intValue()); + } + + @Test + public void testShutdownCluster() throws Exception { + ops.shutdownFramework(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/NMProfileManagerTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/NMProfileManagerTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/NMProfileManagerTest.java new file mode 100644 index 0000000..a0aab74 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/NMProfileManagerTest.java @@ -0,0 +1,63 @@ +package org.apache.myriad.scheduler; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import org.junit.Test; + +/** + * Unit test cases for NMProfileManager + */ +public class NMProfileManagerTest { + private NMProfileManager getNMProfileManager() { + NMProfileManager manager = new NMProfileManager(); + NMProfile profile1 = new NMProfile("profile1", 1L, 512L); + NMProfile profile2 = new NMProfile("profile2", 2L, 1024L); + NMProfile profile3 = new NMProfile("profile3", 3L, 2048L); + NMProfile profile4 = new NMProfile("profile4", 4L, 3072L); + NMProfile profile5 = new NMProfile("profile5", 5L, 4096L); + + manager.add(profile1); + manager.add(profile2); + manager.add(profile3); + manager.add(profile4); + manager.add(profile5); + + return manager; + } + + @Test + public void testAdd() throws Exception { + NMProfileManager manager = this.getNMProfileManager(); + assertEquals(5, manager.numberOfProfiles()); + } + + @Test + public void testRetrieval() throws Exception { + NMProfileManager manager = this.getNMProfileManager(); + assertEquals("profile1", manager.get("profile1").getName()); + assertEquals("profile2", manager.get("profile2").getName()); + assertEquals("profile3", manager.get("profile3").getName()); + assertEquals("profile4", manager.get("profile4").getName()); + assertEquals("profile5", manager.get("profile5").getName()); + } + + @Test + public void testExists() throws Exception { + NMProfileManager manager = this.getNMProfileManager(); + assertTrue(manager.exists("profile1")); + assertTrue(manager.exists("profile2")); + assertTrue(manager.exists("profile3")); + assertTrue(manager.exists("profile4")); + assertTrue(manager.exists("profile5")); + } + @Test + public void testToString() throws Exception { + NMProfileManager manager = this.getNMProfileManager(); + String toString = manager.toString(); + assertTrue(toString.contains("\"name\":\"profile1\",\"cpus\":1,\"memory\":512")); + assertTrue(toString.contains("\"name\":\"profile2\",\"cpus\":2,\"memory\":1024")); + assertTrue(toString.contains("\"name\":\"profile3\",\"cpus\":3,\"memory\":2048")); + assertTrue(toString.contains("\"name\":\"profile4\",\"cpus\":4,\"memory\":3072")); + assertTrue(toString.contains("\"name\":\"profile5\",\"cpus\":5,\"memory\":4096")); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/ServiceResourceProfileTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/ServiceResourceProfileTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/ServiceResourceProfileTest.java new file mode 100644 index 0000000..72e0092 --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/ServiceResourceProfileTest.java @@ -0,0 +1,31 @@ +package org.apache.myriad.scheduler; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit test cases for ServiceResourceProfile + * + */ +public class ServiceResourceProfileTest { + ServiceResourceProfile profile; + + @Before + public void setUp() throws Exception { + profile = new ServiceResourceProfile("ServiceResourceProfile", 0.1, 1024.0, 0.1, 512.0); + } + + @Test + public void testRequestedResources() throws Exception { + Assert.assertEquals(new Double(0.1), profile.getCpus()); + Assert.assertEquals(new Double(1024.0), profile.getMemory()); + Assert.assertEquals(new Double(0.1), profile.getExecutorCpu()); + Assert.assertEquals(new Double(512.0), profile.getExecutorMemory()); + } + + @Test + public void testName() throws Exception { + Assert.assertEquals("ServiceResourceProfile", profile.getName()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TaskConstraintsManagerTest.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TaskConstraintsManagerTest.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TaskConstraintsManagerTest.java new file mode 100644 index 0000000..94946ce --- /dev/null +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TaskConstraintsManagerTest.java @@ -0,0 +1,32 @@ +package org.apache.myriad.scheduler; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.myriad.BaseConfigurableTest; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests for TaskConstraintsManager + */ +public class TaskConstraintsManagerTest extends BaseConfigurableTest { + TaskConstraintsManager manager = new TaskConstraintsManager(); + + @Before + public void setUp() throws Exception { + super.setUp(); + manager.addTaskConstraints("jobhistory", new ServiceTaskConstraints(cfg, "jobhistory")); + } + + @Test + public void testAddConstraints() throws Exception { + assertTrue(manager.exists("jobhistory")); + } + + @Test + public void testGetConstraints() throws Exception { + TaskConstraints tCon = manager.getConstraints("jobhistory"); + assertEquals(3, tCon.portsCount()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java index e49c19c..87aee56 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java @@ -18,24 +18,17 @@ */ package org.apache.myriad.scheduler; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import static org.junit.Assert.assertTrue; + import org.apache.mesos.Protos.CommandInfo; -import org.apache.myriad.configuration.MyriadConfiguration; +import org.apache.myriad.BaseConfigurableTest; import org.apache.myriad.scheduler.TaskFactory.NMTaskFactoryImpl; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertTrue; - /** * Class to test CommandLine generation */ -public class TestServiceCommandLine { - - static MyriadConfiguration cfg; - +public class TestServiceCommandLine extends BaseConfigurableTest { static String toJHSCompare = "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; " + "export TASK_DIR=`basename $PWD`; sudo chmod +wx /sys/fs/cgroup/cpu/mesos/$TASK_DIR;" + @@ -44,18 +37,6 @@ public class TestServiceCommandLine { static String toCompare = "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;"; - @BeforeClass - public static void setUpBeforeClass() throws Exception { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), - MyriadConfiguration.class); - - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - } - @Test public void testJHSCommandLineGeneration() throws Exception { ServiceTaskFactoryImpl jhs = new ServiceTaskFactoryImpl(cfg, null); http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java index 06fa698..29d7691 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java @@ -17,48 +17,25 @@ */ package org.apache.myriad.scheduler; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.google.common.collect.Range; -import com.google.common.collect.Ranges; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import org.apache.mesos.Protos; +import org.apache.myriad.BaseConfigurableTest; import org.apache.myriad.configuration.MyriadBadConfigurationException; -import org.apache.myriad.configuration.MyriadConfiguration; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import com.google.common.collect.Range; +import com.google.common.collect.Ranges; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; /** * Tests for TaskUtils */ -public class TestTaskUtils { - - static MyriadConfiguration cfg; - static MyriadConfiguration cfgWithRole; - static MyriadConfiguration cfgWithDocker; +public class TestTaskUtils extends BaseConfigurableTest { static double epsilon = .0001; - @BeforeClass - public static void setUpBeforeClass() throws Exception { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"), - MyriadConfiguration.class); - cfgWithRole = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default-with-framework-role.yml"), - MyriadConfiguration.class); - cfgWithDocker = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default-with-docker-info.yml"), - MyriadConfiguration.class); - - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - } - @Test public void testGetResource() { TaskUtils taskUtils = new TaskUtils(cfg);
