http://git-wip-us.apache.org/repos/asf/hadoop/blob/84cea001/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java index 2b1971a..794cd47 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java @@ -17,20 +17,25 @@ */ package org.apache.hadoop.yarn.sls; +import org.apache.commons.math3.random.JDKRandomGenerator; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.tools.rumen.TaskAttemptInfo; import org.apache.hadoop.yarn.sls.synthetic.SynthJob; import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Arrays; import static org.junit.Assert.assertTrue; +import static org.codehaus.jackson.JsonParser.Feature.INTERN_FIELD_NAMES; +import static org.codehaus.jackson.map.DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES; + /** * Simple test class driving the {@code SynthTraceJobProducer}, and validating * jobs produce are within expected range. @@ -38,10 +43,60 @@ import static org.junit.Assert.assertTrue; public class TestSynthJobGeneration { public final static Logger LOG = - Logger.getLogger(TestSynthJobGeneration.class); + LoggerFactory.getLogger(TestSynthJobGeneration.class); @Test - public void test() throws IllegalArgumentException, IOException { + public void testWorkloadGenerateTime() + throws IllegalArgumentException, IOException { + + String workloadJson = "{\"job_classes\": [], \"time_distribution\":[" + + "{\"time\": 0, \"weight\": 1}, " + "{\"time\": 30, \"weight\": 0}," + + "{\"time\": 60, \"weight\": 2}," + "{\"time\": 90, \"weight\": 1}" + + "]}"; + + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(INTERN_FIELD_NAMES, true); + mapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false); + SynthTraceJobProducer.Workload wl = + mapper.readValue(workloadJson, SynthTraceJobProducer.Workload.class); + + JDKRandomGenerator rand = new JDKRandomGenerator(); + rand.setSeed(0); + + wl.init(rand); + + int bucket0 = 0; + int bucket1 = 0; + int bucket2 = 0; + int bucket3 = 0; + for (int i = 0; i < 1000; ++i) { + long time = wl.generateSubmissionTime(); + LOG.info("Generated time " + time); + if (time < 30) { + bucket0++; + } else if (time < 60) { + bucket1++; + } else if (time < 90) { + bucket2++; + } else { + bucket3++; + } + } + + Assert.assertTrue(bucket0 > 0); + Assert.assertTrue(bucket1 == 0); + Assert.assertTrue(bucket2 > 0); + Assert.assertTrue(bucket3 > 0); + Assert.assertTrue(bucket2 > bucket0); + Assert.assertTrue(bucket2 > bucket3); + + LOG.info("bucket0 {}, bucket1 {}, bucket2 {}, bucket3 {}", bucket0, bucket1, + bucket2, bucket3); + + } + + @Test + public void testMapReduce() throws IllegalArgumentException, IOException { Configuration conf = new Configuration(); @@ -50,47 +105,155 @@ public class TestSynthJobGeneration { SynthTraceJobProducer stjp = new SynthTraceJobProducer(conf); + LOG.info(stjp.toString()); + SynthJob js = (SynthJob) stjp.getNextJob(); int jobCount = 0; while (js != null) { - LOG.info((jobCount++) + " " + js.getQueueName() + " -- " - + js.getJobClass().getClassName() + " (conf: " - + js.getJobConf().get(MRJobConfig.QUEUE_NAME) + ") " + " submission: " - + js.getSubmissionTime() + ", " + " duration: " + js.getDuration() - + " numMaps: " + js.getNumberMaps() + " numReduces: " - + js.getNumberReduces()); + LOG.info(js.toString()); + validateJob(js); + js = (SynthJob) stjp.getNextJob(); + jobCount++; + } + Assert.assertEquals(stjp.getNumJobs(), jobCount); + } + + @Test + public void testGeneric() throws IllegalArgumentException, IOException { + Configuration conf = new Configuration(); + + conf.set(SynthTraceJobProducer.SLS_SYNTHETIC_TRACE_FILE, + "src/test/resources/syn_generic.json"); + + SynthTraceJobProducer stjp = new SynthTraceJobProducer(conf); + + LOG.info(stjp.toString()); + + SynthJob js = (SynthJob) stjp.getNextJob(); + + int jobCount = 0; + + while (js != null) { + LOG.info(js.toString()); validateJob(js); js = (SynthJob) stjp.getNextJob(); + jobCount++; } Assert.assertEquals(stjp.getNumJobs(), jobCount); } - private void validateJob(SynthJob js) { + @Test + public void testStream() throws IllegalArgumentException, IOException { + Configuration conf = new Configuration(); - assertTrue(js.getSubmissionTime() > 0); - assertTrue(js.getDuration() > 0); - assertTrue(js.getNumberMaps() >= 0); - assertTrue(js.getNumberReduces() >= 0); - assertTrue(js.getNumberMaps() + js.getNumberReduces() > 0); - assertTrue(js.getTotalSlotTime() >= 0); + conf.set(SynthTraceJobProducer.SLS_SYNTHETIC_TRACE_FILE, + "src/test/resources/syn_stream.json"); + + SynthTraceJobProducer stjp = new SynthTraceJobProducer(conf); + + LOG.info(stjp.toString()); + + SynthJob js = (SynthJob) stjp.getNextJob(); + + int jobCount = 0; + + while (js != null) { + LOG.info(js.toString()); + validateJob(js); + js = (SynthJob) stjp.getNextJob(); + jobCount++; + } + + Assert.assertEquals(stjp.getNumJobs(), jobCount); + } - for (int i = 0; i < js.getNumberMaps(); i++) { - TaskAttemptInfo tai = js.getTaskAttemptInfo(TaskType.MAP, i, 0); - assertTrue(tai.getRuntime() > 0); + @Test + public void testSample() throws IOException { + + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(INTERN_FIELD_NAMES, true); + mapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false); + + JDKRandomGenerator rand = new JDKRandomGenerator(); + rand.setSeed(0); + + String valJson = "{\"val\" : 5 }"; + SynthTraceJobProducer.Sample valSample = + mapper.readValue(valJson, SynthTraceJobProducer.Sample.class); + valSample.init(rand); + int val = valSample.getInt(); + Assert.assertEquals(5, val); + + String distJson = "{\"val\" : 5, \"std\" : 1 }"; + SynthTraceJobProducer.Sample distSample = + mapper.readValue(distJson, SynthTraceJobProducer.Sample.class); + distSample.init(rand); + double dist = distSample.getDouble(); + Assert.assertTrue(dist > 2 && dist < 8); + + String normdistJson = "{\"val\" : 5, \"std\" : 1, \"dist\": \"NORM\" }"; + SynthTraceJobProducer.Sample normdistSample = + mapper.readValue(normdistJson, SynthTraceJobProducer.Sample.class); + normdistSample.init(rand); + double normdist = normdistSample.getDouble(); + Assert.assertTrue(normdist > 2 && normdist < 8); + + String discreteJson = "{\"discrete\" : [2, 4, 6, 8]}"; + SynthTraceJobProducer.Sample discreteSample = + mapper.readValue(discreteJson, SynthTraceJobProducer.Sample.class); + discreteSample.init(rand); + int discrete = discreteSample.getInt(); + Assert.assertTrue( + Arrays.asList(new Integer[] {2, 4, 6, 8}).contains(discrete)); + + String discreteWeightsJson = + "{\"discrete\" : [2, 4, 6, 8], " + "\"weights\": [0, 0, 0, 1]}"; + SynthTraceJobProducer.Sample discreteWeightsSample = mapper + .readValue(discreteWeightsJson, SynthTraceJobProducer.Sample.class); + discreteWeightsSample.init(rand); + int discreteWeights = discreteWeightsSample.getInt(); + Assert.assertEquals(8, discreteWeights); + + String invalidJson = "{\"val\" : 5, \"discrete\" : [2, 4, 6, 8], " + + "\"weights\": [0, 0, 0, 1]}"; + try { + mapper.readValue(invalidJson, SynthTraceJobProducer.Sample.class); + Assert.fail(); + } catch (JsonMappingException e) { + Assert.assertTrue(e.getMessage().startsWith("Instantiation of")); } - for (int i = 0; i < js.getNumberReduces(); i++) { - TaskAttemptInfo tai = js.getTaskAttemptInfo(TaskType.REDUCE, i, 0); - assertTrue(tai.getRuntime() > 0); + String invalidDistJson = + "{\"val\" : 5, \"std\" : 1, " + "\"dist\": \"INVALID\" }"; + try { + mapper.readValue(invalidDistJson, SynthTraceJobProducer.Sample.class); + Assert.fail(); + } catch (JsonMappingException e) { + Assert.assertTrue(e.getMessage().startsWith("Instantiation of")); } + } + + private void validateJob(SynthJob js) { + + assertTrue(js.getSubmissionTime() > 0); + assertTrue(js.getDuration() > 0); + assertTrue(js.getTotalSlotTime() >= 0); if (js.hasDeadline()) { assertTrue(js.getDeadline() > js.getSubmissionTime() + js.getDuration()); } + assertTrue(js.getTasks().size() > 0); + + for (SynthJob.SynthTask t : js.getTasks()) { + assertTrue(t.getType() != null); + assertTrue(t.getTime() > 0); + assertTrue(t.getMemory() > 0); + assertTrue(t.getVcores() > 0); + } } }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84cea001/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java index a67845b..bfc7d0c 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java @@ -139,7 +139,7 @@ public class TestAMSimulator { String queue = "default"; List<ContainerSimulator> containers = new ArrayList<>(); app.init(1000, containers, rm, null, 0, 1000000L, "user1", queue, true, - appId, 0, SLSConfiguration.getAMContainerResource(conf)); + appId, 0, SLSConfiguration.getAMContainerResource(conf), null); app.firstStep(); verifySchedulerMetrics(appId); http://git-wip-us.apache.org/repos/asf/hadoop/blob/84cea001/hadoop-tools/hadoop-sls/src/test/resources/sls-runner.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/test/resources/sls-runner.xml b/hadoop-tools/hadoop-sls/src/test/resources/sls-runner.xml index 2f076c2..344024a 100644 --- a/hadoop-tools/hadoop-sls/src/test/resources/sls-runner.xml +++ b/hadoop-tools/hadoop-sls/src/test/resources/sls-runner.xml @@ -45,6 +45,10 @@ <name>yarn.sls.am.type.mapreduce</name> <value>org.apache.hadoop.yarn.sls.appmaster.MRAMSimulator</value> </property> + <property> + <name>yarn.sls.am.type.stream</name> + <value>org.apache.hadoop.yarn.sls.appmaster.StreamAMSimulator</value> + </property> <!-- Containers configuration --> <property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/84cea001/hadoop-tools/hadoop-sls/src/test/resources/syn.json ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/test/resources/syn.json b/hadoop-tools/hadoop-sls/src/test/resources/syn.json index 8479d23..c6e2c92 100644 --- a/hadoop-tools/hadoop-sls/src/test/resources/syn.json +++ b/hadoop-tools/hadoop-sls/src/test/resources/syn.json @@ -45,7 +45,7 @@ }, { "time": 60, - "jobs": 0 + "weight": 0 } ] } http://git-wip-us.apache.org/repos/asf/hadoop/blob/84cea001/hadoop-tools/hadoop-sls/src/test/resources/syn_generic.json ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/test/resources/syn_generic.json b/hadoop-tools/hadoop-sls/src/test/resources/syn_generic.json new file mode 100644 index 0000000..bde4cd0 --- /dev/null +++ b/hadoop-tools/hadoop-sls/src/test/resources/syn_generic.json @@ -0,0 +1,54 @@ +{ + "description": "tiny jobs workload", + "num_nodes": 20, + "nodes_per_rack": 4, + "num_jobs": 10, + "rand_seed": 2, + "workloads": [ + { + "workload_name": "tiny-test", + "workload_weight": 0.5, + "description": "Sort jobs", + "queue_name": "sls_queue_1", + "job_classes": [ + { + "class_name": "class_1", + "user_name": "foobar", + "class_weight": 1.0, + "type": "mapreduce", + "deadline_factor": {"val": 10}, + "duration": {"val": 60, "std": 5}, + "reservation": {"val": 0.5}, + "tasks":[ + { + "type": "map", + "priority": 20, + "count": { "val": 5, "std": 1}, + "time": {"val": 10, "std": 2}, + "max_memory": {"val": 1024}, + "max_vcores": {"val": 1} + }, + { + "type": "reduce", + "priority": 10, + "count": { "val": 5, "std": 1}, + "time": {"val": 20, "std": 4}, + "max_memory": {"val": 2048}, + "max_vcores": {"val": 2} + } + ] + } + ], + "time_distribution": [ + { + "time": 1, + "weight": 100 + }, + { + "time": 60, + "weight": 0 + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/84cea001/hadoop-tools/hadoop-sls/src/test/resources/syn_stream.json ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/test/resources/syn_stream.json b/hadoop-tools/hadoop-sls/src/test/resources/syn_stream.json new file mode 100644 index 0000000..a85065b --- /dev/null +++ b/hadoop-tools/hadoop-sls/src/test/resources/syn_stream.json @@ -0,0 +1,46 @@ +{ + "description": "stream workload", + "num_nodes": 20, + "nodes_per_rack": 4, + "num_jobs": 5, + "rand_seed": 2, + "workloads": [ + { + "workload_name": "tiny-test", + "workload_weight": 1, + "description": "long lived streaming jobs", + "queue_name": "sls_queue_1", + "job_classes": [ + { + "class_name": "class_1", + "user_name": "foobar", + "class_weight": 1.0, + "type": "stream", + "deadline_factor": {"val": 10}, + "duration": {"val": 30, "std": 5}, + "reservation": {"val": 0.5}, + "tasks":[ + { + "type": "stream", + "priority": 20, + "count": { "val": 2}, + "time": {"val": 60000}, + "max_memory": {"val": 4096}, + "max_vcores": {"val": 4} + } + ] + } + ], + "time_distribution": [ + { + "time": 1, + "weight": 100 + }, + { + "time": 2, + "weight": 0 + } + ] + } + ] +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
