Added: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java?rev=1463807&view=auto ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java (added) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java Wed Apr 3 01:46:39 2013 @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.gridmix; + +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.tools.rumen.JobStory; +import org.apache.hadoop.tools.rumen.JobStoryProducer; +import org.apache.hadoop.util.ExitUtil; +import org.apache.log4j.Level; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.InputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.zip.GZIPInputStream; + +import static org.junit.Assert.*; + +public class TestGridmixSubmission extends CommonJobTest { + private static File inSpace = new File("src" + File.separator + "test" + + File.separator + "resources" + File.separator + "data"); + + + static { + ((Log4JLogger) LogFactory.getLog("org.apache.hadoop.mapred.gridmix")) + .getLogger().setLevel(Level.DEBUG); + } + +// private static final int NJOBS = 1; +// private static final long GENDATA = 3; // in megabytes + + @BeforeClass + public static void init() throws IOException { + GridmixTestUtils.initCluster(TestGridmixSubmission.class); + + System.setProperty("src.test.data", inSpace.getAbsolutePath()); + } + + @AfterClass + public static void shutDown() throws IOException { + GridmixTestUtils.shutdownCluster(); + } + + /** + * Verifies that the given {@code JobStory} corresponds to the checked-in + * WordCount {@code JobStory}. The verification is effected via JUnit + * assertions. + * + * @param js the candidate JobStory. + */ + private void verifyWordCountJobStory(JobStory js) { + assertNotNull("Null JobStory", js); + String expectedJobStory = "WordCount:johndoe:default:1285322645148:3:1"; + String actualJobStory = js.getName() + ":" + js.getUser() + ":" + + js.getQueueName() + ":" + js.getSubmissionTime() + ":" + + js.getNumberMaps() + ":" + js.getNumberReduces(); + assertEquals("Unexpected JobStory", expectedJobStory, actualJobStory); + } + + /** + * Expands a file compressed using {@code gzip}. + * + * @param fs the {@code FileSystem} corresponding to the given file. + * @param in the path to the compressed file. + * @param out the path to the uncompressed output. + * @throws Exception if there was an error during the operation. + */ + private void expandGzippedTrace(FileSystem fs, Path in, Path out) + throws Exception { + byte[] buff = new byte[4096]; + GZIPInputStream gis = new GZIPInputStream(fs.open(in)); + FSDataOutputStream fsdos = fs.create(out); + int numRead; + while ((numRead = gis.read(buff, 0, buff.length)) != -1) { + fsdos.write(buff, 0, numRead); + } + gis.close(); + fsdos.close(); + } + + /** + * Tests the reading of traces in GridMix3. These traces are generated by + * Rumen and are in the JSON format. The traces can optionally be compressed + * and uncompressed traces can also be passed to GridMix3 via its standard + * input stream. The testing is effected via JUnit assertions. + * + * @throws Exception if there was an error. + */ + @Test (timeout=50000) + public void testTraceReader() throws Exception { + Configuration conf = new Configuration(); + FileSystem lfs = FileSystem.getLocal(conf); + Path rootInputDir = new Path(System.getProperty("src.test.data")); + rootInputDir = rootInputDir.makeQualified(lfs.getUri(), + lfs.getWorkingDirectory()); + Path rootTempDir = new Path(System.getProperty("test.build.data", + System.getProperty("java.io.tmpdir")), "testTraceReader"); + rootTempDir = rootTempDir.makeQualified(lfs.getUri(), + lfs.getWorkingDirectory()); + Path inputFile = new Path(rootInputDir, "wordcount.json.gz"); + Path tempFile = new Path(rootTempDir, "gridmix3-wc.json"); + + InputStream origStdIn = System.in; + InputStream tmpIs = null; + try { + DebugGridmix dgm = new DebugGridmix(); + JobStoryProducer jsp = dgm.createJobStoryProducer(inputFile.toString(), + conf); + + LOG.info("Verifying JobStory from compressed trace..."); + verifyWordCountJobStory(jsp.getNextJob()); + + expandGzippedTrace(lfs, inputFile, tempFile); + jsp = dgm.createJobStoryProducer(tempFile.toString(), conf); + LOG.info("Verifying JobStory from uncompressed trace..."); + verifyWordCountJobStory(jsp.getNextJob()); + + tmpIs = lfs.open(tempFile); + System.setIn(tmpIs); + LOG.info("Verifying JobStory from trace in standard input..."); + jsp = dgm.createJobStoryProducer("-", conf); + verifyWordCountJobStory(jsp.getNextJob()); + } finally { + System.setIn(origStdIn); + if (tmpIs != null) { + tmpIs.close(); + } + lfs.delete(rootTempDir, true); + } + } + + @Test (timeout=500000) + public void testReplaySubmit() throws Exception { + policy = GridmixJobSubmissionPolicy.REPLAY; + LOG.info(" Replay started at " + System.currentTimeMillis()); + doSubmission(null, false); + LOG.info(" Replay ended at " + System.currentTimeMillis()); + + } + + @Test (timeout=500000) + public void testStressSubmit() throws Exception { + policy = GridmixJobSubmissionPolicy.STRESS; + LOG.info(" Stress started at " + System.currentTimeMillis()); + doSubmission(null, false); + LOG.info(" Stress ended at " + System.currentTimeMillis()); + } + + // test empty request should be hint message + @Test (timeout=5000) + public void testMain() throws Exception { + + SecurityManager securityManager = System.getSecurityManager(); + + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + final PrintStream out = new PrintStream(bytes); + final PrintStream oldOut = System.out; + System.setErr(out); + ExitUtil.disableSystemExit(); + try { + String[] argv = new String[0]; + DebugGridmix.main(argv); + + } catch (ExitUtil.ExitException e) { + assertEquals("ExitException", e.getMessage()); + ExitUtil.resetFirstExitException(); + } finally { + System.setErr(oldOut); + System.setSecurityManager(securityManager); + } + String print = bytes.toString(); + // should be printed tip in std error stream + assertTrue(print + .contains("Usage: gridmix [-generate <MiB>] [-users URI] [-Dname=value ...] <iopath> <trace>")); + assertTrue(print.contains("e.g. gridmix -generate 100m foo -")); + } + + +}
Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java?rev=1463807&r1=1463806&r2=1463807&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java Wed Apr 3 01:46:39 2013 @@ -45,7 +45,7 @@ public class TestGridmixSummary { /** * Test {@link DataStatistics}. */ - @Test + @Test (timeout=10000) public void testDataStatistics() throws Exception { // test data-statistics getters with compression enabled DataStatistics stats = new DataStatistics(10, 2, true); @@ -133,7 +133,8 @@ public class TestGridmixSummary { /** * A fake {@link JobFactory}. */ - @SuppressWarnings("unchecked") + + @SuppressWarnings("rawtypes") private static class FakeJobFactory extends JobFactory { /** * A fake {@link JobStoryProducer} for {@link FakeJobFactory}. @@ -166,7 +167,7 @@ public class TestGridmixSummary { /** * Test {@link ExecutionSummarizer}. */ - @Test + @Test (timeout=10000) @SuppressWarnings("unchecked") public void testExecutionSummarizer() throws IOException { Configuration conf = new Configuration(); @@ -344,8 +345,7 @@ public class TestGridmixSummary { /** * Test {@link ClusterSummarizer}. */ - @Test - @SuppressWarnings("deprecation") + @Test (timeout=10000) public void testClusterSummarizer() throws IOException { ClusterSummarizer cs = new ClusterSummarizer(); Configuration conf = new Configuration(); Added: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestLoadJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestLoadJob.java?rev=1463807&view=auto ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestLoadJob.java (added) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestLoadJob.java Wed Apr 3 01:46:39 2013 @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.gridmix; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.log4j.Level; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +/* + Test LoadJob Gridmix sends data to job and after that + */ +public class TestLoadJob extends CommonJobTest { + + public static final Log LOG = LogFactory.getLog(Gridmix.class); + + static { + ((Log4JLogger) LogFactory.getLog("org.apache.hadoop.mapred.gridmix")) + .getLogger().setLevel(Level.DEBUG); + ((Log4JLogger) LogFactory.getLog(StressJobFactory.class)).getLogger() + .setLevel(Level.DEBUG); + } + + + @BeforeClass + public static void init() throws IOException { + GridmixTestUtils.initCluster(TestLoadJob.class); + } + + @AfterClass + public static void shutDown() throws IOException { + GridmixTestUtils.shutdownCluster(); + } + + + /* + * test serial policy with LoadJob. Task should execute without exceptions + */ + @Test (timeout=500000) + public void testSerialSubmit() throws Exception { + policy = GridmixJobSubmissionPolicy.SERIAL; + LOG.info("Serial started at " + System.currentTimeMillis()); + doSubmission(JobCreator.LOADJOB.name(), false); + + LOG.info("Serial ended at " + System.currentTimeMillis()); + } + + /* + * test reply policy with LoadJob + */ + @Test (timeout=500000) + public void testReplaySubmit() throws Exception { + policy = GridmixJobSubmissionPolicy.REPLAY; + LOG.info(" Replay started at " + System.currentTimeMillis()); + doSubmission(JobCreator.LOADJOB.name(), false); + + LOG.info(" Replay ended at " + System.currentTimeMillis()); + } + + +} Added: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestSleepJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestSleepJob.java?rev=1463807&view=auto ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestSleepJob.java (added) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestSleepJob.java Wed Apr 3 01:46:39 2013 @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.gridmix; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.tools.rumen.JobStory; +import org.apache.log4j.Level; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.*; + +public class TestSleepJob extends CommonJobTest { + + public static final Log LOG = LogFactory.getLog(Gridmix.class); + + static { + ((Log4JLogger) LogFactory.getLog("org.apache.hadoop.mapred.gridmix")) + .getLogger().setLevel(Level.DEBUG); + } + + static GridmixJobSubmissionPolicy policy = GridmixJobSubmissionPolicy.REPLAY; + + @BeforeClass + public static void init() throws IOException { + GridmixTestUtils.initCluster(TestSleepJob.class); + } + + @AfterClass + public static void shutDown() throws IOException { + GridmixTestUtils.shutdownCluster(); + } + + + /* + * test RandomLocation + */ + @Test (timeout=100000) + public void testRandomLocation() throws Exception { + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + + testRandomLocation(1, 10, ugi); + testRandomLocation(2, 10, ugi); + } + + @Test (timeout=500000) + public void testMapTasksOnlySleepJobs() throws Exception { + Configuration configuration = GridmixTestUtils.mrvl.getConfig(); + + DebugJobProducer jobProducer = new DebugJobProducer(5, configuration); + configuration.setBoolean(SleepJob.SLEEPJOB_MAPTASK_ONLY, true); + + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + JobStory story; + int seq = 1; + while ((story = jobProducer.getNextJob()) != null) { + GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(configuration, 0, + story, new Path("ignored"), ugi, seq++); + gridmixJob.buildSplits(null); + Job job = gridmixJob.call(); + assertEquals(0, job.getNumReduceTasks()); + } + jobProducer.close(); + assertEquals(6, seq); + } + + // test Serial submit + @Test (timeout=500000) + public void testSerialSubmit() throws Exception { + // set policy + policy = GridmixJobSubmissionPolicy.SERIAL; + LOG.info("Serial started at " + System.currentTimeMillis()); + doSubmission(JobCreator.SLEEPJOB.name(), false); + LOG.info("Serial ended at " + System.currentTimeMillis()); + } + + @Test (timeout=500000) + public void testReplaySubmit() throws Exception { + policy = GridmixJobSubmissionPolicy.REPLAY; + LOG.info(" Replay started at " + System.currentTimeMillis()); + doSubmission(JobCreator.SLEEPJOB.name(), false); + LOG.info(" Replay ended at " + System.currentTimeMillis()); + } + + @Test (timeout=500000) + public void testStressSubmit() throws Exception { + policy = GridmixJobSubmissionPolicy.STRESS; + LOG.info(" Replay started at " + System.currentTimeMillis()); + doSubmission(JobCreator.SLEEPJOB.name(), false); + LOG.info(" Replay ended at " + System.currentTimeMillis()); + } + + private void testRandomLocation(int locations, int njobs, + UserGroupInformation ugi) throws Exception { + Configuration configuration = new Configuration(); + + DebugJobProducer jobProducer = new DebugJobProducer(njobs, configuration); + Configuration jconf = GridmixTestUtils.mrvl.getConfig(); + jconf.setInt(JobCreator.SLEEPJOB_RANDOM_LOCATIONS, locations); + + JobStory story; + int seq = 1; + while ((story = jobProducer.getNextJob()) != null) { + GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(jconf, 0, + story, new Path("ignored"), ugi, seq++); + gridmixJob.buildSplits(null); + List<InputSplit> splits = new SleepJob.SleepInputFormat() + .getSplits(gridmixJob.getJob()); + for (InputSplit split : splits) { + assertEquals(locations, split.getLocations().length); + } + } + jobProducer.close(); + } + +} Added: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount.json URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount.json?rev=1463807&view=auto ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount.json (added) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount.json Wed Apr 3 01:46:39 2013 @@ -0,0 +1,414 @@ +{ + "priority" : "NORMAL", + "jobID" : "job_201009241532_0001", + "user" : "johndoe", + "jobName" : "WordCount", + "mapTasks" : [ { + "startTime" : 1285322651360, + "taskID" : "task_201009241532_0001_m_000000", + "taskType" : "MAP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322651366, + "finishTime" : 1285322658262, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000000_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : 704270, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : 48266, + "mapInputRecords" : 13427, + "mapOutputBytes" : 1182333, + "mapOutputRecords" : 126063, + "combineInputRecords" : 126063, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 6612, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322660778, + "preferredLocations" : [ { + "layers" : [ "default-rack", "foo.example.com" ] + } ], + "taskStatus" : "SUCCESS", + "inputBytes" : 704270, + "inputRecords" : 13427, + "outputBytes" : 48266, + "outputRecords" : 126063 + }, { + "startTime" : 1285322651361, + "taskID" : "task_201009241532_0001_m_000001", + "taskType" : "MAP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322651378, + "finishTime" : 1285322657906, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000001_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : 577214, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : 58143, + "mapInputRecords" : 13015, + "mapOutputBytes" : 985534, + "mapOutputRecords" : 108400, + "combineInputRecords" : 108400, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 8214, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322660781, + "preferredLocations" : [ { + "layers" : [ "default-rack", "foo.example.com" ] + } ], + "taskStatus" : "SUCCESS", + "inputBytes" : 577214, + "inputRecords" : 13015, + "outputBytes" : 58143, + "outputRecords" : 108400 + }, { + "startTime" : 1285322660789, + "taskID" : "task_201009241532_0001_m_000002", + "taskType" : "MAP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322660807, + "finishTime" : 1285322664865, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000002_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : 163907, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : 21510, + "mapInputRecords" : 3736, + "mapOutputBytes" : 275796, + "mapOutputRecords" : 30528, + "combineInputRecords" : 30528, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 3040, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322666805, + "preferredLocations" : [ { + "layers" : [ "default-rack", "foo.example.com" ] + } ], + "taskStatus" : "SUCCESS", + "inputBytes" : 163907, + "inputRecords" : 3736, + "outputBytes" : 21510, + "outputRecords" : 30528 + } ], + "finishTime" : 1285322675837, + "reduceTasks" : [ { + "startTime" : 1285322660790, + "taskID" : "task_201009241532_0001_r_000000", + "taskType" : "REDUCE", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322660807, + "finishTime" : 1285322670759, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_r_000000_0", + "shuffleFinished" : 1285322667962, + "sortFinished" : 1285322668146, + "hdfsBytesRead" : -1, + "hdfsBytesWritten" : 122793, + "fileBytesRead" : 111026, + "fileBytesWritten" : 111026, + "mapInputRecords" : -1, + "mapOutputBytes" : -1, + "mapOutputRecords" : -1, + "combineInputRecords" : 0, + "reduceInputGroups" : 11713, + "reduceInputRecords" : 17866, + "reduceShuffleBytes" : 127823, + "reduceOutputRecords" : 11713, + "spilledRecords" : 17866, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322672821, + "preferredLocations" : [ ], + "taskStatus" : "SUCCESS", + "inputBytes" : 127823, + "inputRecords" : 17866, + "outputBytes" : 122793, + "outputRecords" : 11713 + } ], + "submitTime" : 1285322645148, + "launchTime" : 1285322645614, + "totalMaps" : 3, + "totalReduces" : 1, + "otherTasks" : [ { + "startTime" : 1285322648294, + "taskID" : "task_201009241532_0001_m_000004", + "taskType" : "SETUP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322648482, + "finishTime" : 1285322649588, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000004_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : -1, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : -1, + "mapInputRecords" : -1, + "mapOutputBytes" : -1, + "mapOutputRecords" : -1, + "combineInputRecords" : -1, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 0, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322651351, + "preferredLocations" : [ ], + "taskStatus" : "SUCCESS", + "inputBytes" : -1, + "inputRecords" : -1, + "outputBytes" : -1, + "outputRecords" : -1 + }, { + "startTime" : 1285322672829, + "taskID" : "task_201009241532_0001_m_000003", + "taskType" : "CLEANUP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322672838, + "finishTime" : 1285322673971, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000003_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : -1, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : -1, + "mapInputRecords" : -1, + "mapOutputBytes" : -1, + "mapOutputRecords" : -1, + "combineInputRecords" : -1, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 0, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322675835, + "preferredLocations" : [ ], + "taskStatus" : "SUCCESS", + "inputBytes" : -1, + "inputRecords" : -1, + "outputBytes" : -1, + "outputRecords" : -1 + } ], + "computonsPerMapInputByte" : -1, + "computonsPerMapOutputByte" : -1, + "computonsPerReduceInputByte" : -1, + "computonsPerReduceOutputByte" : -1, + "heapMegabytes" : 1024, + "outcome" : "SUCCESS", + "jobtype" : "JAVA", + "directDependantJobs" : [ ], + "successfulMapAttemptCDFs" : [ { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 6896, + "minimum" : 4058, + "rankings" : [ { + "datum" : 4058, + "relativeRanking" : 0.05 + }, { + "datum" : 4058, + "relativeRanking" : 0.1 + }, { + "datum" : 4058, + "relativeRanking" : 0.15 + }, { + "datum" : 4058, + "relativeRanking" : 0.2 + }, { + "datum" : 4058, + "relativeRanking" : 0.25 + }, { + "datum" : 4058, + "relativeRanking" : 0.3 + }, { + "datum" : 4058, + "relativeRanking" : 0.35 + }, { + "datum" : 4058, + "relativeRanking" : 0.4 + }, { + "datum" : 4058, + "relativeRanking" : 0.45 + }, { + "datum" : 4058, + "relativeRanking" : 0.5 + }, { + "datum" : 4058, + "relativeRanking" : 0.55 + }, { + "datum" : 4058, + "relativeRanking" : 0.6 + }, { + "datum" : 4058, + "relativeRanking" : 0.65 + }, { + "datum" : 6528, + "relativeRanking" : 0.7 + }, { + "datum" : 6528, + "relativeRanking" : 0.75 + }, { + "datum" : 6528, + "relativeRanking" : 0.8 + }, { + "datum" : 6528, + "relativeRanking" : 0.85 + }, { + "datum" : 6528, + "relativeRanking" : 0.9 + }, { + "datum" : 6528, + "relativeRanking" : 0.95 + } ], + "numberValues" : 3 + } ], + "failedMapAttemptCDFs" : [ { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + } ], + "successfulReduceAttemptCDF" : { + "maximum" : 9952, + "minimum" : 9952, + "rankings" : [ { + "datum" : 9952, + "relativeRanking" : 0.05 + }, { + "datum" : 9952, + "relativeRanking" : 0.1 + }, { + "datum" : 9952, + "relativeRanking" : 0.15 + }, { + "datum" : 9952, + "relativeRanking" : 0.2 + }, { + "datum" : 9952, + "relativeRanking" : 0.25 + }, { + "datum" : 9952, + "relativeRanking" : 0.3 + }, { + "datum" : 9952, + "relativeRanking" : 0.35 + }, { + "datum" : 9952, + "relativeRanking" : 0.4 + }, { + "datum" : 9952, + "relativeRanking" : 0.45 + }, { + "datum" : 9952, + "relativeRanking" : 0.5 + }, { + "datum" : 9952, + "relativeRanking" : 0.55 + }, { + "datum" : 9952, + "relativeRanking" : 0.6 + }, { + "datum" : 9952, + "relativeRanking" : 0.65 + }, { + "datum" : 9952, + "relativeRanking" : 0.7 + }, { + "datum" : 9952, + "relativeRanking" : 0.75 + }, { + "datum" : 9952, + "relativeRanking" : 0.8 + }, { + "datum" : 9952, + "relativeRanking" : 0.85 + }, { + "datum" : 9952, + "relativeRanking" : 0.9 + }, { + "datum" : 9952, + "relativeRanking" : 0.95 + } ], + "numberValues" : 1 + }, + "failedReduceAttemptCDF" : { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, + "mapperTriesToSucceed" : [ 1.0 ], + "failedMapperFraction" : 0.0, + "relativeTime" : 0, + "queue" : "default", + "clusterMapMB" : -1, + "clusterReduceMB" : -1, + "jobMapMB" : 1024, + "jobReduceMB" : 1024 +} Added: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount2.json URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount2.json?rev=1463807&view=auto ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount2.json (added) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount2.json Wed Apr 3 01:46:39 2013 @@ -0,0 +1,828 @@ +{ + "priority" : "NORMAL", + "jobID" : "job_201009241532_0001", + "user" : "johndoe", + "jobName" : "WordCount", + "mapTasks" : [ { + "startTime" : 1285322651360, + "taskID" : "task_201009241532_0001_m_000000", + "taskType" : "MAP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322651366, + "finishTime" : 1285322658262, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000000_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : 704270, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : 48266, + "mapInputRecords" : 13427, + "mapOutputBytes" : 1182333, + "mapOutputRecords" : 126063, + "combineInputRecords" : 126063, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 6612, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322660778, + "preferredLocations" : [ { + "layers" : [ "default-rack", "foo.example.com" ] + } ], + "taskStatus" : "SUCCESS", + "inputBytes" : 704270, + "inputRecords" : 13427, + "outputBytes" : 48266, + "outputRecords" : 126063 + }, { + "startTime" : 1285322651361, + "taskID" : "task_201009241532_0001_m_000001", + "taskType" : "MAP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322651378, + "finishTime" : 1285322657906, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000001_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : 577214, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : 58143, + "mapInputRecords" : 13015, + "mapOutputBytes" : 985534, + "mapOutputRecords" : 108400, + "combineInputRecords" : 108400, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 8214, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322660781, + "preferredLocations" : [ { + "layers" : [ "default-rack", "foo.example.com" ] + } ], + "taskStatus" : "SUCCESS", + "inputBytes" : 577214, + "inputRecords" : 13015, + "outputBytes" : 58143, + "outputRecords" : 108400 + }, { + "startTime" : 1285322660789, + "taskID" : "task_201009241532_0001_m_000002", + "taskType" : "MAP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322660807, + "finishTime" : 1285322664865, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000002_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : 163907, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : 21510, + "mapInputRecords" : 3736, + "mapOutputBytes" : 275796, + "mapOutputRecords" : 30528, + "combineInputRecords" : 30528, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 3040, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322666805, + "preferredLocations" : [ { + "layers" : [ "default-rack", "foo.example.com" ] + } ], + "taskStatus" : "SUCCESS", + "inputBytes" : 163907, + "inputRecords" : 3736, + "outputBytes" : 21510, + "outputRecords" : 30528 + } ], + "finishTime" : 1285322675837, + "reduceTasks" : [ { + "startTime" : 1285322660790, + "taskID" : "task_201009241532_0001_r_000000", + "taskType" : "REDUCE", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322660807, + "finishTime" : 1285322670759, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_r_000000_0", + "shuffleFinished" : 1285322667962, + "sortFinished" : 1285322668146, + "hdfsBytesRead" : -1, + "hdfsBytesWritten" : 122793, + "fileBytesRead" : 111026, + "fileBytesWritten" : 111026, + "mapInputRecords" : -1, + "mapOutputBytes" : -1, + "mapOutputRecords" : -1, + "combineInputRecords" : 0, + "reduceInputGroups" : 11713, + "reduceInputRecords" : 17866, + "reduceShuffleBytes" : 127823, + "reduceOutputRecords" : 11713, + "spilledRecords" : 17866, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322672821, + "preferredLocations" : [ ], + "taskStatus" : "SUCCESS", + "inputBytes" : 127823, + "inputRecords" : 17866, + "outputBytes" : 122793, + "outputRecords" : 11713 + } ], + "submitTime" : 1285322645148, + "launchTime" : 1285322645614, + "totalMaps" : 3, + "totalReduces" : 1, + "otherTasks" : [ { + "startTime" : 1285322648294, + "taskID" : "task_201009241532_0001_m_000004", + "taskType" : "SETUP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322648482, + "finishTime" : 1285322649588, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000004_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : -1, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : -1, + "mapInputRecords" : -1, + "mapOutputBytes" : -1, + "mapOutputRecords" : -1, + "combineInputRecords" : -1, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 0, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322651351, + "preferredLocations" : [ ], + "taskStatus" : "SUCCESS", + "inputBytes" : -1, + "inputRecords" : -1, + "outputBytes" : -1, + "outputRecords" : -1 + }, { + "startTime" : 1285322672829, + "taskID" : "task_201009241532_0001_m_000003", + "taskType" : "CLEANUP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322672838, + "finishTime" : 1285322673971, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000003_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : -1, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : -1, + "mapInputRecords" : -1, + "mapOutputBytes" : -1, + "mapOutputRecords" : -1, + "combineInputRecords" : -1, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 0, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322675835, + "preferredLocations" : [ ], + "taskStatus" : "SUCCESS", + "inputBytes" : -1, + "inputRecords" : -1, + "outputBytes" : -1, + "outputRecords" : -1 + } ], + "computonsPerMapInputByte" : -1, + "computonsPerMapOutputByte" : -1, + "computonsPerReduceInputByte" : -1, + "computonsPerReduceOutputByte" : -1, + "heapMegabytes" : 1024, + "outcome" : "SUCCESS", + "jobtype" : "JAVA", + "directDependantJobs" : [ ], + "successfulMapAttemptCDFs" : [ { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 6896, + "minimum" : 4058, + "rankings" : [ { + "datum" : 4058, + "relativeRanking" : 0.05 + }, { + "datum" : 4058, + "relativeRanking" : 0.1 + }, { + "datum" : 4058, + "relativeRanking" : 0.15 + }, { + "datum" : 4058, + "relativeRanking" : 0.2 + }, { + "datum" : 4058, + "relativeRanking" : 0.25 + }, { + "datum" : 4058, + "relativeRanking" : 0.3 + }, { + "datum" : 4058, + "relativeRanking" : 0.35 + }, { + "datum" : 4058, + "relativeRanking" : 0.4 + }, { + "datum" : 4058, + "relativeRanking" : 0.45 + }, { + "datum" : 4058, + "relativeRanking" : 0.5 + }, { + "datum" : 4058, + "relativeRanking" : 0.55 + }, { + "datum" : 4058, + "relativeRanking" : 0.6 + }, { + "datum" : 4058, + "relativeRanking" : 0.65 + }, { + "datum" : 6528, + "relativeRanking" : 0.7 + }, { + "datum" : 6528, + "relativeRanking" : 0.75 + }, { + "datum" : 6528, + "relativeRanking" : 0.8 + }, { + "datum" : 6528, + "relativeRanking" : 0.85 + }, { + "datum" : 6528, + "relativeRanking" : 0.9 + }, { + "datum" : 6528, + "relativeRanking" : 0.95 + } ], + "numberValues" : 3 + } ], + "failedMapAttemptCDFs" : [ { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + } ], + "successfulReduceAttemptCDF" : { + "maximum" : 9952, + "minimum" : 9952, + "rankings" : [ { + "datum" : 9952, + "relativeRanking" : 0.05 + }, { + "datum" : 9952, + "relativeRanking" : 0.1 + }, { + "datum" : 9952, + "relativeRanking" : 0.15 + }, { + "datum" : 9952, + "relativeRanking" : 0.2 + }, { + "datum" : 9952, + "relativeRanking" : 0.25 + }, { + "datum" : 9952, + "relativeRanking" : 0.3 + }, { + "datum" : 9952, + "relativeRanking" : 0.35 + }, { + "datum" : 9952, + "relativeRanking" : 0.4 + }, { + "datum" : 9952, + "relativeRanking" : 0.45 + }, { + "datum" : 9952, + "relativeRanking" : 0.5 + }, { + "datum" : 9952, + "relativeRanking" : 0.55 + }, { + "datum" : 9952, + "relativeRanking" : 0.6 + }, { + "datum" : 9952, + "relativeRanking" : 0.65 + }, { + "datum" : 9952, + "relativeRanking" : 0.7 + }, { + "datum" : 9952, + "relativeRanking" : 0.75 + }, { + "datum" : 9952, + "relativeRanking" : 0.8 + }, { + "datum" : 9952, + "relativeRanking" : 0.85 + }, { + "datum" : 9952, + "relativeRanking" : 0.9 + }, { + "datum" : 9952, + "relativeRanking" : 0.95 + } ], + "numberValues" : 1 + }, + "failedReduceAttemptCDF" : { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, + "mapperTriesToSucceed" : [ 1.0 ], + "failedMapperFraction" : 0.0, + "relativeTime" : 0, + "queue" : "default", + "clusterMapMB" : -1, + "clusterReduceMB" : -1, + "jobMapMB" : 1024, + "jobReduceMB" : 1024 +} +{ + "priority" : "NORMAL", + "jobID" : "job_201009241532_0001", + "user" : "johndoe", + "jobName" : "WordCount", + "mapTasks" : [ { + "startTime" : 1285322651360, + "taskID" : "task_201009241532_0001_m_000000", + "taskType" : "MAP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322651366, + "finishTime" : 1285322658262, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000000_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : 704270, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : 48266, + "mapInputRecords" : 13427, + "mapOutputBytes" : 1182333, + "mapOutputRecords" : 126063, + "combineInputRecords" : 126063, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 6612, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322660778, + "preferredLocations" : [ { + "layers" : [ "default-rack", "foo.example.com" ] + } ], + "taskStatus" : "SUCCESS", + "inputBytes" : 704270, + "inputRecords" : 13427, + "outputBytes" : 48266, + "outputRecords" : 126063 + }, { + "startTime" : 1285322651361, + "taskID" : "task_201009241532_0001_m_000001", + "taskType" : "MAP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322651378, + "finishTime" : 1285322657906, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000001_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : 577214, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : 58143, + "mapInputRecords" : 13015, + "mapOutputBytes" : 985534, + "mapOutputRecords" : 108400, + "combineInputRecords" : 108400, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 8214, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322660781, + "preferredLocations" : [ { + "layers" : [ "default-rack", "foo.example.com" ] + } ], + "taskStatus" : "SUCCESS", + "inputBytes" : 577214, + "inputRecords" : 13015, + "outputBytes" : 58143, + "outputRecords" : 108400 + }, { + "startTime" : 1285322660789, + "taskID" : "task_201009241532_0001_m_000002", + "taskType" : "MAP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322660807, + "finishTime" : 1285322664865, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000002_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : 163907, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : 21510, + "mapInputRecords" : 3736, + "mapOutputBytes" : 275796, + "mapOutputRecords" : 30528, + "combineInputRecords" : 30528, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 3040, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322666805, + "preferredLocations" : [ { + "layers" : [ "default-rack", "foo.example.com" ] + } ], + "taskStatus" : "SUCCESS", + "inputBytes" : 163907, + "inputRecords" : 3736, + "outputBytes" : 21510, + "outputRecords" : 30528 + } ], + "finishTime" : 1285322675837, + "reduceTasks" : [ { + "startTime" : 1285322660790, + "taskID" : "task_201009241532_0001_r_000000", + "taskType" : "REDUCE", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322660807, + "finishTime" : 1285322670759, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_r_000000_0", + "shuffleFinished" : 1285322667962, + "sortFinished" : 1285322668146, + "hdfsBytesRead" : -1, + "hdfsBytesWritten" : 122793, + "fileBytesRead" : 111026, + "fileBytesWritten" : 111026, + "mapInputRecords" : -1, + "mapOutputBytes" : -1, + "mapOutputRecords" : -1, + "combineInputRecords" : 0, + "reduceInputGroups" : 11713, + "reduceInputRecords" : 17866, + "reduceShuffleBytes" : 127823, + "reduceOutputRecords" : 11713, + "spilledRecords" : 17866, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322672821, + "preferredLocations" : [ ], + "taskStatus" : "SUCCESS", + "inputBytes" : 127823, + "inputRecords" : 17866, + "outputBytes" : 122793, + "outputRecords" : 11713 + } ], + "submitTime" : 1285322645148, + "launchTime" : 1285322645614, + "totalMaps" : 3, + "totalReduces" : 1, + "otherTasks" : [ { + "startTime" : 1285322648294, + "taskID" : "task_201009241532_0001_m_000004", + "taskType" : "SETUP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322648482, + "finishTime" : 1285322649588, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000004_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : -1, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : -1, + "mapInputRecords" : -1, + "mapOutputBytes" : -1, + "mapOutputRecords" : -1, + "combineInputRecords" : -1, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 0, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322651351, + "preferredLocations" : [ ], + "taskStatus" : "SUCCESS", + "inputBytes" : -1, + "inputRecords" : -1, + "outputBytes" : -1, + "outputRecords" : -1 + }, { + "startTime" : 1285322672829, + "taskID" : "task_201009241532_0001_m_000003", + "taskType" : "CLEANUP", + "attempts" : [ { + "location" : null, + "hostName" : "/default-rack/foo.example.com", + "startTime" : 1285322672838, + "finishTime" : 1285322673971, + "result" : "SUCCESS", + "attemptID" : "attempt_201009241532_0001_m_000003_0", + "shuffleFinished" : -1, + "sortFinished" : -1, + "hdfsBytesRead" : -1, + "hdfsBytesWritten" : -1, + "fileBytesRead" : -1, + "fileBytesWritten" : -1, + "mapInputRecords" : -1, + "mapOutputBytes" : -1, + "mapOutputRecords" : -1, + "combineInputRecords" : -1, + "reduceInputGroups" : -1, + "reduceInputRecords" : -1, + "reduceShuffleBytes" : -1, + "reduceOutputRecords" : -1, + "spilledRecords" : 0, + "mapInputBytes" : -1 + } ], + "finishTime" : 1285322675835, + "preferredLocations" : [ ], + "taskStatus" : "SUCCESS", + "inputBytes" : -1, + "inputRecords" : -1, + "outputBytes" : -1, + "outputRecords" : -1 + } ], + "computonsPerMapInputByte" : -1, + "computonsPerMapOutputByte" : -1, + "computonsPerReduceInputByte" : -1, + "computonsPerReduceOutputByte" : -1, + "heapMegabytes" : 1024, + "outcome" : "SUCCESS", + "jobtype" : "JAVA", + "directDependantJobs" : [ ], + "successfulMapAttemptCDFs" : [ { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 6896, + "minimum" : 4058, + "rankings" : [ { + "datum" : 4058, + "relativeRanking" : 0.05 + }, { + "datum" : 4058, + "relativeRanking" : 0.1 + }, { + "datum" : 4058, + "relativeRanking" : 0.15 + }, { + "datum" : 4058, + "relativeRanking" : 0.2 + }, { + "datum" : 4058, + "relativeRanking" : 0.25 + }, { + "datum" : 4058, + "relativeRanking" : 0.3 + }, { + "datum" : 4058, + "relativeRanking" : 0.35 + }, { + "datum" : 4058, + "relativeRanking" : 0.4 + }, { + "datum" : 4058, + "relativeRanking" : 0.45 + }, { + "datum" : 4058, + "relativeRanking" : 0.5 + }, { + "datum" : 4058, + "relativeRanking" : 0.55 + }, { + "datum" : 4058, + "relativeRanking" : 0.6 + }, { + "datum" : 4058, + "relativeRanking" : 0.65 + }, { + "datum" : 6528, + "relativeRanking" : 0.7 + }, { + "datum" : 6528, + "relativeRanking" : 0.75 + }, { + "datum" : 6528, + "relativeRanking" : 0.8 + }, { + "datum" : 6528, + "relativeRanking" : 0.85 + }, { + "datum" : 6528, + "relativeRanking" : 0.9 + }, { + "datum" : 6528, + "relativeRanking" : 0.95 + } ], + "numberValues" : 3 + } ], + "failedMapAttemptCDFs" : [ { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + } ], + "successfulReduceAttemptCDF" : { + "maximum" : 9952, + "minimum" : 9952, + "rankings" : [ { + "datum" : 9952, + "relativeRanking" : 0.05 + }, { + "datum" : 9952, + "relativeRanking" : 0.1 + }, { + "datum" : 9952, + "relativeRanking" : 0.15 + }, { + "datum" : 9952, + "relativeRanking" : 0.2 + }, { + "datum" : 9952, + "relativeRanking" : 0.25 + }, { + "datum" : 9952, + "relativeRanking" : 0.3 + }, { + "datum" : 9952, + "relativeRanking" : 0.35 + }, { + "datum" : 9952, + "relativeRanking" : 0.4 + }, { + "datum" : 9952, + "relativeRanking" : 0.45 + }, { + "datum" : 9952, + "relativeRanking" : 0.5 + }, { + "datum" : 9952, + "relativeRanking" : 0.55 + }, { + "datum" : 9952, + "relativeRanking" : 0.6 + }, { + "datum" : 9952, + "relativeRanking" : 0.65 + }, { + "datum" : 9952, + "relativeRanking" : 0.7 + }, { + "datum" : 9952, + "relativeRanking" : 0.75 + }, { + "datum" : 9952, + "relativeRanking" : 0.8 + }, { + "datum" : 9952, + "relativeRanking" : 0.85 + }, { + "datum" : 9952, + "relativeRanking" : 0.9 + }, { + "datum" : 9952, + "relativeRanking" : 0.95 + } ], + "numberValues" : 1 + }, + "failedReduceAttemptCDF" : { + "maximum" : 9223372036854775807, + "minimum" : -9223372036854775808, + "rankings" : [ ], + "numberValues" : 0 + }, + "mapperTriesToSucceed" : [ 1.0 ], + "failedMapperFraction" : 0.0, + "relativeTime" : 0, + "queue" : "default", + "clusterMapMB" : -1, + "clusterReduceMB" : -1, + "jobMapMB" : 1024, + "jobReduceMB" : 1024 +}
