Repository: hadoop Updated Branches: refs/heads/branch-2 b4c4f30fa -> a33109a0b
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a33109a0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java new file mode 100644 index 0000000..358e9b2 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java @@ -0,0 +1,945 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapreduce.jobhistory; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapred.TaskAttemptID; +import org.apache.hadoop.mapred.TaskID; +import org.apache.hadoop.mapred.TaskStatus; +import org.apache.hadoop.mapreduce.Counters; +import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.JobStatus; +import org.apache.hadoop.mapreduce.TaskType; +import org.junit.Assert; +import org.junit.Test; +import org.skyscreamer.jsonassert.JSONAssert; +import org.skyscreamer.jsonassert.JSONCompareMode; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.HashMap; +import java.util.TimeZone; + +public class TestHistoryViewerPrinter { + + private static final Log LOG = LogFactory.getLog( + TestHistoryViewerPrinter.class); + + @Test + public void testHumanPrinter() throws Exception { + JobHistoryParser.JobInfo job = createJobInfo(); + HumanReadableHistoryViewerPrinter printer = + new HumanReadableHistoryViewerPrinter(job, false, "http://";, + TimeZone.getTimeZone("GMT")); + String outStr = run(printer); + Assert.assertEquals("\n" + + "Hadoop job: job_1317928501754_0001\n" + + "=====================================\n" + + "User: rkanter\n" + + "JobName: my job\n" + + "JobConf: /tmp/job.xml\n" + + "Submitted At: 6-Oct-2011 19:15:01\n" + + "Launched At: 6-Oct-2011 19:15:02 (1sec)\n" + + "Finished At: 6-Oct-2011 19:15:16 (14sec)\n" + + "Status: SUCCEEDED\n" + + "Counters: \n" + + "\n" + + "|Group Name |Counter name |Map Value |Reduce Value|Total Value|\n" + + "---------------------------------------------------------------------------------------\n" + + "|group1 |counter1 |5 |5 |5 \n" + + "|group1 |counter2 |10 |10 |10 \n" + + "|group2 |counter1 |15 |15 |15 \n" + + "\n" + + "=====================================\n" + + "\n" + + "Task Summary\n" + + "============================\n" + + "Kind\tTotal\tSuccessful\tFailed\tKilled\tStartTime\tFinishTime\n" + + "\n" + + "Setup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\n" + + "Map\t6\t5\t\t1\t0\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:16 (12sec)\n" + + "Reduce\t1\t1\t\t0\t0\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\n" + + "Cleanup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\n" + + "============================\n" + + "\n" + + "\n" + + "Analysis\n" + + "=========\n" + + "\n" + + "Time taken by best performing map task task_1317928501754_0001_m_000003: 3sec\n" + + "Average time taken by map tasks: 5sec\n" + + "Worse performing map tasks: \n" + + "TaskId\t\tTimetaken\n" + + "task_1317928501754_0001_m_000007 7sec\n" + + "task_1317928501754_0001_m_000006 6sec\n" + + "task_1317928501754_0001_m_000005 5sec\n" + + "task_1317928501754_0001_m_000004 4sec\n" + + "task_1317928501754_0001_m_000003 3sec\n" + + "The last map task task_1317928501754_0001_m_000007 finished at (relative to the Job launch time): 6-Oct-2011 19:15:16 (14sec)\n" + + "\n" + + "Time taken by best performing shuffle task task_1317928501754_0001_r_000008: 8sec\n" + + "Average time taken by shuffle tasks: 8sec\n" + + "Worse performing shuffle tasks: \n" + + "TaskId\t\tTimetaken\n" + + "task_1317928501754_0001_r_000008 8sec\n" + + "The last shuffle task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" + + "\n" + + "Time taken by best performing reduce task task_1317928501754_0001_r_000008: 0sec\n" + + "Average time taken by reduce tasks: 0sec\n" + + "Worse performing reduce tasks: \n" + + "TaskId\t\tTimetaken\n" + + "task_1317928501754_0001_r_000008 0sec\n" + + "The last reduce task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" + + "=========\n" + + "\n" + + "FAILED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000002\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\t\t\n" + + "\n" + + "FAILED task attempts by nodes\n" + + "Hostname\tFailedTasks\n" + + "===============================\n" + + "localhost\ttask_1317928501754_0001_m_000002, \n", outStr); + } + + @Test + public void testHumanPrinterAll() throws Exception { + JobHistoryParser.JobInfo job = createJobInfo(); + HumanReadableHistoryViewerPrinter printer = + new HumanReadableHistoryViewerPrinter(job, true, "http://";, + TimeZone.getTimeZone("GMT")); + String outStr = run(printer); + if (System.getProperty("java.version").startsWith("1.7")) { + Assert.assertEquals("\n" + + "Hadoop job: job_1317928501754_0001\n" + + "=====================================\n" + + "User: rkanter\n" + + "JobName: my job\n" + + "JobConf: /tmp/job.xml\n" + + "Submitted At: 6-Oct-2011 19:15:01\n" + + "Launched At: 6-Oct-2011 19:15:02 (1sec)\n" + + "Finished At: 6-Oct-2011 19:15:16 (14sec)\n" + + "Status: SUCCEEDED\n" + + "Counters: \n" + + "\n" + + "|Group Name |Counter name |Map Value |Reduce Value|Total Value|\n" + + "---------------------------------------------------------------------------------------\n" + + "|group1 |counter1 |5 |5 |5 \n" + + "|group1 |counter2 |10 |10 |10 \n" + + "|group2 |counter1 |15 |15 |15 \n" + + "\n" + + "=====================================\n" + + "\n" + + "Task Summary\n" + + "============================\n" + + "Kind\tTotal\tSuccessful\tFailed\tKilled\tStartTime\tFinishTime\n" + + "\n" + + "Setup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\n" + + "Map\t6\t5\t\t1\t0\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:16 (12sec)\n" + + "Reduce\t1\t1\t\t0\t0\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\n" + + "Cleanup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\n" + + "============================\n" + + "\n" + + "\n" + + "Analysis\n" + + "=========\n" + + "\n" + + "Time taken by best performing map task task_1317928501754_0001_m_000003: 3sec\n" + + "Average time taken by map tasks: 5sec\n" + + "Worse performing map tasks: \n" + + "TaskId\t\tTimetaken\n" + + "task_1317928501754_0001_m_000007 7sec\n" + + "task_1317928501754_0001_m_000006 6sec\n" + + "task_1317928501754_0001_m_000005 5sec\n" + + "task_1317928501754_0001_m_000004 4sec\n" + + "task_1317928501754_0001_m_000003 3sec\n" + + "The last map task task_1317928501754_0001_m_000007 finished at (relative to the Job launch time): 6-Oct-2011 19:15:16 (14sec)\n" + + "\n" + + "Time taken by best performing shuffle task task_1317928501754_0001_r_000008: 8sec\n" + + "Average time taken by shuffle tasks: 8sec\n" + + "Worse performing shuffle tasks: \n" + + "TaskId\t\tTimetaken\n" + + "task_1317928501754_0001_r_000008 8sec\n" + + "The last shuffle task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" + + "\n" + + "Time taken by best performing reduce task task_1317928501754_0001_r_000008: 0sec\n" + + "Average time taken by reduce tasks: 0sec\n" + + "Worse performing reduce tasks: \n" + + "TaskId\t\tTimetaken\n" + + "task_1317928501754_0001_r_000008 0sec\n" + + "The last reduce task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" + + "=========\n" + + "\n" + + "FAILED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000002\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\t\t\n" + + "\n" + + "SUCCEEDED JOB_SETUP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\n" + + "====================================================\n" + + "task_1317928501754_0001_s_000001\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\t\n" + + "\n" + + "SUCCEEDED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000006\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\t\t\n" + + "\n" + + "SUCCEEDED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000005\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\t\t\n" + + "\n" + + "SUCCEEDED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000004\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\t\t\n" + + "\n" + + "SUCCEEDED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000003\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\t\t\n" + + "\n" + + "SUCCEEDED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000007\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\t\t\n" + + "\n" + + "SUCCEEDED REDUCE task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\n" + + "====================================================\n" + + "task_1317928501754_0001_r_000008\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t\n" + + "\n" + + "SUCCEEDED JOB_CLEANUP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\n" + + "====================================================\n" + + "task_1317928501754_0001_c_000009\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\t\n" + + "\n" + + "JOB_SETUP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" + + "====================================================\n" + + "attempt_1317928501754_0001_s_000001_1\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_s_000001_1\n"; + + "\n" + + "MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" + + "====================================================\n" + + "attempt_1317928501754_0001_m_000002_1\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000002_1\n"; + + "attempt_1317928501754_0001_m_000006_1\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000006_1\n"; + + "attempt_1317928501754_0001_m_000005_1\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000005_1\n"; + + "attempt_1317928501754_0001_m_000004_1\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000004_1\n"; + + "attempt_1317928501754_0001_m_000003_1\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000003_1\n"; + + "attempt_1317928501754_0001_m_000007_1\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000007_1\n"; + + "\n" + + "REDUCE task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tShuffleFinished\tSortFinished\tFinishTime\tHostName\tError\tTaskLogs\n" + + "====================================================\n" + + "attempt_1317928501754_0001_r_000008_1\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t6-Oct-2011 19:15:18 (0sec)6-Oct-2011 19:15:18 (8sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_r_000008_1\n"; + + "\n" + + "JOB_CLEANUP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" + + "====================================================\n" + + "attempt_1317928501754_0001_c_000009_1\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_c_000009_1\n"; + + "\n" + + "FAILED task attempts by nodes\n" + + "Hostname\tFailedTasks\n" + + "===============================\n" + + "localhost\ttask_1317928501754_0001_m_000002, \n", outStr); + } else { + Assert.assertEquals("\n" + + "Hadoop job: job_1317928501754_0001\n" + + "=====================================\n" + + "User: rkanter\n" + + "JobName: my job\n" + + "JobConf: /tmp/job.xml\n" + + "Submitted At: 6-Oct-2011 19:15:01\n" + + "Launched At: 6-Oct-2011 19:15:02 (1sec)\n" + + "Finished At: 6-Oct-2011 19:15:16 (14sec)\n" + + "Status: SUCCEEDED\n" + + "Counters: \n" + + "\n" + + "|Group Name |Counter name |Map Value |Reduce Value|Total Value|\n" + + "---------------------------------------------------------------------------------------\n" + + "|group1 |counter1 |5 |5 |5 \n" + + "|group1 |counter2 |10 |10 |10 \n" + + "|group2 |counter1 |15 |15 |15 \n" + + "\n" + + "=====================================\n" + + "\n" + + "Task Summary\n" + + "============================\n" + + "Kind\tTotal\tSuccessful\tFailed\tKilled\tStartTime\tFinishTime\n" + + "\n" + + "Setup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\n" + + "Map\t6\t5\t\t1\t0\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:16 (12sec)\n" + + "Reduce\t1\t1\t\t0\t0\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\n" + + "Cleanup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\n" + + "============================\n" + + "\n" + + "\n" + + "Analysis\n" + + "=========\n" + + "\n" + + "Time taken by best performing map task task_1317928501754_0001_m_000003: 3sec\n" + + "Average time taken by map tasks: 5sec\n" + + "Worse performing map tasks: \n" + + "TaskId\t\tTimetaken\n" + + "task_1317928501754_0001_m_000007 7sec\n" + + "task_1317928501754_0001_m_000006 6sec\n" + + "task_1317928501754_0001_m_000005 5sec\n" + + "task_1317928501754_0001_m_000004 4sec\n" + + "task_1317928501754_0001_m_000003 3sec\n" + + "The last map task task_1317928501754_0001_m_000007 finished at (relative to the Job launch time): 6-Oct-2011 19:15:16 (14sec)\n" + + "\n" + + "Time taken by best performing shuffle task task_1317928501754_0001_r_000008: 8sec\n" + + "Average time taken by shuffle tasks: 8sec\n" + + "Worse performing shuffle tasks: \n" + + "TaskId\t\tTimetaken\n" + + "task_1317928501754_0001_r_000008 8sec\n" + + "The last shuffle task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" + + "\n" + + "Time taken by best performing reduce task task_1317928501754_0001_r_000008: 0sec\n" + + "Average time taken by reduce tasks: 0sec\n" + + "Worse performing reduce tasks: \n" + + "TaskId\t\tTimetaken\n" + + "task_1317928501754_0001_r_000008 0sec\n" + + "The last reduce task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" + + "=========\n" + + "\n" + + "FAILED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000002\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\t\t\n" + + "\n" + + "SUCCEEDED JOB_SETUP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\n" + + "====================================================\n" + + "task_1317928501754_0001_s_000001\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\t\n" + + "\n" + + "SUCCEEDED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000007\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\t\t\n" + + "\n" + + "SUCCEEDED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000006\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\t\t\n" + + "\n" + + "SUCCEEDED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000005\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\t\t\n" + + "\n" + + "SUCCEEDED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000004\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\t\t\n" + + "\n" + + "SUCCEEDED MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" + + "====================================================\n" + + "task_1317928501754_0001_m_000003\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\t\t\n" + + "\n" + + "SUCCEEDED REDUCE task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\n" + + "====================================================\n" + + "task_1317928501754_0001_r_000008\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t\n" + + "\n" + + "SUCCEEDED JOB_CLEANUP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tError\n" + + "====================================================\n" + + "task_1317928501754_0001_c_000009\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\t\n" + + "\n" + + "JOB_SETUP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" + + "====================================================\n" + + "attempt_1317928501754_0001_s_000001_1\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_s_000001_1\n"; + + "\n" + + "MAP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" + + "====================================================\n" + + "attempt_1317928501754_0001_m_000007_1\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000007_1\n"; + + "attempt_1317928501754_0001_m_000002_1\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000002_1\n"; + + "attempt_1317928501754_0001_m_000006_1\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000006_1\n"; + + "attempt_1317928501754_0001_m_000005_1\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000005_1\n"; + + "attempt_1317928501754_0001_m_000004_1\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000004_1\n"; + + "attempt_1317928501754_0001_m_000003_1\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000003_1\n"; + + "\n" + + "REDUCE task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tShuffleFinished\tSortFinished\tFinishTime\tHostName\tError\tTaskLogs\n" + + "====================================================\n" + + "attempt_1317928501754_0001_r_000008_1\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t6-Oct-2011 19:15:18 (0sec)6-Oct-2011 19:15:18 (8sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_r_000008_1\n"; + + "\n" + + "JOB_CLEANUP task list for job_1317928501754_0001\n" + + "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" + + "====================================================\n" + + "attempt_1317928501754_0001_c_000009_1\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_c_000009_1\n"; + + "\n" + + "FAILED task attempts by nodes\n" + + "Hostname\tFailedTasks\n" + + "===============================\n" + + "localhost\ttask_1317928501754_0001_m_000002, \n", outStr); + } + } + + @Test + public void testJSONPrinter() throws Exception { + JobHistoryParser.JobInfo job = createJobInfo(); + JSONHistoryViewerPrinter printer = + new JSONHistoryViewerPrinter(job, false, "http://";); + String outStr = run(printer); + JSONAssert.assertEquals("{\n" + + " \"counters\": {\n" + + " \"group1\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"mapValue\": 5,\n" + + " \"reduceValue\": 5,\n" + + " \"totalValue\": 5\n" + + " },\n" + + " {\n" + + " \"counterName\": \"counter2\",\n" + + " \"mapValue\": 10,\n" + + " \"reduceValue\": 10,\n" + + " \"totalValue\": 10\n" + + " }\n" + + " ],\n" + + " \"group2\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"mapValue\": 15,\n" + + " \"reduceValue\": 15,\n" + + " \"totalValue\": 15\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"finishedAt\": 1317928516754,\n" + + " \"hadoopJob\": \"job_1317928501754_0001\",\n" + + " \"jobConf\": \"/tmp/job.xml\",\n" + + " \"jobName\": \"my job\",\n" + + " \"launchedAt\": 1317928502754,\n" + + " \"status\": \"SUCCEEDED\",\n" + + " \"submittedAt\": 1317928501754,\n" + + " \"taskSummary\": {\n" + + " \"cleanup\": {\n" + + " \"failed\": 0,\n" + + " \"finishTime\": 1317928520754,\n" + + " \"killed\": 0,\n" + + " \"startTime\": 1317928511754,\n" + + " \"successful\": 1,\n" + + " \"total\": 1\n" + + " },\n" + + " \"map\": {\n" + + " \"failed\": 1,\n" + + " \"finishTime\": 1317928516754,\n" + + " \"killed\": 0,\n" + + " \"startTime\": 1317928504754,\n" + + " \"successful\": 5,\n" + + " \"total\": 6\n" + + " },\n" + + " \"reduce\": {\n" + + " \"failed\": 0,\n" + + " \"finishTime\": 1317928518754,\n" + + " \"killed\": 0,\n" + + " \"startTime\": 1317928510754,\n" + + " \"successful\": 1,\n" + + " \"total\": 1\n" + + " },\n" + + " \"setup\": {\n" + + " \"failed\": 0,\n" + + " \"finishTime\": 1317928504754,\n" + + " \"killed\": 0,\n" + + " \"startTime\": 1317928503754,\n" + + " \"successful\": 1,\n" + + " \"total\": 1\n" + + " }\n" + + " },\n" + + " \"tasks\": [\n" + + " {\n" + + " \"finishTime\": 1317928506754,\n" + + " \"inputSplits\": \"\",\n" + + " \"startTime\": 1317928504754,\n" + + " \"status\": \"FAILED\",\n" + + " \"taskId\": \"task_1317928501754_0001_m_000002\",\n" + + " \"type\": \"MAP\"\n" + + " }\n" + + " ],\n" + + " \"user\": \"rkanter\"\n" + + "}\n", outStr, JSONCompareMode.NON_EXTENSIBLE); + } + + @Test + public void testJSONPrinterAll() throws Exception { + JobHistoryParser.JobInfo job = createJobInfo(); + JSONHistoryViewerPrinter printer = + new JSONHistoryViewerPrinter(job, true, "http://";); + String outStr = run(printer); + JSONAssert.assertEquals("{\n" + + " \"counters\": {\n" + + " \"group1\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"mapValue\": 5,\n" + + " \"reduceValue\": 5,\n" + + " \"totalValue\": 5\n" + + " },\n" + + " {\n" + + " \"counterName\": \"counter2\",\n" + + " \"mapValue\": 10,\n" + + " \"reduceValue\": 10,\n" + + " \"totalValue\": 10\n" + + " }\n" + + " ],\n" + + " \"group2\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"mapValue\": 15,\n" + + " \"reduceValue\": 15,\n" + + " \"totalValue\": 15\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"finishedAt\": 1317928516754,\n" + + " \"hadoopJob\": \"job_1317928501754_0001\",\n" + + " \"jobConf\": \"/tmp/job.xml\",\n" + + " \"jobName\": \"my job\",\n" + + " \"launchedAt\": 1317928502754,\n" + + " \"status\": \"SUCCEEDED\",\n" + + " \"submittedAt\": 1317928501754,\n" + + " \"taskSummary\": {\n" + + " \"cleanup\": {\n" + + " \"failed\": 0,\n" + + " \"finishTime\": 1317928520754,\n" + + " \"killed\": 0,\n" + + " \"startTime\": 1317928511754,\n" + + " \"successful\": 1,\n" + + " \"total\": 1\n" + + " },\n" + + " \"map\": {\n" + + " \"failed\": 1,\n" + + " \"finishTime\": 1317928516754,\n" + + " \"killed\": 0,\n" + + " \"startTime\": 1317928504754,\n" + + " \"successful\": 5,\n" + + " \"total\": 6\n" + + " },\n" + + " \"reduce\": {\n" + + " \"failed\": 0,\n" + + " \"finishTime\": 1317928518754,\n" + + " \"killed\": 0,\n" + + " \"startTime\": 1317928510754,\n" + + " \"successful\": 1,\n" + + " \"total\": 1\n" + + " },\n" + + " \"setup\": {\n" + + " \"failed\": 0,\n" + + " \"finishTime\": 1317928504754,\n" + + " \"killed\": 0,\n" + + " \"startTime\": 1317928503754,\n" + + " \"successful\": 1,\n" + + " \"total\": 1\n" + + " }\n" + + " },\n" + + " \"tasks\": [\n" + + " {\n" + + " \"attempts\": {\n" + + " \"attemptId\": \"attempt_1317928501754_0001_m_000002_1\",\n" + + " \"finishTime\": 1317928506754,\n" + + " \"hostName\": \"localhost\",\n" + + " \"startTime\": 1317928504754,\n" + + " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000002_1\"\n"; + + " },\n" + + " \"counters\": {\n" + + " \"group1\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 5\n" + + " },\n" + + " {\n" + + " \"counterName\": \"counter2\",\n" + + " \"value\": 10\n" + + " }\n" + + " ],\n" + + " \"group2\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 15\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"finishTime\": 1317928506754,\n" + + " \"inputSplits\": \"\",\n" + + " \"startTime\": 1317928504754,\n" + + " \"status\": \"FAILED\",\n" + + " \"taskId\": \"task_1317928501754_0001_m_000002\",\n" + + " \"type\": \"MAP\"\n" + + " },\n" + + " {\n" + + " \"attempts\": {\n" + + " \"attemptId\": \"attempt_1317928501754_0001_s_000001_1\",\n" + + " \"finishTime\": 1317928504754,\n" + + " \"hostName\": \"localhost\",\n" + + " \"startTime\": 1317928503754,\n" + + " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_s_000001_1\"\n"; + + " },\n" + + " \"counters\": {\n" + + " \"group1\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 5\n" + + " },\n" + + " {\n" + + " \"counterName\": \"counter2\",\n" + + " \"value\": 10\n" + + " }\n" + + " ],\n" + + " \"group2\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 15\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"finishTime\": 1317928504754,\n" + + " \"startTime\": 1317928503754,\n" + + " \"status\": \"SUCCEEDED\",\n" + + " \"taskId\": \"task_1317928501754_0001_s_000001\",\n" + + " \"type\": \"JOB_SETUP\"\n" + + " },\n" + + " {\n" + + " \"attempts\": {\n" + + " \"attemptId\": \"attempt_1317928501754_0001_m_000006_1\",\n" + + " \"finishTime\": 1317928514754,\n" + + " \"hostName\": \"localhost\",\n" + + " \"startTime\": 1317928508754,\n" + + " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000006_1\"\n"; + + " },\n" + + " \"counters\": {\n" + + " \"group1\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 5\n" + + " },\n" + + " {\n" + + " \"counterName\": \"counter2\",\n" + + " \"value\": 10\n" + + " }\n" + + " ],\n" + + " \"group2\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 15\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"finishTime\": 1317928514754,\n" + + " \"inputSplits\": \"\",\n" + + " \"startTime\": 1317928508754,\n" + + " \"status\": \"SUCCEEDED\",\n" + + " \"taskId\": \"task_1317928501754_0001_m_000006\",\n" + + " \"type\": \"MAP\"\n" + + " },\n" + + " {\n" + + " \"attempts\": {\n" + + " \"attemptId\": \"attempt_1317928501754_0001_m_000005_1\",\n" + + " \"finishTime\": 1317928512754,\n" + + " \"hostName\": \"localhost\",\n" + + " \"startTime\": 1317928507754,\n" + + " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000005_1\"\n"; + + " },\n" + + " \"counters\": {\n" + + " \"group1\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 5\n" + + " },\n" + + " {\n" + + " \"counterName\": \"counter2\",\n" + + " \"value\": 10\n" + + " }\n" + + " ],\n" + + " \"group2\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 15\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"finishTime\": 1317928512754,\n" + + " \"inputSplits\": \"\",\n" + + " \"startTime\": 1317928507754,\n" + + " \"status\": \"SUCCEEDED\",\n" + + " \"taskId\": \"task_1317928501754_0001_m_000005\",\n" + + " \"type\": \"MAP\"\n" + + " },\n" + + " {\n" + + " \"attempts\": {\n" + + " \"attemptId\": \"attempt_1317928501754_0001_m_000004_1\",\n" + + " \"finishTime\": 1317928510754,\n" + + " \"hostName\": \"localhost\",\n" + + " \"startTime\": 1317928506754,\n" + + " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000004_1\"\n"; + + " },\n" + + " \"counters\": {\n" + + " \"group1\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 5\n" + + " },\n" + + " {\n" + + " \"counterName\": \"counter2\",\n" + + " \"value\": 10\n" + + " }\n" + + " ],\n" + + " \"group2\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 15\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"finishTime\": 1317928510754,\n" + + " \"inputSplits\": \"\",\n" + + " \"startTime\": 1317928506754,\n" + + " \"status\": \"SUCCEEDED\",\n" + + " \"taskId\": \"task_1317928501754_0001_m_000004\",\n" + + " \"type\": \"MAP\"\n" + + " },\n" + + " {\n" + + " \"attempts\": {\n" + + " \"attemptId\": \"attempt_1317928501754_0001_m_000003_1\",\n" + + " \"finishTime\": 1317928508754,\n" + + " \"hostName\": \"localhost\",\n" + + " \"startTime\": 1317928505754,\n" + + " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000003_1\"\n"; + + " },\n" + + " \"counters\": {\n" + + " \"group1\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 5\n" + + " },\n" + + " {\n" + + " \"counterName\": \"counter2\",\n" + + " \"value\": 10\n" + + " }\n" + + " ],\n" + + " \"group2\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 15\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"finishTime\": 1317928508754,\n" + + " \"inputSplits\": \"\",\n" + + " \"startTime\": 1317928505754,\n" + + " \"status\": \"SUCCEEDED\",\n" + + " \"taskId\": \"task_1317928501754_0001_m_000003\",\n" + + " \"type\": \"MAP\"\n" + + " },\n" + + " {\n" + + " \"attempts\": {\n" + + " \"attemptId\": \"attempt_1317928501754_0001_c_000009_1\",\n" + + " \"finishTime\": 1317928520754,\n" + + " \"hostName\": \"localhost\",\n" + + " \"startTime\": 1317928511754,\n" + + " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_c_000009_1\"\n"; + + " },\n" + + " \"counters\": {\n" + + " \"group1\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 5\n" + + " },\n" + + " {\n" + + " \"counterName\": \"counter2\",\n" + + " \"value\": 10\n" + + " }\n" + + " ],\n" + + " \"group2\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 15\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"finishTime\": 1317928520754,\n" + + " \"startTime\": 1317928511754,\n" + + " \"status\": \"SUCCEEDED\",\n" + + " \"taskId\": \"task_1317928501754_0001_c_000009\",\n" + + " \"type\": \"JOB_CLEANUP\"\n" + + " },\n" + + " {\n" + + " \"attempts\": {\n" + + " \"attemptId\": \"attempt_1317928501754_0001_m_000007_1\",\n" + + " \"finishTime\": 1317928516754,\n" + + " \"hostName\": \"localhost\",\n" + + " \"startTime\": 1317928509754,\n" + + " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000007_1\"\n"; + + " },\n" + + " \"counters\": {\n" + + " \"group1\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 5\n" + + " },\n" + + " {\n" + + " \"counterName\": \"counter2\",\n" + + " \"value\": 10\n" + + " }\n" + + " ],\n" + + " \"group2\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 15\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"finishTime\": 1317928516754,\n" + + " \"inputSplits\": \"\",\n" + + " \"startTime\": 1317928509754,\n" + + " \"status\": \"SUCCEEDED\",\n" + + " \"taskId\": \"task_1317928501754_0001_m_000007\",\n" + + " \"type\": \"MAP\"\n" + + " },\n" + + " {\n" + + " \"attempts\": {\n" + + " \"attemptId\": \"attempt_1317928501754_0001_r_000008_1\",\n" + + " \"finishTime\": 1317928518754,\n" + + " \"hostName\": \"localhost\",\n" + + " \"shuffleFinished\": 1317928518754,\n" + + " \"sortFinished\": 1317928518754,\n" + + " \"startTime\": 1317928510754,\n" + + " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_r_000008_1\"\n"; + + " },\n" + + " \"counters\": {\n" + + " \"group1\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 5\n" + + " },\n" + + " {\n" + + " \"counterName\": \"counter2\",\n" + + " \"value\": 10\n" + + " }\n" + + " ],\n" + + " \"group2\": [\n" + + " {\n" + + " \"counterName\": \"counter1\",\n" + + " \"value\": 15\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"finishTime\": 1317928518754,\n" + + " \"startTime\": 1317928510754,\n" + + " \"status\": \"SUCCEEDED\",\n" + + " \"taskId\": \"task_1317928501754_0001_r_000008\",\n" + + " \"type\": \"REDUCE\"\n" + + " }\n" + + " ],\n" + + " \"user\": \"rkanter\"\n" + + "}\n", outStr, JSONCompareMode.NON_EXTENSIBLE); + } + + private String run(HistoryViewerPrinter printer) throws Exception { + ByteArrayOutputStream boas = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(boas, true); + printer.print(out); + out.close(); + String outStr = boas.toString("UTF-8"); + LOG.info("out = " + outStr); + return outStr; + } + + private static JobHistoryParser.JobInfo createJobInfo() { + JobHistoryParser.JobInfo job = new JobHistoryParser.JobInfo(); + job.submitTime = 1317928501754L; + job.finishTime = job.submitTime + 15000; + job.jobid = JobID.forName("job_1317928501754_0001"); + job.username = "rkanter"; + job.jobname = "my job"; + job.jobQueueName = "my queue"; + job.jobConfPath = "/tmp/job.xml"; + job.launchTime = job.submitTime + 1000; + job.totalMaps = 5; + job.totalReduces = 1; + job.failedMaps = 1; + job.failedReduces = 0; + job.finishedMaps = 5; + job.finishedReduces = 1; + job.jobStatus = JobStatus.State.SUCCEEDED.name(); + job.totalCounters = createCounters(); + job.mapCounters = createCounters(); + job.reduceCounters = createCounters(); + job.tasksMap = new HashMap<>(); + addTaskInfo(job, TaskType.JOB_SETUP, 1, TaskStatus.State.SUCCEEDED); + addTaskInfo(job, TaskType.MAP, 2, TaskStatus.State.FAILED); + addTaskInfo(job, TaskType.MAP, 3, TaskStatus.State.SUCCEEDED); + addTaskInfo(job, TaskType.MAP, 4, TaskStatus.State.SUCCEEDED); + addTaskInfo(job, TaskType.MAP, 5, TaskStatus.State.SUCCEEDED); + addTaskInfo(job, TaskType.MAP, 6, TaskStatus.State.SUCCEEDED); + addTaskInfo(job, TaskType.MAP, 7, TaskStatus.State.SUCCEEDED); + addTaskInfo(job, TaskType.REDUCE, 8, TaskStatus.State.SUCCEEDED); + addTaskInfo(job, TaskType.JOB_CLEANUP, 9, TaskStatus.State.SUCCEEDED); + return job; + } + + private static Counters createCounters() { + Counters counters = new Counters(); + counters.findCounter("group1", "counter1").setValue(5); + counters.findCounter("group1", "counter2").setValue(10); + counters.findCounter("group2", "counter1").setValue(15); + return counters; + } + + private static void addTaskInfo(JobHistoryParser.JobInfo job, + TaskType type, int id, TaskStatus.State status) { + JobHistoryParser.TaskInfo task = new JobHistoryParser.TaskInfo(); + task.taskId = new TaskID(job.getJobId(), type, id); + task.startTime = job.getLaunchTime() + id * 1000; + task.finishTime = task.startTime + id * 1000; + task.taskType = type; + task.counters = createCounters(); + task.status = status.name(); + task.attemptsMap = new HashMap<>(); + addTaskAttemptInfo(task, 1); + job.tasksMap.put(task.getTaskId(), task); + } + + private static void addTaskAttemptInfo( + JobHistoryParser.TaskInfo task, int id) { + JobHistoryParser.TaskAttemptInfo attempt = + new JobHistoryParser.TaskAttemptInfo(); + attempt.attemptId = new TaskAttemptID( + TaskID.downgrade(task.getTaskId()), id); + attempt.startTime = task.getStartTime(); + attempt.finishTime = task.getFinishTime(); + attempt.shuffleFinishTime = task.getFinishTime(); + attempt.sortFinishTime = task.getFinishTime(); + attempt.mapFinishTime = task.getFinishTime(); + attempt.status = task.getTaskStatus(); + attempt.taskType = task.getTaskType(); + attempt.trackerName = "localhost"; + attempt.httpPort = 1234; + attempt.hostname = "localhost"; + task.attemptsMap.put(attempt.getAttemptId(), attempt); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a33109a0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java index bd78b15..49d1d62 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java @@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; +import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; @@ -29,6 +30,8 @@ import java.io.PipedInputStream; import java.io.PipedOutputStream; import java.io.PrintStream; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import org.junit.Assert; import org.apache.commons.logging.Log; @@ -359,27 +362,152 @@ public class TestMRJobClient extends ClusterMapReduceTestCase { String historyFileUri = new Path(f.getAbsolutePath()) .makeQualified(localFs.getUri(), localFs.getWorkingDirectory()).toUri() .toString(); - - // bad command - int exitCode = runTool(conf, jc, new String[] { "-history", "pul", - historyFileUri }, out); - assertEquals("Exit code", -1, exitCode); - exitCode = runTool(conf, jc, new String[] { "-history", "all", - historyFileUri }, out); + // Try a bunch of different valid combinations of the command to test + // argument parsing + int exitCode = runTool(conf, jc, new String[] { + "-history", + "all", + historyFileUri, + }, out); assertEquals("Exit code", 0, exitCode); - String line; + checkHistoryHumanOutput(out); + File outFile = File.createTempFile("myout", ".txt"); + exitCode = runTool(conf, jc, new String[] { + "-history", + "all", + historyFileUri, + "-outfile", + outFile.getAbsolutePath() + }, out); + assertEquals("Exit code", 0, exitCode); + checkHistoryHumanFileOutput(out, outFile); + outFile = File.createTempFile("myout", ".txt"); + exitCode = runTool(conf, jc, new String[] { + "-history", + "all", + historyFileUri, + "-outfile", + outFile.getAbsolutePath(), + "-format", + "human" + }, out); + assertEquals("Exit code", 0, exitCode); + checkHistoryHumanFileOutput(out, outFile); + + exitCode = runTool(conf, jc, new String[] { + "-history", + historyFileUri, + "-format", + "human" + }, out); + assertEquals("Exit code", 0, exitCode); + checkHistoryHumanOutput(out); + + exitCode = runTool(conf, jc, new String[] { + "-history", + "all", + historyFileUri, + "-format", + "json" + }, out); + assertEquals("Exit code", 0, exitCode); + checkHistoryJSONOutput(out); + outFile = File.createTempFile("myout", ".txt"); + exitCode = runTool(conf, jc, new String[] { + "-history", + "all", + historyFileUri, + "-outfile", + outFile.getAbsolutePath(), + "-format", + "json" + }, out); + assertEquals("Exit code", 0, exitCode); + checkHistoryJSONFileOutput(out, outFile); + exitCode = runTool(conf, jc, new String[] { + "-history", + historyFileUri, + "-format", + "json" + }, out); + assertEquals("Exit code", 0, exitCode); + checkHistoryJSONOutput(out); + + // Check some bad arguments + exitCode = runTool(conf, jc, new String[] { + "-history", + historyFileUri, + "foo" + }, out); + assertEquals("Exit code", -1, exitCode); + exitCode = runTool(conf, jc, new String[] { + "-history", + historyFileUri, + "-format" + }, out); + assertEquals("Exit code", -1, exitCode); + runTool(conf, jc, new String[] { + "-history", + historyFileUri, + "-outfile", + }, out); + assertEquals("Exit code", -1, exitCode); + try { + runTool(conf, jc, new String[]{ + "-history", + historyFileUri, + "-format", + "foo" + }, out); + fail(); + } catch (IllegalArgumentException e) { + // Expected + } + } + + private void checkHistoryHumanOutput(ByteArrayOutputStream out) + throws IOException, JSONException { BufferedReader br = new BufferedReader(new InputStreamReader( new ByteArrayInputStream(out.toByteArray()))); - int counter = 0; - while ((line = br.readLine()) != null) { - LOG.info("line = " + line); - if (line.startsWith("task_")) { - counter++; - } - } - assertEquals(23, counter); + br.readLine(); + String line = br.readLine(); + br.close(); + assertEquals("Hadoop job: job_1329348432655_0001", line); + out.reset(); + } + + private void checkHistoryJSONOutput(ByteArrayOutputStream out) + throws IOException, JSONException { + BufferedReader br = new BufferedReader(new InputStreamReader( + new ByteArrayInputStream(out.toByteArray()))); + String line = org.apache.commons.io.IOUtils.toString(br); + br.close(); + JSONObject json = new JSONObject(line); + assertEquals("job_1329348432655_0001", json.getString("hadoopJob")); + out.reset(); + } + + private void checkHistoryHumanFileOutput(ByteArrayOutputStream out, + File outFile) throws IOException, JSONException { + BufferedReader br = new BufferedReader(new FileReader(outFile)); + br.readLine(); + String line = br.readLine(); + br.close(); + assertEquals("Hadoop job: job_1329348432655_0001", line); + assertEquals(0, out.size()); + } + + private void checkHistoryJSONFileOutput(ByteArrayOutputStream out, + File outFile) throws IOException, JSONException { + BufferedReader br = new BufferedReader(new FileReader(outFile)); + String line = org.apache.commons.io.IOUtils.toString(br); + br.close(); + JSONObject json = new JSONObject(line); + assertEquals("job_1329348432655_0001", json.getString("hadoopJob")); + assertEquals(0, out.size()); } + /** * print job events list */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/a33109a0/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3f0dd07..35443b7 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -981,6 +981,12 @@ </exclusions> </dependency> + <dependency> + <groupId>org.skyscreamer</groupId> + <artifactId>jsonassert</artifactId> + <version>1.3.0</version> + </dependency> + </dependencies> </dependencyManagement>
