Author: omalley
Date: Tue Mar 8 06:02:29 2011
New Revision: 1079269
URL: http://svn.apache.org/viewvc?rev=1079269&view=rev
Log:
commit 37accb250d94a67f6cba1eadbd15237df7400251
Author: Vinay Kumar Thota <[email protected]>
Date: Sun Feb 20 20:45:20 2011 -0800
4344850 from
+++ b/YAHOO-CHANGES.txt
+ Bug 4344850 [MR-2192] : Implement gridmix system tests with
+ different time intervals for MR streaming job traces. available
+ at
+
Added:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/2m_stream_trace.json.gz
(with props)
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/3m_stream_trace.json.gz
(with props)
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/5m_stream_trace.json.gz
(with props)
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridmixSystemTestCase.java
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobSubmission.java
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridmixSystemTestCase.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridmixSystemTestCase.java?rev=1079269&r1=1079268&r2=1079269&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridmixSystemTestCase.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/GridmixSystemTestCase.java
Tue Mar 8 06:02:29 2011
@@ -26,6 +26,7 @@ import org.apache.hadoop.mapreduce.test.
import org.apache.hadoop.mapreduce.test.system.JTClient;
import org.apache.hadoop.mapred.gridmix.test.system.GridmixJobSubmission;
import org.apache.hadoop.mapred.gridmix.test.system.GridmixJobVerification;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.mapreduce.JobID;
@@ -89,17 +90,29 @@ public class GridmixSystemTestCase {
}
public static void runGridmixAndVerify(String[] runtimeValues,
- String [] otherValues, String tracePath) throws Exception {
- gridmixJS = new GridmixJobSubmission(rtClient.getDaemonConf(),
- jtClient, gridmixDir);
- gridmixJS.submitJobs(runtimeValues, otherValues);
- jobids = UtilsForGridmix.listGridmixJobIDs(jtClient.getClient(),
- gridmixJS.getGridmixJobCount());
+ String [] otherValues, String tracePath) throws Exception {
+ runGridmixAndVerify(runtimeValues, otherValues, tracePath ,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX);
+ }
+
+ public static void runGridmixAndVerify(String [] runtimeValues,
+ String [] otherValues, String tracePath, int mode) throws Exception {
+ jobids = runGridmix(runtimeValues, otherValues, mode);
gridmixJV = new GridmixJobVerification(
new Path(tracePath), gridmixJS.getJobConf(), jtClient);
gridmixJV.verifyGridmixJobsWithJobStories(jobids);
}
+ public static List<JobID> runGridmix(String[] runtimeValues,
+ String[] otherValues, int mode) throws Exception {
+ gridmixJS = new GridmixJobSubmission(rtClient.getDaemonConf(),
+ jtClient, gridmixDir);
+ gridmixJS.submitJobs(runtimeValues, otherValues, mode);
+ jobids = UtilsForGridmix.listGridmixJobIDs(jtClient.getClient(),
+ gridmixJS.getGridmixJobCount());
+ return jobids;
+ }
+
public static String getTraceFile(String regExp) throws IOException {
List<String> listTraces = UtilsForGridmix.listMRTraces(
Added:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java?rev=1079269&view=auto
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java
(added)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith2minStreamingJobTrace.java
Tue Mar 8 06:02:29 2011
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Run the Gridmix with 2 minutes job trace which has been generated with
+ * streaming jobs histories and verify each job history against
+ * the corresponding job story in a given trace file.
+ */
+public class TestGridmixWith2minStreamingJobTrace extends
+ GridmixSystemTestCase {
+ private static final Log LOG = LogFactory.
+ getLog("TestGridmixWith2minStreamingJobTrace.class");
+
+
+ /**
+ * Generate input data and run Gridmix by load job with STRESS submission
+ * policy in a SubmitterUserResolver mode against 2 minutes job
+ * trace file of streaming jobs. Verify each Gridmix job history with
+ * a corresponding job story in a trace file after completion of all
+ * the jobs execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixWith2minStreamJobTrace() throws Exception {
+ final long inputSize = cSize * 250;
+ final long minFileSize = 150 * 1024 * 1024;
+ String tracePath = getTraceFile("2m_stream");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ String [] runtimeValues = {"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSize + "m",
+ tracePath
+ };
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=true",
+ "-D", GridMixConfig.GRIDMIX_MINIMUM_FILE_SIZE + "=" + minFileSize,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+ "=false"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+ }
+}
Added:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java?rev=1079269&view=auto
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java
(added)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith3minStreamingJobTrace.java
Tue Mar 8 06:02:29 2011
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Run the Gridmix with 3 minutes job trace which has been generated with
+ * streaming jobs histories and verify each job history against
+ * corresponding job story in a given trace file.
+ */
+public class TestGridmixWith3minStreamingJobTrace extends
+ GridmixSystemTestCase{
+ private static final Log LOG = LogFactory.
+ getLog("TestGridmixWith3minStreamingJobTrace.class");
+
+ /**
+ * Generate input data and run gridmix by load job with REPLAY submission
+ * policy in a RoundRobinUserResolver mode against 3 minutes job trace file
+ * of streaming job. Verify each gridmix job history with a corresponding
+ * job story in a trace file after completion of all the jobs execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixWith3minStreamJobTrace() throws Exception {
+ final long inputSize = cSize * 200;
+ final long bytesPerFile = 150 * 1024 * 1024;
+ String tracePath = getTraceFile("3m_stream");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ String [] runtimeValues ={"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ inputSize + "m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath
+ };
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE + "=true",
+ "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+ "=false"
+ };
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+ }
+}
Added:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java?rev=1079269&view=auto
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java
(added)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixWith5minStreamingJobTrace.java
Tue Mar 8 06:02:29 2011
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Run the Gridmix with 5 minutes job trace which has been generated with
+ * streaming jobs histories and verify each job history against
+ * corresponding job story in a given trace file.
+ */
+public class TestGridmixWith5minStreamingJobTrace extends
+ GridmixSystemTestCase {
+ private static final Log LOG = LogFactory.
+ getLog("TestGridmixWith5minStreamingJobTrace.class");
+
+ /**
+ * Generate input data and run gridmix by load job with SERIAL submission
+ * policy in a SubmitterUserResolver mode against 5 minutes job trace file
+ * of streaming job. Verify each gridmix job history with a corresponding
+ * job story in a trace file after completion of all the jobs execution.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixWith5minStreamJobTrace() throws Exception {
+ String tracePath = getTraceFile("5m_stream");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ final long inputSize = cSize * 200;
+ final long bytesPerFile = 150 * 1024 * 1024;
+ String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "SERIAL",
+ inputSize + "m",
+ tracePath
+ };
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_KEY_FRC + "= 0.5f",
+ "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile,
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+ "=false"
+ };
+
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath);
+ }
+}
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobSubmission.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobSubmission.java?rev=1079269&r1=1079268&r2=1079269&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobSubmission.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridmixJobSubmission.java
Tue Mar 8 06:02:29 2011
@@ -37,16 +37,16 @@ public class GridmixJobSubmission {
* @throws Exception
*/
public void submitJobs(String [] runtimeArgs,
- String [] otherArgs) throws Exception {
+ String [] otherArgs, int mode) throws Exception {
int prvJobCount = jtClient.getClient().getAllJobs().length;
int exitCode = -1;
if (otherArgs == null) {
exitCode = UtilsForGridmix.runGridmixJob(gridmixDir,
- conf, GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX,
+ conf, mode,
runtimeArgs);
} else {
exitCode = UtilsForGridmix.runGridmixJob(gridmixDir,
- conf, GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX,
+ conf, mode,
runtimeArgs, otherArgs);
}
Assert.assertEquals("Gridmix jobs have failed.", 0 , exitCode);
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java?rev=1079269&r1=1079268&r2=1079269&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
Tue Mar 8 06:02:29 2011
@@ -27,8 +27,8 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.mapred.gridmix.Gridmix;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapreduce.JobID;
import java.util.Date;
import java.util.HashMap;
@@ -260,11 +260,13 @@ public class UtilsForGridmix {
public static List<JobID> listGridmixJobIDs(JobClient client,
int execJobCount) throws IOException {
List<JobID> jobids = new ArrayList<JobID>();
+ JobStatus [] jobStatus = client.getAllJobs();
+ int numJobs = jobStatus.length;
for (int index = 1; index <= execJobCount; index++) {
- JobID jobid = client.getAllJobs()[client.getAllJobs().
- length - index].getJobID();
- RunningJob runJob = client.getJob(jobid.toString());
- if (!runJob.getJobName().equals("GRIDMIX_GENERATE_INPUT_DATA")) {
+ JobStatus js = jobStatus[numJobs - index];
+ JobID jobid = js.getJobID();
+ String jobName = js.getJobName();
+ if (!jobName.equals("GRIDMIX_GENERATE_INPUT_DATA")) {
jobids.add(jobid);
}
}
Added:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/2m_stream_trace.json.gz
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/2m_stream_trace.json.gz?rev=1079269&view=auto
==============================================================================
Binary file - no diff available.
Propchange:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/2m_stream_trace.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/3m_stream_trace.json.gz
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/3m_stream_trace.json.gz?rev=1079269&view=auto
==============================================================================
Binary file - no diff available.
Propchange:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/3m_stream_trace.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/5m_stream_trace.json.gz
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/5m_stream_trace.json.gz?rev=1079269&view=auto
==============================================================================
Binary file - no diff available.
Propchange:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/resources/5m_stream_trace.json.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
