Author: tgraves Date: Thu Jan 17 19:54:00 2013 New Revision: 1434878 URL: http://svn.apache.org/viewvc?rev=1434878&view=rev Log: MAPREDUCE-4278. cannot run two local jobs in parallel from the same gateway. (Sandy Ryza via tgraves)
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1434878&r1=1434877&r2=1434878&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Thu Jan 17 19:54:00 2013 @@ -15,6 +15,9 @@ Release 0.23.7 - UNRELEASED BUG FIXES + MAPREDUCE-4278. cannot run two local jobs in parallel from the same + gateway. (Sandy Ryza via tgraves) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=1434878&r1=1434877&r2=1434878&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java Thu Jan 17 19:54:00 2013 @@ -609,8 +609,12 @@ public class LocalJobRunner implements C // JobSubmissionProtocol methods private static int jobid = 0; + // used for making sure that local jobs run in different jvms don't + // collide on staging or job directories + private int randid; + public synchronized org.apache.hadoop.mapreduce.JobID getNewJobID() { - return new org.apache.hadoop.mapreduce.JobID("local", ++jobid); + return new org.apache.hadoop.mapreduce.JobID("local" + randid, ++jobid); } public org.apache.hadoop.mapreduce.JobStatus submitJob( @@ -739,10 +743,11 @@ public class LocalJobRunner implements C "/tmp/hadoop/mapred/staging")); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); String user; + randid = rand.nextInt(Integer.MAX_VALUE); if (ugi != null) { - user = ugi.getShortUserName() + rand.nextInt(); + user = ugi.getShortUserName() + randid; } else { - user = "dummy" + rand.nextInt(); + user = "dummy" + randid; } return fs.makeQualified(new Path(stagingRootDir, user+"/.staging")).toString(); } Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java?rev=1434878&r1=1434877&r2=1434878&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java Thu Jan 17 19:54:00 2013 @@ -32,7 +32,7 @@ import org.apache.hadoop.io.Text; * the job. JobID consists of two parts. First part * represents the jobtracker identifier, so that jobID to jobtracker map * is defined. For cluster setup this string is the jobtracker - * start time, for local setting, it is "local". + * start time, for local setting, it is "local" and a random number. * Second part of the JobID is the job number. <br> * An example JobID is : * <code>job_200707121733_0003</code> , which represents the third job