Author: cutting
Date: Thu Mar 23 14:52:44 2006
New Revision: 388288
URL: http://svn.apache.org/viewcvs?rev=388288&view=rev
Log:
Fix for HADOOP-46. Jobs can now be named. Contributed by Owen O'Malley.
Modified:
lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Grep.java
lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobProfile.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp
Modified: lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Grep.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Grep.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Grep.java
(original)
+++ lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/Grep.java Thu
Mar 23 14:52:44 2006
@@ -17,18 +17,15 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.lib.RegexMapper;
import org.apache.hadoop.mapred.lib.InverseMapper;
import org.apache.hadoop.mapred.lib.LongSumReducer;
-import org.apache.hadoop.mapred.lib.IdentityReducer;
import org.apache.hadoop.io.UTF8;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.conf.Configuration;
@@ -52,6 +49,7 @@
Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
JobConf grepJob = new JobConf(defaults, Grep.class);
+ grepJob.setJobName("grep-search");
grepJob.setInputDir(new File(args[0]));
@@ -71,6 +69,7 @@
JobClient.runJob(grepJob);
JobConf sortJob = new JobConf(defaults, Grep.class);
+ sortJob.setJobName("grep-sort");
sortJob.setInputDir(tempDir);
sortJob.setInputFormat(SequenceFileInputFormat.class);
Modified:
lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java
(original)
+++ lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java
Thu Mar 23 14:52:44 2006
@@ -17,7 +17,6 @@
package org.apache.hadoop.examples;
import java.io.*;
-import java.net.URL;
import java.util.*;
import org.apache.hadoop.conf.Configuration;
@@ -110,24 +109,25 @@
public static void main(String[] args) throws IOException {
Configuration defaults = new Configuration();
- JobConf countJob = new JobConf(defaults, WordCount.class);
+ JobConf conf = new JobConf(defaults, WordCount.class);
+ conf.setJobName("wordcount");
// the keys are words (strings)
- countJob.setOutputKeyClass(UTF8.class);
+ conf.setOutputKeyClass(UTF8.class);
// the values are counts (ints)
- countJob.setOutputValueClass(IntWritable.class);
+ conf.setOutputValueClass(IntWritable.class);
- countJob.setMapperClass(MapClass.class);
- countJob.setCombinerClass(Reduce.class);
- countJob.setReducerClass(Reduce.class);
+ conf.setMapperClass(MapClass.class);
+ conf.setCombinerClass(Reduce.class);
+ conf.setReducerClass(Reduce.class);
List other_args = new ArrayList();
for(int i=0; i < args.length; ++i) {
try {
if ("-m".equals(args[i])) {
- countJob.setNumMapTasks(Integer.parseInt(args[++i]));
+ conf.setNumMapTasks(Integer.parseInt(args[++i]));
} else if ("-r".equals(args[i])) {
- countJob.setNumReduceTasks(Integer.parseInt(args[++i]));
+ conf.setNumReduceTasks(Integer.parseInt(args[++i]));
} else {
other_args.add(args[i]);
}
@@ -146,13 +146,13 @@
other_args.size() + " instead of 2.");
printUsage();
}
- countJob.setInputDir(new File((String) other_args.get(0)));
- countJob.setOutputDir(new File((String) other_args.get(1)));
+ conf.setInputDir(new File((String) other_args.get(0)));
+ conf.setOutputDir(new File((String) other_args.get(1)));
// Uncomment to run locally in a single process
// countJob.set("mapred.job.tracker", "local");
- JobClient.runJob(countJob);
+ JobClient.runJob(conf);
}
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Thu Mar
23 14:52:44 2006
@@ -302,6 +302,23 @@
public int getNumReduceTasks() { return getInt("mapred.reduce.tasks", 1); }
public void setNumReduceTasks(int n) { setInt("mapred.reduce.tasks", n); }
+ /**
+ * Get the user-specified job name. This is only used to identify the
+ * job to the user.
+ * @return the job's name, defaulting to ""
+ */
+ public String getJobName() {
+ return get("mapred.job.name", "");
+ }
+
+ /**
+ * Set the user-specified job name.
+ * @param name the job's new name
+ */
+ public void setJobName(String name) {
+ set("mapred.job.name", name);
+ }
+
public Object newInstance(Class theClass) {
Object result;
try {
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
Thu Mar 23 14:52:44 2006
@@ -74,8 +74,8 @@
fs.copyToLocalFile(new File(jobFile), localJobFile);
conf = new JobConf(localJobFile);
- this.profile = new JobProfile(conf.getUser(), jobid, jobFile, url);
-
+ this.profile = new JobProfile(conf.getUser(), jobid, jobFile, url,
+ conf.getJobName());
String jarFile = conf.getJar();
if (jarFile != null) {
fs.copyToLocalFile(new File(jarFile), localJarFile);
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobProfile.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobProfile.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobProfile.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobProfile.java Thu
Mar 23 14:52:44 2006
@@ -40,6 +40,7 @@
String jobid;
String jobFile;
String url;
+ String name;
/**
*/
@@ -48,11 +49,13 @@
/**
*/
- public JobProfile(String user, String jobid, String jobFile, String url) {
+ public JobProfile(String user, String jobid, String jobFile, String url,
+ String name) {
this.user = user;
this.jobid = jobid;
this.jobFile = jobFile;
this.url = url;
+ this.name = name;
}
/**
@@ -85,6 +88,13 @@
}
}
+ /**
+ * Get the user-specified job name.
+ */
+ public String getJobName() {
+ return name;
+ }
+
///////////////////////////////////////
// Writable
///////////////////////////////////////
@@ -93,12 +103,14 @@
UTF8.writeString(out, jobFile);
UTF8.writeString(out, url);
UTF8.writeString(out, user);
+ UTF8.writeString(out, name);
}
public void readFields(DataInput in) throws IOException {
this.jobid = UTF8.readString(in);
this.jobFile = UTF8.readString(in);
this.url = UTF8.readString(in);
this.user = UTF8.readString(in);
+ this.name = UTF8.readString(in);
}
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
Thu Mar 23 14:52:44 2006
@@ -56,7 +56,7 @@
fs.copyToLocalFile(new File(file), localFile);
this.job = new JobConf(localFile);
profile = new JobProfile(job.getUser(), id, file,
- "http://localhost:8080/";);
+ "http://localhost:8080/";, job.getJobName());
this.status.jobid = id;
this.status.runState = JobStatus.RUNNING;
Modified: lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp?rev=388288&r1=388287&r2=388288&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp Thu Mar 23 14:52:44
2006
@@ -48,6 +48,7 @@
out.print("<tr><td align=\"center\" colspan=\"8\"><b>" + label + " Jobs
</b></td></tr>\n");
if (jobs.size() > 0) {
out.print("<tr><td><b>Jobid</b></td><td><b>User</b></td>");
+ out.print("<td><b>Name</b></td>");
out.print("<td><b>% complete</b></td><td><b>Required maps</b></td>");
out.print("<td><b>maps completed</b></td>");
out.print("<td><b>Required reduces</b></td>");
@@ -64,9 +65,11 @@
int desiredReduces = job.desiredReduces();
int completedMaps = job.finishedMaps();
int completedReduces = job.finishedReduces();
+ String name = profile.getJobName();
out.print("<tr><td><a href=\"jobdetails.jsp?jobid=" + jobid + "\">"
+
jobid + "</a></td><td>"+ profile.getUser() + "</td><td>" +
+ ("".equals(name) ? " " : name) + "</td><td>" +
percentFormat.format(completedRatio) + "%</td><td>" +
desiredMaps + "</td><td>" + completedMaps + "</td><td>" +
desiredReduces + "</td><td> " + completedReduces +
