Modified: lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/GenericOptionsParser.java URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/GenericOptionsParser.java?rev=588035&r1=588034&r2=588035&view=diff ============================================================================== --- lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/GenericOptionsParser.java (original) +++ lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/GenericOptionsParser.java Wed Oct 24 14:24:51 2007 @@ -32,50 +32,54 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -/************************************************************* - * This class can be used as a utility to parse command line - * arguments generic to the Hadoop framework. This class - * recognizes several command line arguments, which allow a user - * to specify a namenode, a job tracker etc. Generic options - * supported are - * <p>-conf <configuration file> specify an application configuration file - * <p>-D <property=value> use value for given property - * <p>-fs <local|namenode:port> specify a namenode - * <p>-jt <local|jobtracker:port> specify a job tracker - * <br> - * <p>The general command line syntax is - * <p>bin/hadoop command [genericOptions] [commandOptions] - * <br> - * Generic command line arguments <strong>might</strong> modify - * <code>Configuration </code> objects, given to constructors - * <br><br> - * The functionality is implemented using Commons CLI. - * <br> - * <p>Examples using generic options are - * <p>bin/hadoop dfs -fs darwin:8020 -ls /data - * <p><blockquote><pre> - * list /data directory in dfs with namenode darwin:8020 - * </pre></blockquote> - * <p>bin/hadoop dfs -D fs.default.name=darwin:8020 -ls /data - * <p><blockquote><pre> - * list /data directory in dfs with namenode darwin:8020 - * </pre></blockquote> - * <p>bin/hadoop dfs -conf hadoop-site.xml -ls /data - * <p><blockquote><pre> - * list /data directory in dfs with conf specified in hadoop-site.xml - * </pre></blockquote> - * <p>bin/hadoop job -D mapred.job.tracker=darwin:50020 -submit job.xml - * <p><blockquote><pre> - * submit a job to job tracker darwin:50020 - * </pre></blockquote> - * <p>bin/hadoop job -jt darwin:50020 -submit job.xml - * <p><blockquote><pre> - * submit a job to job tracker darwin:50020 - * </pre></blockquote> - * <p>bin/hadoop job -jt local -submit job.xml +/** + * <code>GenericOptionsParser</code> is a utility to parse command line + * arguments generic to the Hadoop framework. + * + * <code>GenericOptionsParser</code> recognizes several standarad command + * line arguments, enabling applications to easily specify a namenode, a + * jobtracker, additional configuration resources etc. + * + * <h4 id="GenericOptions">Generic Options</h4> + * + * <p>The supported generic options are:</p> + * <p><blockquote><pre> + * -conf <configuration file> specify a configuration file + * -D <property=value> use value for given property + * -fs <local|namenode:port> specify a namenode + * -jt <local|jobtracker:port> specify a job tracker + * </pre></blockquote></p> + * + * <p>The general command line syntax is:</p> + * <p><tt><pre> + * bin/hadoop command [genericOptions] [commandOptions] + * </pre></tt></p> + * + * <p>Generic command line arguments <strong>might</strong> modify + * <code>Configuration </code> objects, given to constructors.</p> + * + * <p>The functionality is implemented using Commons CLI.</p> + * + * <p>Examples:</p> * <p><blockquote><pre> - * submit a job to local runner - * </pre></blockquote> + * $ bin/hadoop dfs -fs darwin:8020 -ls /data + * list /data directory in dfs with namenode darwin:8020 + * + * $ bin/hadoop dfs -D fs.default.name=darwin:8020 -ls /data + * list /data directory in dfs with namenode darwin:8020 + * + * $ bin/hadoop dfs -conf hadoop-site.xml -ls /data + * list /data directory in dfs with conf specified in hadoop-site.xml + * + * $ bin/hadoop job -D mapred.job.tracker=darwin:50020 -submit job.xml + * submit a job to job tracker darwin:50020 + * + * $ bin/hadoop job -jt darwin:50020 -submit job.xml + * submit a job to job tracker darwin:50020 + * + * $ bin/hadoop job -jt local -submit job.xml + * submit a job to local runner + * </pre></blockquote></p> * * @see Tool * @see ToolRunner @@ -86,21 +90,27 @@ private CommandLine commandLine; - /** Instantinates a GenericOptionsParser to parse only - * the generic Hadoop arguments. The array of string arguments - * other than the generic arguments can be obtained by - * [EMAIL PROTECTED] #getRemainingArgs()} - * @param conf the configuration to modify - * @param args User-specified arguments + /** + * Create a <code>GenericOptionsParser<code> to parse only the generic Hadoop + * arguments. + * + * The array of string arguments other than the generic arguments can be + * obtained by [EMAIL PROTECTED] #getRemainingArgs()}. + * + * @param conf the <code>Configuration</code> to modify. + * @param args command-line arguments. */ public GenericOptionsParser(Configuration conf, String[] args) { this(conf, new Options(), args); } /** - * Instantinates a GenericOptionsParser to parse given options - * as well as generic Hadoop options. The resulting <code> - * CommandLine</code> object can be obtained by [EMAIL PROTECTED] #getCommandLine()} + * Create a <code>GenericOptionsParser</code> to parse given options as well + * as generic Hadoop options. + * + * The resulting <code>CommandLine</code> object can be obtained by + * [EMAIL PROTECTED] #getCommandLine()}. + * * @param conf the configuration to modify * @param options options built by the caller * @param args User-specified arguments @@ -110,9 +120,9 @@ } /** - * Returns an array of Strings containing only command-specific - * arguments. - * @return String array of remaining arguments not parsed + * Returns an array of Strings containing only application-specific arguments. + * + * @return array of <code>String</code>s containing the un-parsed arguments. */ public String[] getRemainingArgs() { return commandLine.getArgs(); @@ -120,12 +130,14 @@ /** * Returns the commons-cli <code>CommandLine</code> object - * to process the parsed arguments. Note : if the object is - * created with <code>GenericCommandLineParser(Configuration, String[])</code>, - * then returned object will only contain parsed generic - * options. - * @return CommandLine object representing list of arguments - * parsed against Options descriptor. + * to process the parsed arguments. + * + * Note: If the object is created with + * [EMAIL PROTECTED] #GenericOptionsParser(Configuration, String[])}, then returned + * object will only contain parsed generic options. + * + * @return <code>CommandLine</code> representing list of arguments + * parsed against Options descriptor. */ public CommandLine getCommandLine() { return commandLine; @@ -212,6 +224,11 @@ return args; } + /** + * Print the usage message for generic command-line options supported. + * + * @param out stream to print the usage message to. + */ public static void printGenericCommandUsage(PrintStream out) { out.println("Generic options supported are"); out.println("-conf <configuration file> specify an application configuration file");
Modified: lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Progressable.java URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Progressable.java?rev=588035&r1=588034&r2=588035&view=diff ============================================================================== --- lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Progressable.java (original) +++ lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Progressable.java Wed Oct 24 14:24:51 2007 @@ -18,16 +18,18 @@ package org.apache.hadoop.util; -import java.io.IOException; - - - /** - * An interface for callbacks when an method makes some progress. + * A facility for reporting progress. + * + * <p>Clients and/or applications can use the provided <code>Progressable</code> + * to explicitly report progress to the Hadoop framework. This is especially + * important for operations which take an insignificant amount of time since, + * in-lieu of the reported progress, the framework has to assume that an error + * has occured and time-out the operation.</p> */ public interface Progressable { - /** callback for reporting progress. Used by DFSclient to report - * progress while writing a block of DFS file. + /** + * Report progress to the Hadoop framework. */ public void progress(); } Modified: lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Tool.java URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Tool.java?rev=588035&r1=588034&r2=588035&view=diff ============================================================================== --- lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Tool.java (original) +++ lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Tool.java Wed Oct 24 14:24:51 2007 @@ -21,14 +21,58 @@ import org.apache.hadoop.conf.Configurable; /** - * A tool interface that support generic options handling + * A tool interface that supports handling of generic command-line options. * + * <p><code>Tool</code>, is the standard for any Map-Reduce tool/application. + * The tool/application should delegate the handling of + * <a href="[EMAIL PROTECTED]/org/apache/hadoop/util/GenericOptionsParser.html#GenericOptions"> + * standard command-line options</a> to [EMAIL PROTECTED] ToolRunner#run(Tool, String[])} + * and only handle its custom arguments.</p> + * + * <p>Here is how a typical <code>Tool</code> is implemented:</p> + * <p><blockquote><pre> + * public class MyApp extends Configured implements Tool { + * + * public int run(String[] args) throws Exception { + * // <code>Configuration</code> processed by <code>ToolRunner</code> + * Configuration conf = getConf(); + * + * // Create a JobConf using the processed <code>conf</code> + * JobConf job = new JobConf(conf, MyApp.class); + * + * // Process custom command-line options + * Path in = new Path(args[1]); + * Path out = new Path(args[2]); + * + * // Specify various job-specific parameters + * job.setJobName("my-app"); + * job.setInputPath(in); + * job.setOutputPath(out); + * job.setMapperClass(MyApp.MyMapper.class); + * job.setReducerClass(MyApp.MyReducer.class); + * + * // Submit the job, then poll for progress until the job is complete + * JobClient.runJob(job); + * } + * + * public static void main(String[] args) throws Exception { + * // Let <code>ToolRunner</code> handle generic command-line options + * int res = ToolRunner.run(new Configuration(), new Sort(), args); + * + * System.exit(res); + * } + * } + * </pre></blockquote></p> + * + * @see GenericOptionsParser + * @see ToolRunner */ public interface Tool extends Configurable { /** * Execute the command with the given arguments. - * @param args command specific arguments - * @return exit code + * + * @param args command specific arguments. + * @return exit code. * @throws Exception */ int run(String [] args) throws Exception; Modified: lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/ToolRunner.java URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/ToolRunner.java?rev=588035&r1=588034&r2=588035&view=diff ============================================================================== --- lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/ToolRunner.java (original) +++ lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/ToolRunner.java Wed Oct 24 14:24:51 2007 @@ -22,23 +22,34 @@ import org.apache.hadoop.conf.Configuration; /** - * ToolRunner can be used to run classes implementing [EMAIL PROTECTED] Tool} - * interface. Static method [EMAIL PROTECTED] #run(Tool, String[])} is used. - * [EMAIL PROTECTED] GenericOptionsParser} is used to parse the hadoop generic - * arguments to modify the <code>Configuration</code>. + * A utility to help run [EMAIL PROTECTED] Tool}s. + * + * <p><code>ToolRunner</code> can be used to run classes implementing + * <code>Tool</code> interface. It works in conjunction with + * [EMAIL PROTECTED] GenericOptionsParser} to parse the + * <a href="[EMAIL PROTECTED]/org/apache/hadoop/util/GenericOptionsParser.html#GenericOptions"> + * generic hadoop command line arguments</a> and modifies the + * <code>Configuration</code> of the <code>Tool</code>. The + * application-specific options are passed along without being modified. + * </p> + * + * @see Tool + * @see GenericOptionsParser */ public class ToolRunner { /** - * Runs the given Tool by [EMAIL PROTECTED] Tool#run(String[])}, with the - * given arguments. Uses the given configuration, or builds one if null. - * Sets the possibly modified version of the conf by Tool#setConf() + * Runs the given <code>Tool</code> by [EMAIL PROTECTED] Tool#run(String[])}, after + * parsing with the given generic arguments. Uses the given + * <code>Configuration</code>, or builds one if null. * - * @param conf Configuration object to use - * @param tool The Tool to run - * @param args the arguments to the tool(including generic arguments - * , see [EMAIL PROTECTED] GenericOptionsParser}) - * @return exit code of the [EMAIL PROTECTED] Tool#run(String[])} method + * Sets the <code>Tool</code>'s configuration with the possibly modified + * version of the <code>conf</code>. + * + * @param conf <code>Configuration</code> for the <code>Tool</code>. + * @param tool <code>Tool</code> to run. + * @param args command-line arguments to the tool. + * @return exit code of the [EMAIL PROTECTED] Tool#run(String[])} method. */ public static int run(Configuration conf, Tool tool, String[] args) throws Exception{ @@ -55,19 +66,24 @@ } /** - * Runs the tool with the tool's Configuration + * Runs the <code>Tool</code> with its <code>Configuration</code>. + * * Equivalent to <code>run(tool.getConf(), tool, args)</code>. - * @param tool The Tool to run - * @param args the arguments to the tool(including generic arguments - * , see [EMAIL PROTECTED] GenericOptionsParser}) - * @return exit code of the [EMAIL PROTECTED] Tool#run(String[])} method + * + * @param tool <code>Tool</code> to run. + * @param args command-line arguments to the tool. + * @return exit code of the [EMAIL PROTECTED] Tool#run(String[])} method. */ public static int run(Tool tool, String[] args) throws Exception{ return run(tool.getConf(), tool, args); } - /** Delegates to GenericOptionsParser#printGenericCommandUsage() */ + /** + * Prints generic command-line argurments and usage information. + * + * @param out stream to write usage information to. + */ public static void printGenericCommandUsage(PrintStream out) { GenericOptionsParser.printGenericCommandUsage(out); } Modified: lucene/hadoop/branches/branch-0.15/src/test/org/apache/hadoop/mapred/NotificationTestCase.java URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.15/src/test/org/apache/hadoop/mapred/NotificationTestCase.java?rev=588035&r1=588034&r2=588035&view=diff ============================================================================== --- lucene/hadoop/branches/branch-0.15/src/test/org/apache/hadoop/mapred/NotificationTestCase.java (original) +++ lucene/hadoop/branches/branch-0.15/src/test/org/apache/hadoop/mapred/NotificationTestCase.java Wed Oct 24 14:24:51 2007 @@ -142,7 +142,7 @@ protected JobConf createJobConf() { JobConf conf = super.createJobConf(); - conf.set("job.end.notification.url", getNotificationUrlTemplate()); + conf.setJobEndNotificationURI(getNotificationUrlTemplate()); conf.setInt("job.end.retry.attempts", 3); conf.setInt("job.end.retry.interval", 200); return conf;
