Updated Branches: refs/heads/master cfccf494d -> 65696ea3b
CRUNCH-68: Fix command line parsing in examples. Remove mainClass from manifest. Remove superfluous use of GenericArgumentParser. Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/65696ea3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/65696ea3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/65696ea3 Branch: refs/heads/master Commit: 65696ea3b39b905051e711771d86138919f5c502 Parents: cfccf49 Author: Matthias Friedrich <[email protected]> Authored: Sat Sep 22 17:09:37 2012 +0200 Committer: Matthias Friedrich <[email protected]> Committed: Sun Sep 23 09:20:39 2012 +0200 ---------------------------------------------------------------------- crunch-examples/pom.xml | 5 ----- .../apache/crunch/examples/AverageBytesByIP.java | 8 +++----- .../org/apache/crunch/examples/TotalBytesByIP.java | 8 +++----- .../java/org/apache/crunch/examples/WordCount.java | 8 +++----- 4 files changed, 9 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/65696ea3/crunch-examples/pom.xml ---------------------------------------------------------------------- diff --git a/crunch-examples/pom.xml b/crunch-examples/pom.xml index 64f2f20..087a3b4 100644 --- a/crunch-examples/pom.xml +++ b/crunch-examples/pom.xml @@ -66,11 +66,6 @@ under the License. <descriptors> <descriptor>src/main/assembly/hadoop-job.xml</descriptor> </descriptors> - <archive> - <manifest> - <mainClass>org.apache.crunch.examples.WordCount</mainClass> - </manifest> - </archive> </configuration> <executions> <execution> http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/65696ea3/crunch-examples/src/main/java/org/apache/crunch/examples/AverageBytesByIP.java ---------------------------------------------------------------------- diff --git a/crunch-examples/src/main/java/org/apache/crunch/examples/AverageBytesByIP.java b/crunch-examples/src/main/java/org/apache/crunch/examples/AverageBytesByIP.java index 52b542a..d6c2a86 100644 --- a/crunch-examples/src/main/java/org/apache/crunch/examples/AverageBytesByIP.java +++ b/crunch-examples/src/main/java/org/apache/crunch/examples/AverageBytesByIP.java @@ -48,9 +48,7 @@ public class AverageBytesByIP extends Configured implements Tool, Serializable { static final String logRegex = "^([\\d.]+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(.+?)\" (\\d{3}) (\\d+) \"([^\"]+)\" \"([^\"]+)\""; public int run(String[] args) throws Exception { - String[] remainingArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs(); - - if (remainingArgs.length != 3) { + if (args.length != 2) { System.err.println(); System.err.println("Two and only two arguments are accepted."); System.err.println("Usage: " + this.getClass().getName() + " [generic options] input output"); @@ -61,7 +59,7 @@ public class AverageBytesByIP extends Configured implements Tool, Serializable { // Create an object to coordinate pipeline creation and execution. Pipeline pipeline = new MRPipeline(AverageBytesByIP.class, getConf()); // Reference a given text file as a collection of Strings. - PCollection<String> lines = pipeline.readTextFile(remainingArgs[1]); + PCollection<String> lines = pipeline.readTextFile(args[0]); // Combiner used for summing up response size and count CombineFn<String, Pair<Long, Long>> stringPairOfLongsSumCombiner = CombineFn.pairAggregator(CombineFn.SUM_LONGS, @@ -78,7 +76,7 @@ public class AverageBytesByIP extends Configured implements Tool, Serializable { Writables.tableOf(Writables.strings(), Writables.doubles())); // write the result to a text file - pipeline.writeTextFile(avgs, remainingArgs[2]); + pipeline.writeTextFile(avgs, args[1]); // Execute the pipeline as a MapReduce. PipelineResult result = pipeline.done(); http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/65696ea3/crunch-examples/src/main/java/org/apache/crunch/examples/TotalBytesByIP.java ---------------------------------------------------------------------- diff --git a/crunch-examples/src/main/java/org/apache/crunch/examples/TotalBytesByIP.java b/crunch-examples/src/main/java/org/apache/crunch/examples/TotalBytesByIP.java index 59b05fa..f1367e0 100644 --- a/crunch-examples/src/main/java/org/apache/crunch/examples/TotalBytesByIP.java +++ b/crunch-examples/src/main/java/org/apache/crunch/examples/TotalBytesByIP.java @@ -47,9 +47,7 @@ public class TotalBytesByIP extends Configured implements Tool, Serializable { static final String logRegex = "^([\\d.]+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(.+?)\" (\\d{3}) (\\d+) \"([^\"]+)\" \"([^\"]+)\""; public int run(String[] args) throws Exception { - String[] remainingArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs(); - - if (remainingArgs.length != 3) { + if (args.length != 2) { System.err.println(); System.err.println("Two and only two arguments are accepted."); System.err.println("Usage: " + this.getClass().getName() + " [generic options] input output"); @@ -60,7 +58,7 @@ public class TotalBytesByIP extends Configured implements Tool, Serializable { // Create an object to coordinate pipeline creation and execution. Pipeline pipeline = new MRPipeline(TotalBytesByIP.class, getConf()); // Reference a given text file as a collection of Strings. - PCollection<String> lines = pipeline.readTextFile(remainingArgs[1]); + PCollection<String> lines = pipeline.readTextFile(args[0]); // Combiner used for summing up response size CombineFn<String, Long> longSumCombiner = CombineFn.SUM_LONGS(); @@ -70,7 +68,7 @@ public class TotalBytesByIP extends Configured implements Tool, Serializable { .parallelDo(extractIPResponseSize, Writables.tableOf(Writables.strings(), Writables.longs())).groupByKey() .combineValues(longSumCombiner); - pipeline.writeTextFile(ipAddrResponseSize, remainingArgs[2]); + pipeline.writeTextFile(ipAddrResponseSize, args[1]); // Execute the pipeline as a MapReduce. PipelineResult result = pipeline.done(); http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/65696ea3/crunch-examples/src/main/java/org/apache/crunch/examples/WordCount.java ---------------------------------------------------------------------- diff --git a/crunch-examples/src/main/java/org/apache/crunch/examples/WordCount.java b/crunch-examples/src/main/java/org/apache/crunch/examples/WordCount.java index 31d99d3..6c984b7 100644 --- a/crunch-examples/src/main/java/org/apache/crunch/examples/WordCount.java +++ b/crunch-examples/src/main/java/org/apache/crunch/examples/WordCount.java @@ -35,9 +35,7 @@ import org.apache.hadoop.util.ToolRunner; public class WordCount extends Configured implements Tool, Serializable { public int run(String[] args) throws Exception { - String[] remainingArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs(); - - if (remainingArgs.length != 3) { + if (args.length != 2) { System.err.println(); System.err.println("Usage: " + this.getClass().getName() + " [generic options] input output"); System.err.println(); @@ -47,7 +45,7 @@ public class WordCount extends Configured implements Tool, Serializable { // Create an object to coordinate pipeline creation and execution. Pipeline pipeline = new MRPipeline(WordCount.class, getConf()); // Reference a given text file as a collection of Strings. - PCollection<String> lines = pipeline.readTextFile(remainingArgs[1]); + PCollection<String> lines = pipeline.readTextFile(args[0]); // Define a function that splits each line in a PCollection of Strings into // a @@ -67,7 +65,7 @@ public class WordCount extends Configured implements Tool, Serializable { PTable<String, Long> counts = words.count(); // Instruct the pipeline to write the resulting counts to a text file. - pipeline.writeTextFile(counts, remainingArgs[2]); + pipeline.writeTextFile(counts, args[1]); // Execute the pipeline as a MapReduce. PipelineResult result = pipeline.done();
