Author: gsingers
Date: Fri Oct 17 10:45:45 2008
New Revision: 705685
URL: http://svn.apache.org/viewvc?rev=705685&view=rev
Log:
upgraded CLI stuff to use CLI2, various build/pom changes for MAHOUT-69. I
believe ant package now works
Modified:
lucene/mahout/trunk/build.xml
lucene/mahout/trunk/common-build.xml
lucene/mahout/trunk/core/pom.xml
lucene/mahout/trunk/examples/pom.xml
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/PrepareTwentyNewsgroups.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreator.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java
Modified: lucene/mahout/trunk/build.xml
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/build.xml?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/build.xml (original)
+++ lucene/mahout/trunk/build.xml Fri Oct 17 10:45:45 2008
@@ -124,7 +124,18 @@
<mahout-checksum file="${dist}/${fullnamever}.zip"/>
<!-- create maven artifacts -->
- <exec command="mvn -Dtest=false deploy"/>
+ <!--<exec command="mvn -Dtest=false deploy"/>-->
+ <input message="Enter the GPG Passphrase" addproperty="gpg.passphrase"/>
+ <exec executable="mvn">
+ <arg value="-Dtest=false"/>
+ <arg value="install"/>
+ <arg value="-Dgpg.passphrase=${gpg.passphrase}"/>
+ </exec>
+ <exec executable="mvn">
+ <arg value="-Dtest=false"/>
+ <arg value="deploy"/>
+ <arg value="-Dgpg.passphrase=${gpg.passphrase}"/>
+ </exec>
<mkdir dir="${dist}/maven"/>
<copy todir="${dist}/maven">
<fileset dir="core/dist/maven"/>
Modified: lucene/mahout/trunk/common-build.xml
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/common-build.xml?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/common-build.xml (original)
+++ lucene/mahout/trunk/common-build.xml Fri Oct 17 10:45:45 2008
@@ -62,7 +62,9 @@
<property name="dist" value="dist" />
- <property name="shared-lib" value="./lib" />
+ <dirname file="${ant.file.common}" property="common.dir"/>
+
+ <property name="shared-lib" value="${common.dir}/lib" />
<!-- Javadoc properties -->
<property name="javadoc.years" value="2008 - ${year}" />
Modified: lucene/mahout/trunk/core/pom.xml
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/pom.xml?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/core/pom.xml (original)
+++ lucene/mahout/trunk/core/pom.xml Fri Oct 17 10:45:45 2008
@@ -7,7 +7,7 @@
<parent>
<groupId>org.apache.mahout</groupId>
<artifactId>mahout-parent</artifactId>
- <version>0.2-SNAPSHOT</version>
+ <version>0.1-SNAPSHOT</version>
</parent>
<groupId>org.apache.mahout</groupId>
Modified: lucene/mahout/trunk/examples/pom.xml
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/pom.xml?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/pom.xml (original)
+++ lucene/mahout/trunk/examples/pom.xml Fri Oct 17 10:45:45 2008
@@ -7,7 +7,7 @@
<parent>
<groupId>org.apache.mahout</groupId>
<artifactId>mahout-parent</artifactId>
- <version>0.2-SNAPSHOT</version>
+ <version>0.1-SNAPSHOT</version>
</parent>
<groupId>org.apache.mahout</groupId>
@@ -74,14 +74,14 @@
<dependencies>
<dependency>
- <groupId>org.apache.lucene.mahout</groupId>
+ <groupId>org.apache.mahout</groupId>
<artifactId>core</artifactId>
<version>${parent.version}</version>
</dependency>
<!-- core test -->
<dependency>
- <groupId>org.apache.lucene.mahout</groupId>
+ <groupId>org.apache.mahout</groupId>
<artifactId>core</artifactId>
<version>${parent.version}</version>
<classifier>tests</classifier>
@@ -125,6 +125,18 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.5.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jcl</artifactId>
+ <version>1.5.2</version>
+ </dependency>
+
</dependencies>
Modified:
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/PrepareTwentyNewsgroups.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/PrepareTwentyNewsgroups.java?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
---
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/PrepareTwentyNewsgroups.java
(original)
+++
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/PrepareTwentyNewsgroups.java
Fri Oct 17 10:45:45 2008
@@ -16,15 +16,16 @@
* limitations under the License.
*/
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.Parser;
import org.apache.lucene.analysis.Analyzer;
import org.apache.mahout.classifier.BayesFileFormatter;
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.commandline.Parser;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
import java.io.File;
import java.io.IOException;
@@ -41,25 +42,40 @@
public class PrepareTwentyNewsgroups {
@SuppressWarnings("static-access")
- public static void main(String[] args) throws IOException, ParseException,
- ClassNotFoundException, InstantiationException, IllegalAccessException {
- Options options = new Options();
- Option parentOpt =
OptionBuilder.withLongOpt("parent").isRequired().hasArg().withDescription("Parent
dir containing the newsgroups").create("p");
- options.addOption(parentOpt);
- Option outputDirOpt =
OptionBuilder.withLongOpt("outputDir").isRequired().hasArg().withDescription("The
output directory").create("o");
- options.addOption(outputDirOpt);
- Option analyzerNameOpt =
OptionBuilder.withLongOpt("analyzerName").isRequired().hasArg().withDescription("The
class name of the analyzer").create("a");
- options.addOption(analyzerNameOpt);
- Option charsetOpt =
OptionBuilder.withLongOpt("charset").hasArg().isRequired().withDescription("The
name of the character encoding of the input files").create("c");
- options.addOption(charsetOpt);
-
- Parser parser = new PosixParser();
- CommandLine cmdLine = parser.parse(options, args);
-
- File parentDir = new File(cmdLine.getOptionValue(parentOpt.getOpt()));
- File outputDir = new File(cmdLine.getOptionValue(outputDirOpt.getOpt()));
- String analyzerName = cmdLine.getOptionValue(analyzerNameOpt.getOpt());
- Charset charset =
Charset.forName(cmdLine.getOptionValue(charsetOpt.getOpt()));
+ public static void main(String[] args) throws IOException,
+ ClassNotFoundException, InstantiationException,
IllegalAccessException, OptionException {
+ final DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
+ final ArgumentBuilder abuilder = new ArgumentBuilder();
+ final GroupBuilder gbuilder = new GroupBuilder();
+
+ Option parentOpt =
obuilder.withLongName("parent").withRequired(true).withArgument(
+
abuilder.withName("parent").withMinimum(1).withMaximum(1).create()).
+ withDescription("Parent dir containing the
newsgroups").withShortName("p").create();
+
+ Option outputDirOpt =
obuilder.withLongName("outputDir").withRequired(true).withArgument(
+
abuilder.withName("outputDir").withMinimum(1).withMaximum(1).create()).
+ withDescription("The output
directory").withShortName("o").create();
+
+ Option analyzerNameOpt =
obuilder.withLongName("analyzerName").withRequired(true).withArgument(
+
abuilder.withName("analyzerName").withMinimum(1).withMaximum(1).create()).
+ withDescription("The class name of the
analyzer").withShortName("a").create();
+
+ Option charsetOpt =
obuilder.withLongName("charset").withRequired(true).withArgument(
+
abuilder.withName("charset").withMinimum(1).withMaximum(1).create()).
+ withDescription("The name of the character encoding of the input
files").withShortName("c").create();
+
+ Group group =
gbuilder.withName("Options").withOption(analyzerNameOpt).withOption(charsetOpt).withOption(outputDirOpt).withOption(parentOpt).create();
+
+ CommandLine cmdLine;
+ Parser parser = new Parser();
+ parser.setGroup(group);
+ cmdLine = parser.parse(args);
+
+
+ File parentDir = new File((String) cmdLine.getValue(parentOpt));
+ File outputDir = new File((String) cmdLine.getValue(outputDirOpt));
+ String analyzerName = (String) cmdLine.getValue(analyzerNameOpt);
+ Charset charset = Charset.forName((String) cmdLine.getValue(charsetOpt));
Analyzer analyzer = (Analyzer) Class.forName(analyzerName).newInstance();
//parent dir contains dir by category
File [] categoryDirs = parentDir.listFiles();
Modified:
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
---
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
(original)
+++
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
Fri Oct 17 10:45:45 2008
@@ -17,13 +17,6 @@
* limitations under the License.
*/
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.Parser;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
@@ -36,6 +29,14 @@
import org.apache.mahout.classifier.cbayes.CBayesModel;
import org.apache.mahout.common.Classifier;
import org.apache.mahout.common.Model;
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.commandline.Parser;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,37 +59,53 @@
@SuppressWarnings({ "static-access", "unchecked" })
public static void main(String[] args) throws IOException,
- ClassNotFoundException, IllegalAccessException, InstantiationException,
ParseException {
- Options options = new Options();
- Option pathOpt = OptionBuilder.withLongOpt("path").isRequired().hasArg()
- .withDescription("The local file system path").create("p");
- options.addOption(pathOpt);
- Option dirOpt = OptionBuilder.withLongOpt("testDir").isRequired().hasArg()
- .withDescription("The directory where test documents resides
in").create("t");
- options.addOption(dirOpt);
- Option encodingOpt = OptionBuilder.withLongOpt("encoding").hasArg()
- .withDescription("The file encoding. defaults to UTF-8").create("e");
- options.addOption(encodingOpt);
- Option analyzerOpt = OptionBuilder.withLongOpt("analyzer").hasArg()
- .withDescription("The Analyzer to use").create("a");
- options.addOption(analyzerOpt);
- Option defaultCatOpt = OptionBuilder.withLongOpt("defaultCat").hasArg()
- .withDescription("The default category").create("d");
- options.addOption(defaultCatOpt);
- Option gramSizeOpt = OptionBuilder.withLongOpt("gramSize").hasArg()
- .withDescription("Size of the n-gram").create("ng");
- options.addOption(gramSizeOpt);
- Option typeOpt = OptionBuilder.withLongOpt("classifierType").isRequired()
- .hasArg().withDescription("Type of classifier").create("type");
- options.addOption(typeOpt);
+ ClassNotFoundException, IllegalAccessException,
InstantiationException, OptionException {
+ final DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
+ final ArgumentBuilder abuilder = new ArgumentBuilder();
+ final GroupBuilder gbuilder = new GroupBuilder();
+
+ Option pathOpt =
obuilder.withLongName("path").withRequired(true).withArgument(
+ abuilder.withName("path").withMinimum(1).withMaximum(1).create()).
+ withDescription("The local file system
path").withShortName("p").create();
+
+ Option dirOpt =
obuilder.withLongName("testDir").withRequired(true).withArgument(
+
abuilder.withName("testDir").withMinimum(1).withMaximum(1).create()).
+ withDescription("The directory where test documents resides
in").withShortName("t").create();
+
+ Option encodingOpt =
obuilder.withLongName("encoding").withRequired(true).withArgument(
+
abuilder.withName("encoding").withMinimum(1).withMaximum(1).create()).
+ withDescription("The file encoding. Defaults to
UTF-8").withShortName("e").create();
+
+ Option analyzerOpt =
obuilder.withLongName("analyzer").withRequired(true).withArgument(
+
abuilder.withName("analyzer").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Analyzer to use").withShortName("a").create();
+
+ Option defaultCatOpt =
obuilder.withLongName("defaultCat").withRequired(true).withArgument(
+
abuilder.withName("defaultCat").withMinimum(1).withMaximum(1).create()).
+ withDescription("The default
category").withShortName("d").create();
+
+ Option gramSizeOpt =
obuilder.withLongName("gramSize").withRequired(true).withArgument(
+
abuilder.withName("gramSize").withMinimum(1).withMaximum(1).create()).
+ withDescription("Size of the n-gram").withShortName("ng").create();
+
+ Option typeOpt =
obuilder.withLongName("classifierType").withRequired(true).withArgument(
+
abuilder.withName("classifierType").withMinimum(1).withMaximum(1).create()).
+ withDescription("Type of classifier:
bayes|cbayes").withShortName("type").create();
+
+ Group group =
gbuilder.withName("Options").withOption(analyzerOpt).withOption(defaultCatOpt).withOption(dirOpt).withOption(encodingOpt).withOption(gramSizeOpt).withOption(pathOpt)
+ .withOption(typeOpt).create();
+
+ CommandLine cmdLine;
+ Parser parser = new Parser();
+ parser.setGroup(group);
+ cmdLine = parser.parse(args);
+
- Parser parser = new PosixParser();
- CommandLine cmdLine = parser.parse(options, args);
SequenceFileModelReader reader = new SequenceFileModelReader();
JobConf conf = new JobConf(TestClassifier.class);
Map<String, Path> modelPaths = new HashMap<String, Path>();
- String modelBasePath = cmdLine.getOptionValue(pathOpt.getOpt());
+ String modelBasePath = (String) cmdLine.getValue(pathOpt);
modelPaths.put("sigma_j", new Path(modelBasePath +
"/trainer-weights/Sigma_j/part-*"));
modelPaths.put("sigma_k", new Path(modelBasePath +
"/trainer-weights/Sigma_k/part-*"));
modelPaths.put("sigma_kSigma_j", new Path(modelBasePath +
"/trainer-weights/Sigma_kSigma_j/part-*"));
@@ -102,7 +119,7 @@
Model model;
Classifier classifier;
- String classifierType = cmdLine.getOptionValue(typeOpt.getOpt());
+ String classifierType = (String) cmdLine.getValue(typeOpt);
if (classifierType.equalsIgnoreCase("bayes")) {
log.info("Testing Bayes Classifier");
@@ -123,17 +140,17 @@
log.info("Done generating Model");
String defaultCat = "unknown";
- if (cmdLine.hasOption(defaultCatOpt.getOpt())) {
- defaultCat = cmdLine.getOptionValue(defaultCatOpt.getOpt());
+ if (cmdLine.hasOption(defaultCatOpt)) {
+ defaultCat = (String) cmdLine.getValue(defaultCatOpt);
}
String encoding = "UTF-8";
- if (cmdLine.hasOption(encodingOpt.getOpt())) {
- encoding = cmdLine.getOptionValue(encodingOpt.getOpt());
+ if (cmdLine.hasOption(encodingOpt)) {
+ encoding = (String) cmdLine.getValue(encodingOpt);
}
Analyzer analyzer = null;
- if (cmdLine.hasOption(analyzerOpt.getOpt())) {
- String className = cmdLine.getOptionValue(analyzerOpt.getOpt());
+ if (cmdLine.hasOption(analyzerOpt)) {
+ String className = (String) cmdLine.getValue(analyzerOpt);
Class clazz = Class.forName(className);
analyzer = (Analyzer) clazz.newInstance();
}
@@ -142,13 +159,13 @@
}
// TODO srowen says analyzer is never used?
int gramSize = 1;
- if (cmdLine.hasOption(gramSizeOpt.getOpt())) {
- gramSize = Integer.parseInt(cmdLine
- .getOptionValue(gramSizeOpt.getOpt()));
+ if (cmdLine.hasOption(gramSizeOpt)) {
+ gramSize = Integer.parseInt((String) cmdLine
+ .getValue(gramSizeOpt));
}
- String testDirPath = cmdLine.getOptionValue(dirOpt.getOpt());
+ String testDirPath = (String) cmdLine.getValue(dirOpt);
File dir = new File(testDirPath);
File[] subdirs = dir.listFiles();
Modified:
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
---
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java
(original)
+++
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java
Fri Oct 17 10:45:45 2008
@@ -16,13 +16,14 @@
* limitations under the License.
*/
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.Parser;
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
+import org.apache.commons.cli2.commandline.Parser;
import org.apache.mahout.classifier.cbayes.CBayesDriver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,7 +32,7 @@
/**
* Train the Naive Bayes Complement classifier with improved weighting on the
Twenty Newsgroups data
(http://people.csail.mit.edu/jrennie/20Newsgroups/20news-18828.tar.gz)
- *
+ * <p/>
* To run:
* Assume MAHOUT_HOME refers to the location where you checked out/installed
Mahout
* <ol>
@@ -40,13 +41,13 @@
* <li>Start up Hadoop and copy the files to the system. See
http://hadoop.apache.org/core/docs/r0.16.2/quickstart.html</li>
* <li>From the Hadoop dir (where Hadoop is installed):
* <ol>
- * <li>emacs conf/hadoop-site.xml (add in local settings per
quickstart)</li>
- * <li>bin/hadoop namenode -format //Format the HDFS</li>
- * <li>bin/start-all.sh //Start Hadoop</li>
- * <li>bin/hadoop dfs -put <MAHOUT_HOME>/work/20news-18828-collapse
20newsInput //Copies the extracted text to HDFS</li>
- * <li>bin/hadoop jar
<MAHOUT_HOME>/build/apache-mahout-0.1-dev-ex.jar
org.apache.mahout.examples.classifiers.cbayes.TwentyNewsgroups -t -i
20newsInput -o 20newsOutput</li>
+ * <li>emacs conf/hadoop-site.xml (add in local settings per quickstart)</li>
+ * <li>bin/hadoop namenode -format //Format the HDFS</li>
+ * <li>bin/start-all.sh //Start Hadoop</li>
+ * <li>bin/hadoop dfs -put <MAHOUT_HOME>/work/20news-18828-collapse
20newsInput //Copies the extracted text to HDFS</li>
+ * <li>bin/hadoop jar <MAHOUT_HOME>/build/apache-mahout-0.1-dev-ex.jar
org.apache.mahout.examples.classifiers.cbayes.TwentyNewsgroups -t -i
20newsInput -o 20newsOutput</li>
* </ol>
- * </li>
+ * </li>
* </ol>
*/
public class TrainClassifier {
@@ -60,36 +61,47 @@
public void trainCNaiveBayes(String dir, String outputDir, int gramSize)
throws IOException {
CBayesDriver.runJob(dir, outputDir, gramSize);
}
-
- @SuppressWarnings("static-access")
- public static void main(String[] args) throws IOException, ParseException {
- Options options = new Options();
- Option trainOpt =
OptionBuilder.withLongOpt("train").withDescription("Train the
classifier").create("t");
- options.addOption(trainOpt);
- Option inputDirOpt =
OptionBuilder.withLongOpt("inputDir").hasArg().withDescription("The Directory
on HDFS containing the collapsed, properly formatted files").create("i");
- options.addOption(inputDirOpt);
- Option outputOpt =
OptionBuilder.withLongOpt("output").isRequired().hasArg().withDescription("The
location of the model on the HDFS").create("o");
- options.addOption(outputOpt);
- Option gramSizeOpt =
OptionBuilder.withLongOpt("gramSize").hasArg().withDescription("Size of the
n-gram").create("ng");
- options.addOption(gramSizeOpt);
- Option typeOpt =
OptionBuilder.withLongOpt("classifierType").isRequired().hasArg().withDescription("Type
of classifier").create("type");
- options.addOption(typeOpt);
-
- Parser parser = new PosixParser();
- CommandLine cmdLine = parser.parse(options, args);
- boolean train = cmdLine.hasOption(trainOpt.getOpt());
+ @SuppressWarnings("static-access")
+ public static void main(String[] args) throws IOException, OptionException {
+ final DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
+ final ArgumentBuilder abuilder = new ArgumentBuilder();
+ final GroupBuilder gbuilder = new GroupBuilder();
+ Option trainOpt = obuilder.withLongName("train").withRequired(true).
+ withDescription("Train the
classifier").withShortName("t").create();
+
+ Option inputDirOpt =
obuilder.withLongName("inputDir").withRequired(true).withArgument(
+
abuilder.withName("inputDir").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Directory on HDFS containing the collapsed,
properly formatted files").withShortName("i").create();
+
+ Option outputOpt =
obuilder.withLongName("output").withRequired(true).withArgument(
+
abuilder.withName("output").withMinimum(1).withMaximum(1).create()).
+ withDescription("The location of the modelon the
HDFS").withShortName("o").create();
+
+ Option gramSizeOpt =
obuilder.withLongName("gramSize").withRequired(true).withArgument(
+
abuilder.withName("gramSize").withMinimum(1).withMaximum(1).create()).
+ withDescription("Size of the n-gram").withShortName("ng").create();
+
+ Option typeOpt =
obuilder.withLongName("classifierType").withRequired(true).withArgument(
+
abuilder.withName("classifierType").withMinimum(1).withMaximum(1).create()).
+ withDescription("Type of classifier: bayes or
cbayes").withShortName("type").create();
+ Group group =
gbuilder.withName("Options").withOption(gramSizeOpt).withOption(inputDirOpt).withOption(outputOpt).withOption(trainOpt).withOption(typeOpt).create();
+ CommandLine cmdLine;
+ Parser parser = new Parser();
+ parser.setGroup(group);
+ cmdLine = parser.parse(args);
+ boolean train = cmdLine.hasOption(trainOpt);
TrainClassifier tn = new TrainClassifier();
- if (train){
- String classifierType = cmdLine.getOptionValue(typeOpt.getOpt());
- if(classifierType.equalsIgnoreCase("bayes")){
+ if (train) {
+ String classifierType = (String) cmdLine.getValue(typeOpt);
+ if (classifierType.equalsIgnoreCase("bayes")) {
log.info("Training Bayes Classifier");
- tn.trainNaiveBayes(cmdLine.getOptionValue(inputDirOpt.getOpt()),
cmdLine.getOptionValue(outputOpt.getOpt()),
Integer.parseInt(cmdLine.getOptionValue(gramSizeOpt.getOpt())));
+ tn.trainNaiveBayes((String)cmdLine.getValue(inputDirOpt),
(String)cmdLine.getValue(outputOpt), Integer.parseInt((String)
cmdLine.getValue(gramSizeOpt)));
- } else if(classifierType.equalsIgnoreCase("cbayes")) {
+ } else if (classifierType.equalsIgnoreCase("cbayes")) {
log.info("Training Complementary Bayes Classifier");
//setup the HDFS and copy the files there, then run the trainer
- tn.trainCNaiveBayes(cmdLine.getOptionValue(inputDirOpt.getOpt()),
cmdLine.getOptionValue(outputOpt.getOpt()),
Integer.parseInt(cmdLine.getOptionValue(gramSizeOpt.getOpt())));
+ tn.trainCNaiveBayes((String) cmdLine.getValue(inputDirOpt), (String)
cmdLine.getValue(outputOpt), Integer.parseInt((String)
cmdLine.getValue(gramSizeOpt)));
}
}
Modified:
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreator.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreator.java?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
---
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreator.java
(original)
+++
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreator.java
Fri Oct 17 10:45:45 2008
@@ -17,13 +17,15 @@
* limitations under the License.
*/
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.Parser;
+
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.commandline.Parser;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
import java.io.IOException;
@@ -31,24 +33,35 @@
@SuppressWarnings("static-access")
public static void main(String[] args) throws IOException,
- ClassNotFoundException, IllegalAccessException, InstantiationException,
ParseException {
- Options options = new Options();
- Option dirInputPathOpt =
OptionBuilder.withLongOpt("dirInputPath").isRequired().hasArg()
- .withDescription("The input Directory Path").create("i");
- options.addOption(dirInputPathOpt);
- Option dirOutputPathOpt =
OptionBuilder.withLongOpt("dirOuputPath").isRequired().hasArg()
- .withDescription("The output Directory Path").create("o");
- options.addOption(dirOutputPathOpt);
- Option countriesFileOpt =
OptionBuilder.withLongOpt("countriesFile").isRequired().hasArg()
- .withDescription("Location of the Countries File").create("c");
- options.addOption(countriesFileOpt);
+ ClassNotFoundException, IllegalAccessException,
InstantiationException, OptionException {
+ final DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
+ final ArgumentBuilder abuilder = new ArgumentBuilder();
+ final GroupBuilder gbuilder = new GroupBuilder();
+
+ Option dirInputPathOpt =
obuilder.withLongName("dirInputPath").withRequired(true).withArgument(
+
abuilder.withName("dirInputPath").withMinimum(1).withMaximum(1).create()).
+ withDescription("The input directory
path").withShortName("i").create();
+
+ Option dirOutputPathOpt =
obuilder.withLongName("dirOutputPath").withRequired(true).withArgument(
+
abuilder.withName("dirOutputPath").withMinimum(1).withMaximum(1).create()).
+ withDescription("The output directory
Path").withShortName("o").create();
+
+ Option countriesFileOpt =
obuilder.withLongName("countriesFile").withRequired(true).withArgument(
+
abuilder.withName("countriesFile").withMinimum(1).withMaximum(1).create()).
+ withDescription("Location of the countries
file").withShortName("c").create();
+
+ Group group =
gbuilder.withName("Options").withOption(countriesFileOpt).withOption(dirInputPathOpt).withOption(dirOutputPathOpt).create();
+
+ CommandLine cmdLine;
+ Parser parser = new Parser();
+ parser.setGroup(group);
+ cmdLine = parser.parse(args);
- Parser parser = new PosixParser();
- CommandLine cmdLine = parser.parse(options, args);
- String dirInputPath = cmdLine.getOptionValue(dirInputPathOpt.getOpt());
- String dirOutputPath = cmdLine.getOptionValue(dirOutputPathOpt.getOpt());
- String countriesFile = cmdLine.getOptionValue(countriesFileOpt.getOpt());
+
+ String dirInputPath = (String) cmdLine.getValue(dirInputPathOpt);
+ String dirOutputPath = (String) cmdLine.getValue(dirOutputPathOpt);
+ String countriesFile = (String) cmdLine.getValue(countriesFileOpt);
WikipediaDatasetCreatorDriver.runJob(dirInputPath, dirOutputPath,
countriesFile);
Modified:
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
---
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java
(original)
+++
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java
Fri Oct 17 10:45:45 2008
@@ -17,13 +17,15 @@
* limitations under the License.
*/
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.Parser;
+
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.commandline.Parser;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -39,21 +41,31 @@
public class WikipediaXmlSplitter {
@SuppressWarnings("static-access")
- public static void main(String[] args) throws IOException, ParseException {
- Options options = new Options();
- Option dumpFileOpt =
OptionBuilder.withLongOpt("dumpfile").isRequired().hasArg().withDescription("The
path to the wikipedia dump file").create("d");
- options.addOption(dumpFileOpt);
- Option outputDirOpt =
OptionBuilder.withLongOpt("outputDir").isRequired().hasArg().withDescription("The
output directory to place the splits in").create("o");
- options.addOption(outputDirOpt);
- Option chunkSizeOpt =
OptionBuilder.withLongOpt("chunkSize").isRequired().hasArg().withDescription("the
Size of chunk in Megabytes").create("c");
- options.addOption(chunkSizeOpt);
-
- Parser parser = new PosixParser();
- CommandLine cmdLine = parser.parse(options, args);
-
- String dumpFilePath = cmdLine.getOptionValue(dumpFileOpt.getOpt());
- String outputDirPath = cmdLine.getOptionValue(outputDirOpt.getOpt());
- int chunkSize = 1024 * 1024 *
Integer.parseInt(cmdLine.getOptionValue(chunkSizeOpt.getOpt()));
+ public static void main(String[] args) throws IOException, OptionException {
+ final DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
+ final ArgumentBuilder abuilder = new ArgumentBuilder();
+ final GroupBuilder gbuilder = new GroupBuilder();
+
+ Option dumpFileOpt =
obuilder.withLongName("dumpFile").withRequired(true).withArgument(
+
abuilder.withName("dumpFile").withMinimum(1).withMaximum(1).create()).
+ withDescription("The path to the wikipedia dump
file").withShortName("d").create();
+
+ Option outputDirOpt =
obuilder.withLongName("outputDir").withRequired(true).withArgument(
+
abuilder.withName("outputDir").withMinimum(1).withMaximum(1).create()).
+ withDescription("The output directory to place the splits
in").withShortName("o").create();
+
+ Option chunkSizeOpt =
obuilder.withLongName("chunkSize").withRequired(true).withArgument(
+
abuilder.withName("chunkSize").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Size of the chunk, in
megabytes").withShortName("c").create();
+ Group group =
gbuilder.withName("Options").withOption(dumpFileOpt).withOption(outputDirOpt).withOption(chunkSizeOpt).create();
+ CommandLine cmdLine;
+ Parser parser = new Parser();
+ parser.setGroup(group);
+ cmdLine = parser.parse(args);
+
+ String dumpFilePath = (String) cmdLine.getValue(dumpFileOpt);
+ String outputDirPath = (String) cmdLine.getValue(outputDirOpt);
+ int chunkSize = 1024 * 1024 * Integer.parseInt((String)
cmdLine.getValue(chunkSizeOpt));
BufferedReader dumpReader = new BufferedReader(new InputStreamReader(
new FileInputStream(dumpFilePath), "UTF-8"));
