Author: gsingers
Date: Fri Nov 7 15:39:26 2008
New Revision: 712310
URL: http://svn.apache.org/viewvc?rev=712310&view=rev
Log:
MAHOUT-92: fixed and tested out. Mmmm, Yo Mama's Peanut Butter Burger
Added:
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFeatureMapperTest.java
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/Classify.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ConfusionMatrix.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ResultAnalyzer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesClassifier.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/Classifier.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/CBayesClassifierTest.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/Classify.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/Classify.java?rev=712310&r1=712309&r2=712310&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/Classify.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/Classify.java
Fri Nov 7 15:39:26 2008
@@ -34,7 +34,6 @@
import org.apache.mahout.classifier.bayes.BayesClassifier;
import org.apache.mahout.classifier.bayes.BayesModel;
import org.apache.mahout.classifier.bayes.io.SequenceFileModelReader;
-import org.apache.mahout.classifier.cbayes.CBayesClassifier;
import org.apache.mahout.classifier.cbayes.CBayesModel;
import org.apache.mahout.common.Classifier;
import org.apache.mahout.common.Model;
@@ -117,14 +116,13 @@
if (classifierType.equalsIgnoreCase("bayes")) {
log.info("Testing Bayes Classifier");
model = new BayesModel();
- classifier = new BayesClassifier();
} else if (classifierType.equalsIgnoreCase("cbayes")) {
log.info("Testing Complementary Bayes Classifier");
model = new CBayesModel();
- classifier = new CBayesClassifier();
} else {
throw new IllegalArgumentException("Unrecognized classifier type: " +
classifierType);
}
+ classifier = new BayesClassifier();
model = reader.loadModel(model, fs, modelPaths, conf);
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ConfusionMatrix.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ConfusionMatrix.java?rev=712310&r1=712309&r2=712310&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ConfusionMatrix.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ConfusionMatrix.java
Fri Nov 7 15:39:26 2008
@@ -27,14 +27,27 @@
/**
* The ConfusionMatrix Class stores the result of Classification of a Test
Dataset.
+ *
+ * See http://en.wikipedia.org/wiki/Confusion_matrix for background
*/
public class ConfusionMatrix implements Summarizable {
- private Collection<String> labels = new ArrayList<String>();
+ private Collection<String> labels;
private final Map<String, Integer> labelMap = new HashMap<String, Integer>();
private int[][] confusionMatrix = null;
+ private String defaultLabel = "unknown";
+
+ public ConfusionMatrix(Collection<String> labels, String defaultLabel) {
+ this.labels = labels;
+ confusionMatrix = new int[labels.size() + 1][labels.size() + 1];
+ this.defaultLabel = defaultLabel;
+ for (String label : labels) {
+ labelMap.put(label, labelMap.size());
+ }
+ labelMap.put(defaultLabel, labelMap.size());
+ }
public int[][] getConfusionMatrix() {
return confusionMatrix;
@@ -43,11 +56,11 @@
public Collection<String> getLabels() {
return labels;
}
-
+
public double getAccuracy(String label){
int labelId = labelMap.get(label);
int labelTotal = 0;
- int correct = 0;
+ int correct = 0;
for(int i = 0 ;i < labels.size() ;i++){
labelTotal += confusionMatrix[labelId][i];
if(i == labelId)
@@ -55,12 +68,13 @@
}
return 100.0 * correct / labelTotal;
}
-
+
public int getCorrect(String label){
int labelId = labelMap.get(label);
return confusionMatrix[labelId][labelId];
}
-
+
+
public double getTotal(String label){
int labelId = labelMap.get(label);
int labelTotal = 0;
@@ -69,16 +83,7 @@
}
return labelTotal;
}
-
- public ConfusionMatrix(Collection<String> labels) {
- this.labels = labels;
- confusionMatrix = new int[labels.size()][labels.size()];
- for (String label : labels) {
- labelMap.put(label, labelMap.size());
- }
- }
-
public void addInstance(String correctLabel, ClassifierResult
classifiedResult) {
incrementCount(correctLabel, classifiedResult.getLabel());
}
@@ -88,8 +93,8 @@
}
public int getCount(String correctLabel, String classifiedLabel) {
- if (this.getLabels().contains(correctLabel)
- && this.getLabels().contains(classifiedLabel) == false) {
+ if (labels.contains(correctLabel)
+ && labels.contains(classifiedLabel) == false &&
defaultLabel.equals(classifiedLabel) == false) {
throw new IllegalArgumentException("Label not found " +correctLabel + "
" +classifiedLabel );
}
int correctId = labelMap.get(correctLabel);
@@ -98,8 +103,8 @@
}
public void putCount(String correctLabel, String classifiedLabel, int count)
{
- if (this.getLabels().contains(correctLabel)
- && this.getLabels().contains(classifiedLabel) == false) {
+ if (labels.contains(correctLabel)
+ && labels.contains(classifiedLabel) == false &&
defaultLabel.equals(classifiedLabel) == false) {
throw new IllegalArgumentException("Label not found");
}
int correctId = labelMap.get(correctLabel);
@@ -118,10 +123,10 @@
}
public ConfusionMatrix Merge(ConfusionMatrix b) {
- if (this.getLabels().size() != b.getLabels().size())
+ if (labels.size() != b.getLabels().size())
throw new IllegalArgumentException("The Labels do not Match");
- //if (this.getLabels().containsAll(b.getLabels()))
+ //if (labels.containsAll(b.getLabels()))
// ;
for (String correctLabel : this.labels) {
for (String classifiedLabel : this.labels) {
@@ -133,18 +138,19 @@
}
public String summarize() {
+ String lineSep = System.getProperty("line.separator");
StringBuilder returnString = new StringBuilder();
returnString
- .append("=======================================================\n");
+
.append("=======================================================").append(lineSep);
returnString.append("Confusion Matrix\n");
returnString
- .append("-------------------------------------------------------\n");
+
.append("-------------------------------------------------------").append(lineSep);
for (String correctLabel : this.labels) {
returnString.append(StringUtils.rightPad(getSmallLabel(labelMap.get(correctLabel)),
5)).append('\t');
}
- returnString.append("<--Classified as\n");
+ returnString.append("<--Classified as").append(lineSep);
for (String correctLabel : this.labels) {
int labelTotal = 0;
@@ -155,9 +161,10 @@
}
returnString.append(" |
").append(StringUtils.rightPad(String.valueOf(labelTotal), 6)).append('\t')
.append(StringUtils.rightPad(getSmallLabel(labelMap.get(correctLabel)), 5))
- .append(" = ").append(correctLabel).append('\n');
+ .append(" = ").append(correctLabel).append(lineSep);
}
- returnString.append('\n');
+ returnString.append("Default Category: ").append(defaultLabel).append(":
").append(labelMap.get(defaultLabel)).append(lineSep);
+ returnString.append(lineSep);
return returnString.toString();
}
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ResultAnalyzer.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ResultAnalyzer.java?rev=712310&r1=712309&r2=712310&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ResultAnalyzer.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ResultAnalyzer.java
Fri Nov 7 15:39:26 2008
@@ -39,8 +39,8 @@
private int incorrectlyClassified = 0;
- public ResultAnalyzer(Collection<String> labelSet) {
- confusionMatrix = new ConfusionMatrix(labelSet);
+ public ResultAnalyzer(Collection<String> labelSet, String defaultLabel) {
+ confusionMatrix = new ConfusionMatrix(labelSet, defaultLabel);
}
public ConfusionMatrix getConfusionMatrix(){
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesClassifier.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesClassifier.java?rev=712310&r1=712309&r2=712310&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesClassifier.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesClassifier.java
Fri Nov 7 15:39:26 2008
@@ -50,8 +50,8 @@
PriorityQueue pq = new ClassifierResultPriorityQueue(numResults);
ClassifierResult tmp;
for (String category : categories){
- double prob = documentProbability(model, category, document);
- if (prob < 0.0) {
+ double prob = documentWeight(model, category, document);
+ if (prob > 0.0) {
tmp = new ClassifierResult(category, prob);
pq.insert(tmp);
}
@@ -77,22 +77,22 @@
*/
public ClassifierResult classify(Model model, String[] document, String
defaultCategory) {
ClassifierResult result = new ClassifierResult(defaultCategory);
- double min = 0.0;
+ double max = Double.MAX_VALUE;
Collection<String> categories = model.getLabels();
for (String category : categories) {
- double prob = documentProbability(model, category, document);
- if (prob < min) {
- min = prob;
+ double prob = documentWeight(model, category, document);
+ if (prob < max) {
+ max = prob;
result.setLabel(category);
}
}
- result.setScore(min);
+ result.setScore(max);
return result;
}
/**
- * Calculate the document probability as the multiplication of the
+ * Calculate the document weight as the multiplication of the
* [EMAIL PROTECTED] org.apache.mahout.common.Model#featureWeight(String,
String)} for each word given the label
*
* @param model The [EMAIL PROTECTED] org.apache.mahout.common.Model}
@@ -101,20 +101,21 @@
* @return The probability
* @see Model# featureWeight (String, String)
*/
- public double documentProbability(Model model, String label, String[]
document) {
+ public double documentWeight(Model model, String label, String[] document) {
double result = 0.0;
- Map<String, Integer> wordList = new HashMap<String, Integer>(1000);
+ Map<String, Integer[]> wordList = new HashMap<String, Integer[]>(1000);
for (String word : document) {
- if (wordList.containsKey(word)) {
- int count = wordList.get(word);
- wordList.put(word, count + 1);
- } else {
- wordList.put(word, 1);
+ Integer [] count = wordList.get(word);
+ if (count == null) {
+ count = new Integer[1];
+ count[0] = 0;
+ wordList.put(word, count);
}
+ count[0]++;
}
- for (Map.Entry<String, Integer> entry : wordList.entrySet()) {
+ for (Map.Entry<String, Integer[]> entry : wordList.entrySet()) {
String word = entry.getKey();
- int count = entry.getValue();
+ int count = entry.getValue()[0];
result += count * model.featureWeight(label, word);
}
return result;
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java?rev=712310&r1=712309&r2=712310&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java
Fri Nov 7 15:39:26 2008
@@ -48,7 +48,7 @@
PriorityQueue pq = new ClassifierResultPriorityQueue(numResults);
ClassifierResult tmp;
for (String category : categories){
- double prob = documentProbability(model, category, document);
+ double prob = documentWeight(model, category, document);
if (prob < 0.0) {
tmp = new ClassifierResult(category, prob);
pq.insert(tmp);
@@ -79,7 +79,7 @@
Collection<String> categories = model.getLabels();
for (String category : categories) {
- double prob = documentProbability(model, category, document);
+ double prob = documentWeight(model, category, document);
if (prob < min) {
min = prob;
result.setLabel(category);
@@ -90,7 +90,7 @@
}
/**
- * Calculate the document probability as the multiplication of the
+ * Calculate the document weight as the multiplication of the
* [EMAIL PROTECTED] Model#featureWeight(String, String)} for each word
given the label
*
* @param model The [EMAIL PROTECTED] org.apache.mahout.common.Model}
@@ -99,20 +99,21 @@
* @return The probability
* @see Model# featureWeight (String, String)
*/
- public double documentProbability(Model model, String label, String[]
document) {
+ public double documentWeight(Model model, String label, String[] document) {
double result = 0.0;
- Map<String, Integer> wordList = new HashMap<String, Integer>(1000);
+ Map<String, Integer[]> wordList = new HashMap<String, Integer[]>(1000);
for (String word : document) {
- if (wordList.containsKey(word)) {
- int count = wordList.get(word);
- wordList.put(word, count + 1);
- } else {
- wordList.put(word, 1);
- }
+ Integer [] count = wordList.get(word);
+ if (count == null) {
+ count = new Integer[1];
+ count[0] = 0;
+ wordList.put(word, count);
+ }
+ count[0]++;
}
- for (Map.Entry<String, Integer> entry : wordList.entrySet()) {
+ for (Map.Entry<String, Integer[]> entry : wordList.entrySet()) {
String word = entry.getKey();
- int count = entry.getValue();
+ int count = entry.getValue()[0];
result += count * model.featureWeight(label, word);
}
return result;
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/Classifier.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/Classifier.java?rev=712310&r1=712309&r2=712310&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/Classifier.java
(original)
+++
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/Classifier.java
Fri Nov 7 15:39:26 2008
@@ -59,7 +59,7 @@
* @return The probability
* @see Model#featureWeight (String, String)
*/
- public double documentProbability(Model model, String label, String[]
document);
+ public double documentWeight(Model model, String label, String[] document);
}
Modified:
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierTest.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierTest.java?rev=712310&r1=712309&r2=712310&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierTest.java
(original)
+++
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierTest.java
Fri Nov 7 15:39:26 2008
@@ -82,7 +82,7 @@
document = new String[]{"ff"};
result = classifier.classify(model, document, "unknown");
assertTrue("category is null and it shouldn't be", result != null);
- assertTrue(result + " is not equal to " + "unknown",
result.getLabel().equals("unknown"));
+ assertTrue(result + " is not equal to " + "d",
result.getLabel().equals("d"));//GSI: was unknown, but we now just pick the
first cat
document = new String[]{"cc"};
result = classifier.classify(model, document, "unknown");
Added:
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFeatureMapperTest.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFeatureMapperTest.java?rev=712310&view=auto
==============================================================================
---
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFeatureMapperTest.java
(added)
+++
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFeatureMapperTest.java
Fri Nov 7 15:39:26 2008
@@ -0,0 +1,44 @@
+package org.apache.mahout.classifier.bayes;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.io.DefaultStringifier;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.mahout.classifier.bayes.common.BayesFeatureMapper;
+import org.apache.mahout.utils.DummyOutputCollector;
+
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ *
+ *
+ **/
+public class BayesFeatureMapperTest extends TestCase {
+
+ public void test() throws Exception {
+ BayesFeatureMapper mapper = new BayesFeatureMapper();
+ JobConf conf = new JobConf();
+ conf.set("io.serializations",
+
"org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
+ DefaultStringifier<Integer> intStringifier = new
DefaultStringifier<Integer>(conf, Integer.class);
+ conf.set("bayes.gramSize", intStringifier.toString(3));
+ mapper.configure(conf);
+
+ DummyOutputCollector<Text, DoubleWritable> output = new
DummyOutputCollector<Text, DoubleWritable>();
+ mapper.map(new Text("foo"), new Text("big brown shoe"), output, null);
+ Map<String, List<DoubleWritable>> outMap = output.getData();
+ System.out.println("Map: " + outMap);
+ assertTrue("outMap is null and it shouldn't be", outMap != null);
+ //TODO: How about not such a lame test here?
+ for (Map.Entry<String, List<DoubleWritable>> entry : outMap.entrySet()) {
+ assertTrue("entry.getKey() Size: " + entry.getKey().length() + " is not
greater than: " + 0, entry.getKey().length() > 0);
+ assertTrue("entry.getValue() Size: " + entry.getValue().size() + " is
not: " + 1, entry.getValue().size() == 1);
+ assertTrue("value is not valie", entry.getValue().get(0).get() > 0);
+ }
+
+ }
+
+}
Modified:
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/CBayesClassifierTest.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/CBayesClassifierTest.java?rev=712310&r1=712309&r2=712310&view=diff
==============================================================================
---
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/CBayesClassifierTest.java
(original)
+++
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/CBayesClassifierTest.java
Fri Nov 7 15:39:26 2008
@@ -89,7 +89,7 @@
document = new String[]{"ff"};
result = classifier.classify(model, document, "unknown");
assertTrue("category is null and it shouldn't be", result != null);
- assertTrue(result + " is not equal to " + "unknown",
result.getLabel().equals("unknown"));
+ assertTrue(result + " is not equal to " + "d",
result.getLabel().equals("d"));
document = new String[]{"cc"};
result = classifier.classify(model, document, "unknown");
Modified:
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java?rev=712310&r1=712309&r2=712310&view=diff
==============================================================================
---
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
(original)
+++
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
Fri Nov 7 15:39:26 2008
@@ -168,7 +168,7 @@
File dir = new File(testDirPath);
File[] subdirs = dir.listFiles();
- ResultAnalyzer resultAnalyzer = new ResultAnalyzer(model.getLabels());
+ ResultAnalyzer resultAnalyzer = new ResultAnalyzer(model.getLabels(),
defaultCat);
if (subdirs != null) {
for (int loop = 0; loop < subdirs.length; loop++) {
Modified:
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java
URL:
http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java?rev=712310&r1=712309&r2=712310&view=diff
==============================================================================
---
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java
(original)
+++
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java
Fri Nov 7 15:39:26 2008
@@ -68,8 +68,6 @@
final DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
final ArgumentBuilder abuilder = new ArgumentBuilder();
final GroupBuilder gbuilder = new GroupBuilder();
- Option trainOpt = obuilder.withLongName("train").withRequired(true).
- withDescription("Train the
classifier").withShortName("t").create();
Option inputDirOpt =
obuilder.withLongName("inputDir").withRequired(true).withArgument(
abuilder.withName("inputDir").withMinimum(1).withMaximum(1).create()).
@@ -86,25 +84,21 @@
Option typeOpt =
obuilder.withLongName("classifierType").withRequired(true).withArgument(
abuilder.withName("classifierType").withMinimum(1).withMaximum(1).create()).
withDescription("Type of classifier: bayes or
cbayes").withShortName("type").create();
- Group group =
gbuilder.withName("Options").withOption(gramSizeOpt).withOption(inputDirOpt).withOption(outputOpt).withOption(trainOpt).withOption(typeOpt).create();
+ Group group =
gbuilder.withName("Options").withOption(gramSizeOpt).withOption(inputDirOpt).withOption(outputOpt).withOption(typeOpt).create();
CommandLine cmdLine;
Parser parser = new Parser();
parser.setGroup(group);
cmdLine = parser.parse(args);
- boolean train = cmdLine.hasOption(trainOpt);
TrainClassifier tn = new TrainClassifier();
- if (train) {
- String classifierType = (String) cmdLine.getValue(typeOpt);
- if (classifierType.equalsIgnoreCase("bayes")) {
- log.info("Training Bayes Classifier");
- tn.trainNaiveBayes((String)cmdLine.getValue(inputDirOpt),
(String)cmdLine.getValue(outputOpt), Integer.parseInt((String)
cmdLine.getValue(gramSizeOpt)));
-
- } else if (classifierType.equalsIgnoreCase("cbayes")) {
- log.info("Training Complementary Bayes Classifier");
- //setup the HDFS and copy the files there, then run the trainer
- tn.trainCNaiveBayes((String) cmdLine.getValue(inputDirOpt), (String)
cmdLine.getValue(outputOpt), Integer.parseInt((String)
cmdLine.getValue(gramSizeOpt)));
- }
+ String classifierType = (String) cmdLine.getValue(typeOpt);
+ if (classifierType.equalsIgnoreCase("bayes")) {
+ log.info("Training Bayes Classifier");
+ tn.trainNaiveBayes((String)cmdLine.getValue(inputDirOpt),
(String)cmdLine.getValue(outputOpt), Integer.parseInt((String)
cmdLine.getValue(gramSizeOpt)));
+
+ } else if (classifierType.equalsIgnoreCase("cbayes")) {
+ log.info("Training Complementary Bayes Classifier");
+ //setup the HDFS and copy the files there, then run the trainer
+ tn.trainCNaiveBayes((String) cmdLine.getValue(inputDirOpt), (String)
cmdLine.getValue(outputOpt), Integer.parseInt((String)
cmdLine.getValue(gramSizeOpt)));
}
-
}
}
