Author: colen
Date: Wed Jan 5 16:44:58 2011
New Revision: 1055519
URL: http://svn.apache.org/viewvc?rev=1055519&view=rev
Log:
OPENNLP-59 New strategy to compute Precision, Recall and FM
Modified:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
Modified:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
URL:
http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java?rev=1055519&r1=1055518&r2=1055519&view=diff
==============================================================================
---
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
(original)
+++
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
Wed Jan 5 16:44:58 2011
@@ -29,15 +29,13 @@ package opennlp.tools.util.eval;
*/
public final class FMeasure {
- /**
- * The mean of all calculated precision scores.
- */
- private Mean precisionScore = new Mean();
-
- /**
- * The mean of all calculated recall scores.
- */
- private Mean recallScore = new Mean();
+ /** |selected| = tp + fp */
+ private long selected;
+
+ /** |target| = tp + fp */
+ private long target;
+
+ private long truePositive;
/**
* Retrieves the arithmetic mean of the precision scores
@@ -46,7 +44,7 @@ public final class FMeasure {
* @return the arithmetic mean of all precision scores
*/
public double getPrecisionScore() {
- return precisionScore.mean();
+ return selected> 0 ? (double)truePositive / (double)selected : 0;
}
/**
@@ -56,7 +54,7 @@ public final class FMeasure {
* @return the arithmetic mean of all recall scores
*/
public double getRecallScore() {
- return recallScore.mean();
+ return target> 0 ? (double)truePositive / (double)target : 0;
}
/**
@@ -79,19 +77,16 @@ public final class FMeasure {
}
public void updateScores(Object references[], Object predictions[]) {
-
- double precision = FMeasure.precision(references, predictions);
- if (!Double.isNaN(precision))
- precisionScore.add(precision, references.length);
-
- double recall = FMeasure.recall(references, predictions);
- if (!Double.isNaN(recall))
- recallScore.add(FMeasure.recall(references, predictions),
references.length);
+
+ truePositive += countTruePositives(references, predictions);
+ selected += predictions.length;
+ target += references.length;
}
public void mergeInto(FMeasure measure) {
- precisionScore.add(measure.getPrecisionScore(),
measure.precisionScore.count());
- recallScore.add(measure.getRecallScore(), measure.recallScore.count());
+ this.selected += measure.selected;
+ this.target += measure.target;
+ this.truePositive += measure.truePositive;
}
/**
Modified:
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
URL:
http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java?rev=1055519&r1=1055518&r2=1055519&view=diff
==============================================================================
---
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
(original)
+++
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
Wed Jan 5 16:44:58 2011
@@ -53,6 +53,27 @@ public class FMeasureTest {
new Span(212, 220),
new Span(220, 230)
};
+
+ private Span goldToMerge[] = {
+ new Span(8, 9),
+ new Span(9, 10),
+ new Span(11, 11),
+ new Span(13, 14),
+ new Span(14, 15),
+ new Span(15, 16),
+ new Span(18, 19),
+ };
+
+ private Span predictedToMerge[] = {
+ new Span(8, 9),
+ new Span(14, 15),
+ new Span(15, 16),
+ new Span(100, 120),
+ new Span(210, 220),
+ new Span(220, 230)
+ };
+
+
/**
* Test for the {...@link EvaluatorUtil#countTruePositives(Span[], Span[])}
method.
@@ -88,4 +109,49 @@ public class FMeasureTest {
assertEquals(Double.NaN, FMeasure.recall(new Object[]{}, gold), DELTA);
assertEquals(2d / gold.length, FMeasure.recall(gold, predicted), DELTA);
}
+
+ @Test
+ public void testEmpty() {
+ FMeasure fm = new FMeasure();
+ assertEquals(-1, fm.getFMeasure(), DELTA);
+ assertEquals(0, fm.getRecallScore(), DELTA);
+ assertEquals(0, fm.getPrecisionScore(), DELTA);
+ }
+
+ @Test
+ public void testPerfect() {
+ FMeasure fm = new FMeasure();
+ fm.updateScores(gold, gold);
+ assertEquals(1, fm.getFMeasure(), DELTA);
+ assertEquals(1, fm.getRecallScore(), DELTA);
+ assertEquals(1, fm.getPrecisionScore(), DELTA);
+ }
+
+ @Test
+ public void testMerge() {
+ FMeasure fm = new FMeasure();
+ fm.updateScores(gold, predicted);
+ fm.updateScores(goldToMerge, predictedToMerge);
+
+ FMeasure fmMerge = new FMeasure();
+ fmMerge.updateScores(gold, predicted);
+ FMeasure toMerge = new FMeasure();
+ toMerge.updateScores(goldToMerge, predictedToMerge);
+ fmMerge.mergeInto(toMerge);
+
+ double selected1 = predicted.length;
+ double target1 = gold.length;
+ double tp1 = FMeasure.countTruePositives(gold, predicted);
+
+ double selected2 = predictedToMerge.length;
+ double target2 = goldToMerge.length;
+ double tp2 = FMeasure.countTruePositives(goldToMerge,
predictedToMerge);
+
+
+ assertEquals((tp1 + tp2) / (target1 + target2), fm.getRecallScore(),
DELTA);
+ assertEquals((tp1 + tp2) / (selected1 + selected2),
fm.getPrecisionScore(), DELTA);
+
+ assertEquals(fm.getRecallScore(), fmMerge.getRecallScore(), DELTA);
+ assertEquals(fm.getPrecisionScore(), fmMerge.getPrecisionScore(),
DELTA);
+ }
}
\ No newline at end of file
