Revision: 16520
http://sourceforge.net/p/gate/code/16520
Author: nirajaswani
Date: 2013-02-08 15:58:48 +0000 (Fri, 08 Feb 2013)
Log Message:
-----------
added a parameter to ignore certain annotators
Modified Paths:
--------------
gate/trunk/plugins/Teamware_Tools/src/gate/qa/QAForTeamwarePR.java
Modified: gate/trunk/plugins/Teamware_Tools/src/gate/qa/QAForTeamwarePR.java
===================================================================
--- gate/trunk/plugins/Teamware_Tools/src/gate/qa/QAForTeamwarePR.java
2013-02-08 14:31:33 UTC (rev 16519)
+++ gate/trunk/plugins/Teamware_Tools/src/gate/qa/QAForTeamwarePR.java
2013-02-08 15:58:48 UTC (rev 16520)
@@ -21,7 +21,6 @@
import gate.creole.metadata.RunTime;
import gate.persist.PersistenceException;
import gate.security.SecurityException;
-import gate.util.GateException;
import gate.util.OffsetComparator;
import java.awt.Color;
@@ -43,26 +42,21 @@
import java.util.Set;
/**
- * When documents are annotated using Teamware, anonymous annotation
- * sets are created for annotating annotators. This makes it impossible
- * to run QA on such documents as same name annotation sets in different
- * documents may refer to the annoations created by different
- * annotators. This is especially the case when a requirement is to
- * compute IAA between individual annotators. This PR generates a
- * summary of agreements among annotators. It does this by pairing
+ * When documents are annotated using Teamware, anonymous annotation sets are
+ * created for annotating annotators. This makes it impossible to run QA on
such
+ * documents as same name annotation sets in different documents may refer to
+ * the annoations created by different annotators. This is especially the case
+ * when a requirement is to compute IAA between individual annotators. This PR
+ * generates a summary of agreements among annotators. It does this by pairing
* individual annotators. It also compares each individual annotator's
- * annotations with those available in the consensus annotation set in
- * the respective documents.
- *
+ * annotations with those available in the consensus annotation set in the
+ * respective documents.
+ *
* @author niraj
- *
*/
-@CreoleResource(name = "QA Summariser for Teamware",
- comment = "The Quality Assurance PR for teamware",
- helpURL = "http://gate.ac.uk/userguide/sec:eval:qaForTW";)
+@CreoleResource(name = "QA Summariser for Teamware", comment = "The Quality
Assurance PR for teamware", helpURL =
"http://gate.ac.uk/userguide/sec:eval:qaForTW";)
public class QAForTeamwarePR extends AbstractLanguageAnalyser implements
-
ProcessingResource {
-
+ ProcessingResource {
/**
* types of annotations to use
*/
@@ -94,8 +88,7 @@
private QualityAssurancePR assurancePR;
/**
- * used for keeping record of documents annotated by individual
- * annotators
+ * used for keeping record of documents annotated by individual annotators
*/
private Map<String, Set<Object>> annotatorToDocuments;
@@ -111,6 +104,11 @@
private Map<String, Map<String, Result>> results = null;
/**
+ * ignore annotators
+ */
+ private List<String> annotatorsToIgnore = null;
+
+ /**
* Controller with QA PR as part of it.
*/
private SerialAnalyserController controller;
@@ -129,18 +127,18 @@
public Resource init() throws ResourceInstantiationException {
f.setMaximumFractionDigits(2); // format used for all decimal values
f.setMinimumFractionDigits(2);
-
// using QualityAssurancePR internally to calculate QA stats
// but hiding this PR just in case
FeatureMap hideParams = Factory.newFeatureMap();
Gate.setHiddenAttribute(hideParams, true);
- assurancePR = (QualityAssurancePR)Factory.createResource(
- "gate.qa.QualityAssurancePR", Factory.newFeatureMap(), hideParams);
-
+ assurancePR =
+ (QualityAssurancePR)Factory.createResource("gate.qa.QualityAssurancePR",
+ Factory.newFeatureMap(), hideParams);
// we use controller to execute assurance PR
- controller = (SerialAnalyserController)Factory.createResource(
- SerialAnalyserController.class.getName(), Factory.newFeatureMap(),
- hideParams);
+ controller =
+ (SerialAnalyserController)Factory.createResource(
+ SerialAnalyserController.class.getName(), Factory.newFeatureMap(),
+ hideParams);
controller.add(assurancePR);
return this;
} // init()
@@ -149,23 +147,16 @@
* The execute method
*/
public void execute() throws ExecutionException {
-
// the corpus cannot be null or empty
- if(corpus == null || corpus.size() == 0) {
- throw new ExecutionException("Corpus cannot be null or empty");
- }
-
+ if(corpus == null || corpus.size() == 0) { throw new ExecutionException(
+ "Corpus cannot be null or empty"); }
// similarly user must provide annotation types that they want to
// compare
if(annotationTypes == null || annotationTypes.isEmpty())
throw new ExecutionException(
- "Please provide at least one annotation type to compare");
-
+ "Please provide at least one annotation type to compare");
// also a measure to use for computation
- if(measure == null) {
- throw new ExecutionException("No measure selected");
- }
-
+ if(measure == null) { throw new ExecutionException("No measure selected");
}
// check if we are processing the last document in the corpus
Document firstDocument = (Document)corpus.get(0);
if(firstDocument == document) {
@@ -174,11 +165,10 @@
results = new HashMap<String, Map<String, Result>>();
consensusExists = false;
}
-
// checking if consensus annotation set exists
- consensusExists = (consensusExists || document.getNamedAnnotationSets()
- .containsKey(CONSENSUS_AS_NAME));
-
+ consensusExists =
+ (consensusExists || document.getNamedAnnotationSets().containsKey(
+ CONSENSUS_AS_NAME));
// annotators found in this document
Set<String> annotators = new HashSet<String>();
for(Object featureName : document.getFeatures().keySet()) {
@@ -190,14 +180,19 @@
}
}
}
-
+
+ // we need to ignore some of the annotators
+ if(annotatorsToIgnore != null && !annotatorsToIgnore.isEmpty()) {
+ annotators.removeAll(annotatorsToIgnore);
+ }
+
// if no annotators found print a warning
if(annotators.isEmpty()) {
- System.err.println("No annotators found for the document "
- + document.getName() + "\n"
- + "Please make sure the document is annotated using Teamware!");
+ System.err.println("No annotators found for the document " +
+ document.getName() + "\n" +
+ "Please make sure the document is annotated using Teamware!");
}
-
+
// if documents are loaded from datastore, we store only the
// persistence Ids
// or the document object itself
@@ -206,7 +201,6 @@
if(persistenceId != null) {
toStore = persistenceId;
}
-
// which annotator annotated what documents
for(String annotatorName : annotators) {
Set<Object> docs = annotatorToDocuments.get(annotatorName);
@@ -216,7 +210,6 @@
}
docs.add(toStore);
}
-
// given annotator names, we need to find out possible pairings
List<String> sortedAnnNames = new ArrayList<String>(annotators);
Collections.sort(sortedAnnNames);
@@ -233,74 +226,58 @@
docs.add(toStore);
}
}
-
// check if we are processing the last document in the corpus
Document lastDocument = (Document)corpus.get(corpus.size() - 1);
- if(lastDocument != document) {
- return;
- }
-
+ if(lastDocument != document) { return; }
// if documents are being loaded from a datastore, it should be
// deleted
// after it has been used
boolean deleteDocs = false;
-
// first iteration
// this is for obtaining IAA between each individual annotator and
// concensus
// annotation sets
for(String annName : annotatorToDocuments.keySet()) {
-
// documents annotated by the current annotator
Set<Object> docs = annotatorToDocuments.get(annName);
-
Corpus corpus;
try {
corpus = Factory.newCorpus("qaCorpus");
- }
- catch(ResourceInstantiationException e1) {
+ } catch(ResourceInstantiationException e1) {
throw new ExecutionException(e1);
}
-
// one doc at a time
for(Object aDoc : docs) {
-
// load from ds if not in memory already
Document gateDoc = null;
if(!(aDoc instanceof Document)) {
try {
- gateDoc = (Document)getCorpus().getDataStore().getLr(
- DocumentImpl.class.getName(), aDoc);
+ gateDoc =
+ (Document)getCorpus().getDataStore().getLr(
+ DocumentImpl.class.getName(), aDoc);
deleteDocs = true;
- }
- catch(PersistenceException e) {
+ } catch(PersistenceException e) {
throw new ExecutionException(e);
- }
- catch(SecurityException e) {
+ } catch(SecurityException e) {
throw new ExecutionException(e);
}
- }
- else {
+ } else {
gateDoc = (Document)aDoc;
}
-
// creating temporary annotation set with annotator's name
// across all documents in the corpus and copying annotations
// produced
// by him in that document into his annotationset
createAnnSet(gateDoc, getSetName(gateDoc, annName), annName);
-
// add document to the corpus
// doc will remain in memory until all computations are done
corpus.add(gateDoc);
}
-
// calculating IAA stats between annotator's annotations and
// consensus annotation set
if(consensusExists) {
calculateIAA(corpus, annName, CONSENSUS_AS_NAME, annName);
}
-
// deleting newly created annotation set and unloading documents
// if necessary
for(int k = corpus.size() - 1; k >= 0; k--) {
@@ -311,136 +288,110 @@
}
aDoc.removeAnnotationSet(annName);
}
-
// delete the corpus as well
Factory.deleteResource(corpus);
}
-
// preparing for second iteration
deleteDocs = false;
-
// second iteration
// this is for obtaining IAA between individual annotators
for(String annotatorsPair : annotatorsPairToDocuments.keySet()) {
-
// documents annotated by this pair of annotators
Set<Object> docs = annotatorsPairToDocuments.get(annotatorsPair);
Corpus corpus;
try {
corpus = Factory.newCorpus("qaCorpus");
- }
- catch(ResourceInstantiationException e1) {
+ } catch(ResourceInstantiationException e1) {
throw new ExecutionException(e1);
}
-
// annotators in this pair
- String annotator1 = annotatorsPair.substring(0, annotatorsPair
- .indexOf(';'));
- String annotator2 = annotatorsPair
- .substring(annotatorsPair.indexOf(';') + 1);
-
+ String annotator1 =
+ annotatorsPair.substring(0, annotatorsPair.indexOf(';'));
+ String annotator2 =
+ annotatorsPair.substring(annotatorsPair.indexOf(';') + 1);
// one doc at a time
for(Object aDoc : docs) {
-
// load the doc from DS if needed
Document gateDoc = null;
if(!(aDoc instanceof Document)) {
try {
- gateDoc = (Document)getCorpus().getDataStore().getLr(
- DocumentImpl.class.getName(), aDoc);
+ gateDoc =
+ (Document)getCorpus().getDataStore().getLr(
+ DocumentImpl.class.getName(), aDoc);
deleteDocs = true;
- }
- catch(PersistenceException e) {
+ } catch(PersistenceException e) {
throw new ExecutionException(e);
- }
- catch(SecurityException e) {
+ } catch(SecurityException e) {
throw new ExecutionException(e);
}
- }
- else {
+ } else {
gateDoc = (Document)aDoc;
}
-
// creating sets for individual annotators and copying
// annotations from
// their anonymous annotation sets into their own annotation set
createAnnSet(gateDoc, getSetName(gateDoc, annotator1), annotator1);
createAnnSet(gateDoc, getSetName(gateDoc, annotator2), annotator2);
-
// adding doc to the corpus
corpus.add(gateDoc);
}
-
// calculate IAA
- calculateIAA(corpus, annotator1, annotator2, annotator1 + "-"
- + annotator2);
-
+ calculateIAA(corpus, annotator1, annotator2, annotator1 + "-" +
+ annotator2);
// deleting temporarily created annotation sets and deleting docs
// from
// memory if docs were loaded from the DS
for(int k = corpus.size() - 1; k >= 0; k--) {
Document aDoc = (Document)corpus.get(k);
-
if(deleteDocs) {
Factory.deleteResource(aDoc);
continue;
}
-
aDoc.removeAnnotationSet(annotator1);
aDoc.removeAnnotationSet(annotator2);
}
-
// delete the corpus
Factory.deleteResource(corpus);
}
-
// generating summary
// only accept the files with document-stats at the end
- File[] resultFiles = new File(outputFolderUrl.getFile())
- .listFiles(new FileFilter() {
- public boolean accept(File pathname) {
- return pathname.getAbsolutePath().endsWith(
- "-document-stats.html");
- }
- });
-
+ File[] resultFiles =
+ new File(outputFolderUrl.getFile()).listFiles(new FileFilter() {
+ public boolean accept(File pathname) {
+ return pathname.getAbsolutePath().endsWith("-document-stats.html");
+ }
+ });
// authors found in the corpus
List<String> columnAuthorNames = new ArrayList<String>();
double consensusMacro = 0.0D;
double consensusMicro = 0.0D;
double annotatorMacro = 0.0D;
double annotatorMicro = 0.0D;
-
// one file at a time
for(File file : resultFiles) {
-
// finding author names
- String fileName = file.getName().substring(0,
- file.getName().indexOf("-document-stats.html"));
+ String fileName =
+ file.getName().substring(0,
+ file.getName().indexOf("-document-stats.html"));
int index = fileName.indexOf('-');
String author1 = null;
String author2 = null;
-
// if only one author found, it means this file is for author vs
// consensus
// IAA
if(index < 0) {
author1 = fileName;
author2 = CONSENSUS_AS_NAME;
- }
- else {
+ } else {
author1 = fileName.substring(0, index);
author2 = fileName.substring(index + 1);
}
-
if(!columnAuthorNames.contains(author2)) {
columnAuthorNames.add(author2);
}
-
if(!columnAuthorNames.contains(author1)) {
columnAuthorNames.add(author1);
}
-
// loading the document-stats file as GATE document
// and utilizing original markups to collect the needed
// information
@@ -448,47 +399,39 @@
Document aDoc = null;
try {
aDoc = Factory.newDocument(file.toURI().toURL());
- }
- catch(ResourceInstantiationException e) {
+ } catch(ResourceInstantiationException e) {
throw new ExecutionException(e);
- }
- catch(MalformedURLException e) {
+ } catch(MalformedURLException e) {
throw new ExecutionException(e);
}
-
// we're interested in macro and micro averages figures only
// these are the last two rows in the document-stats
AnnotationSet omSet = aDoc.getAnnotations("Original markups");
List<Annotation> rows = new ArrayList<Annotation>(omSet.get("tr"));
Collections.sort(rows, new OffsetComparator());
-
// temporary result object
Result r = new Result();
r.documentFileName = file.getName();
-
// consider last row for micro summary
Annotation row = rows.get(rows.size() - 1);
- List<Annotation> cols = new ArrayList<Annotation>(omSet.getContained(
- Utils.start(row), Utils.end(row)).get("td"));
+ List<Annotation> cols =
+ new ArrayList<Annotation>(omSet.getContained(Utils.start(row),
+ Utils.end(row)).get("td"));
Collections.sort(cols, new OffsetComparator());
-
// only interested in the last column
- r.micro = Double.parseDouble(Utils.stringFor(aDoc, cols
- .get(cols.size() - 1)));
-
+ r.micro =
+ Double.parseDouble(Utils.stringFor(aDoc, cols.get(cols.size() - 1)));
// consider second last row for macro summary
row = rows.get(rows.size() - 2);
- cols = new ArrayList<Annotation>(omSet.getContained(Utils.start(row),
- Utils.end(row)).get("td"));
+ cols =
+ new ArrayList<Annotation>(omSet.getContained(Utils.start(row),
+ Utils.end(row)).get("td"));
Collections.sort(cols, new OffsetComparator());
-
// only interested in the last column
- r.macro = Double.parseDouble(Utils.stringFor(aDoc, cols
- .get(cols.size() - 1)));
-
+ r.macro =
+ Double.parseDouble(Utils.stringFor(aDoc, cols.get(cols.size() - 1)));
// delete the document from GATe
Factory.deleteResource(aDoc);
-
// making two entries
// i.e. author1-author2 and author2-author1
Map<String, Result> authorResults = results.get(author1);
@@ -496,21 +439,16 @@
authorResults = new HashMap<String, Result>();
results.put(author1, authorResults);
}
-
authorResults.put(author2, r);
-
authorResults = results.get(author2);
if(authorResults == null) {
authorResults = new HashMap<String, Result>();
results.put(author2, authorResults);
}
-
authorResults.put(author1, r);
}
-
// collected all the results
Collections.sort(columnAuthorNames);
-
// generating html file contents
StringBuffer buffer = new StringBuffer();
buffer.append("<html>\n<title>Summary of IAA Results</title>\n<body>\n");
@@ -529,7 +467,6 @@
buffer.append("<br>");
buffer.append("<b>Measure:</b> " + measure.toString() + "<br>");
buffer.append("<table border=\"1\">\n");
-
// first row
buffer.append("\t<tr>\n");
buffer.append("\t\t<td><b>Author Names</b></td>\n");
@@ -538,216 +475,166 @@
}
buffer.append("\t\t<td colspan=\"2\"><b> Averages </b></td>\n");
buffer.append("\t</tr>\n");
-
// second row
buffer.append("\t<tr>\n");
buffer.append("\t\t<td> </td>\n");
-
// additional columns for averages
for(int i = 0; i <= columnAuthorNames.size(); i++) {
buffer.append("\t\t<td><b>Macro</b></td>\n");
buffer.append("\t\t<td><b>Micro</b></td>\n");
}
buffer.append("\t</tr>\n");
-
// color coding with transperancy
// 1.0 = dark green
// ... as we proceed we lighten the green color
// 0.5 = white
// ... as we proceed we brighten the red color
// 0.0 = red
-
List<String> authorNamesList = new ArrayList<String>(results.keySet());
Collections.sort(authorNamesList);
-
// all rows onwards
// producing matrix
for(String author1 : authorNamesList) {
double annMacro = 0.0D;
double annMicro = 0.0D;
int docs = 0;
-
buffer.append("\t<tr>\n");
buffer.append("\t\t<td><b>" + author1 + "</b></td>\n");
-
Map<String, Result> resultsForAuthor1 = results.get(author1);
-
// columns
for(String author2 : columnAuthorNames) {
Result r = resultsForAuthor1.get(author2);
if(r == null) {
buffer.append("\t\t<td colspan=\"2\"> </td>\n");
- }
- else {
-
+ } else {
// append macro micro figures to the table
appendMacroMicroFigures(buffer, r.macro, r.micro);
-
if(author1.equals(CONSENSUS_AS_NAME)) {
consensusMacro += r.macro;
consensusMicro += r.micro;
- }
- else {
+ } else {
annMacro += r.macro;
annMicro += r.micro;
docs++;
}
}
}
-
// adding averages to the last 2 columns
if(author1.equals(CONSENSUS_AS_NAME)) {
- double caMacroAvg = (double)(consensusMacro /
(double)(columnAuthorNames
- .size() - 1));
- double caMicroAvg = (double)(consensusMicro /
(double)(columnAuthorNames
- .size() - 1));
-
+ double caMacroAvg =
+ (double)(consensusMacro / (double)(columnAuthorNames.size() - 1));
+ double caMicroAvg =
+ (double)(consensusMicro / (double)(columnAuthorNames.size() - 1));
appendMacroMicroFigures(buffer, caMacroAvg, caMicroAvg);
- }
- else {
+ } else {
double aaMacroAvg = (double)(annMacro / (double)(docs));
double aaMicroAvg = (double)(annMicro / (double)(docs));
-
annotatorMacro += aaMacroAvg;
annotatorMicro += aaMicroAvg;
-
appendMacroMicroFigures(buffer, aaMacroAvg, aaMicroAvg);
}
-
buffer.append("\t</tr>\n");
-
buffer.append("\t<tr>\n");
buffer.append("\t\t<td> </td>\n");
-
for(String author2 : columnAuthorNames) {
Result r = resultsForAuthor1.get(author2);
if(r == null) {
buffer.append("\t\t<td colspan=\"2\"> </td>\n");
+ } else {
+ buffer.append("\t\t<td colspan=\"2\"><a target=\"_blank\" href=\"" +
+ r.documentFileName + "\">document</a></td>\n");
}
- else {
- buffer.append("\t\t<td colspan=\"2\"><a target=\"_blank\" href=\""
- + r.documentFileName + "\">document</a></td>\n");
- }
}
-
buffer.append("\t\t<td colspan=\"2\"> </td>\n");
}
-
buffer.append("</table>\n");
-
buffer.append("<hr>");
-
if(consensusExists) {
- buffer.append("<br><b>Avg. "
- + CONSENSUS_AS_NAME
- + " macro avg:</b> "
- + (f.format((double)consensusMacro
- / (columnAuthorNames.size() - 1))));
- buffer.append("<br><b>Avg. "
- + CONSENSUS_AS_NAME
- + " micro avg:</b> "
- + (f.format((double)consensusMicro
- / (columnAuthorNames.size() - 1))));
+ buffer.append("<br><b>Avg. " + CONSENSUS_AS_NAME + " macro avg:</b> " +
+ (f.format((double)consensusMacro / (columnAuthorNames.size() - 1))));
+ buffer.append("<br><b>Avg. " + CONSENSUS_AS_NAME + " micro avg:</b> " +
+ (f.format((double)consensusMicro / (columnAuthorNames.size() - 1))));
}
-
// if consensus annotation set exists, make sure the number used in
// demoninator does not include consensus as one annotator
- int totalAuthors = consensusExists ?
- columnAuthorNames.size() - 1 : columnAuthorNames.size();
-
- buffer.append("<br><b>Avg. IAA macro avg:</b> "
- + (f
- .format((double)annotatorMacro
- / totalAuthors)));
- buffer.append("<br><b>Avg. IAA micro avg:</b> "
- + (f
- .format((double)annotatorMicro
- / totalAuthors)));
+ int totalAuthors =
+ consensusExists ? columnAuthorNames.size() - 1 :
columnAuthorNames.size();
+ buffer.append("<br><b>Avg. IAA macro avg:</b> " +
+ (f.format((double)annotatorMacro / totalAuthors)));
+ buffer.append("<br><b>Avg. IAA micro avg:</b> " +
+ (f.format((double)annotatorMicro / totalAuthors)));
buffer.append("</body>\n</html>");
-
BufferedWriter bw = null;
try {
File indexFile = new File(outputFolderUrl.getFile(), "index.html");
bw = new BufferedWriter(new FileWriter(indexFile));
bw.write(buffer.toString());
- }
- catch(IOException ioe) {
+ } catch(IOException ioe) {
throw new ExecutionException(ioe);
- }
- finally {
+ } finally {
if(bw != null) {
try {
bw.close();
- }
- catch(IOException e) {
+ } catch(IOException e) {
throw new ExecutionException(e);
}
}
}
-
}
/**
- * A method that adds two columns to the buffer - one for the macro
- * figure and the other one for the micro figure. It also adds
- * relevant style tag to give a proper color to each cell depending on
- * the value of macro and micro figures. See documentation of the
- * getStyleTag(double) for more information on how cell backgrounds
- * are color coded.
- *
+ * A method that adds two columns to the buffer - one for the macro figure
and
+ * the other one for the micro figure. It also adds relevant style tag to
give
+ * a proper color to each cell depending on the value of macro and micro
+ * figures. See documentation of the getStyleTag(double) for more information
+ * on how cell backgrounds are color coded.
+ *
* @param buffer
* @param macro
* @param micro
*/
private void appendMacroMicroFigures(StringBuffer buffer, double macro,
- double micro) {
+ double micro) {
String macCol = getStyleTag(macro);
String micCol = getStyleTag(micro);
-
buffer.append("\t\t<td " + macCol + " >" + f.format(macro) + "</td>\n");
buffer.append("\t\t<td " + micCol + " >" + f.format(micro) + "</td>\n");
-
}
/**
- * Gets the style tag based on the score.
- *
- * The color green is used for a cell background to indicate full
- * agreement (i.e. 1.0). The background color becomes lighter as the
- * agreement reduces towards 0.5. At 0.5 agreement, the background
- * color of a cell is fully white. From 0.5 downwards, the color red
- * is used and as the agreement reduces further, the color becomes
- * darker with dark red at 0.0 agreement.
- *
+ * Gets the style tag based on the score. The color green is used for a cell
+ * background to indicate full agreement (i.e. 1.0). The background color
+ * becomes lighter as the agreement reduces towards 0.5. At 0.5 agreement,
the
+ * background color of a cell is fully white. From 0.5 downwards, the color
+ * red is used and as the agreement reduces further, the color becomes darker
+ * with dark red at 0.0 agreement.
+ *
* @param score
* @return
*/
private String getStyleTag(double score) {
-
// two colors
Color gc = Color.GREEN;
Color rc = Color.RED;
-
Color c = null;
-
// if score is above .50, use the green color, otherwise the red one
if(score > 0.50) {
- c = new Color(gc.getRed(), gc.getGreen(), gc.getBlue(),
- ((int)(score * 100) * 2) - 100);
+ c =
+ new Color(gc.getRed(), gc.getGreen(), gc.getBlue(),
+ ((int)(score * 100) * 2) - 100);
+ } else {
+ c =
+ new Color(rc.getRed(), rc.getGreen(), rc.getBlue(),
+ 100 - ((int)(score * 100) * 2));
}
- else {
- c = new Color(rc.getRed(), rc.getGreen(), rc.getBlue(),
- 100 - ((int)(score * 100) * 2));
- }
-
- return "style=\"background-color: rgba(" + c.getRed() + "," + c.getGreen()
- + "," + c.getBlue() + "," + ((double)c.getAlpha() / 100) + ")\"";
+ return "style=\"background-color: rgba(" + c.getRed() + "," + c.getGreen()
+
+ "," + c.getBlue() + "," + ((double)c.getAlpha() / 100) + ")\"";
}
/**
- * Given the annotator name, the method finds out the annotation set
- * which contains annotations for that annotator
- *
+ * Given the annotator name, the method finds out the annotation set which
+ * contains annotations for that annotator
+ *
* @param doc
* @param annotatorName
* @return
@@ -766,15 +653,16 @@
/**
* Creating a temporary annotation set
- *
+ *
* @param doc
* @param inputAS
* @param outputAS
*/
private void createAnnSet(Document doc, String inputAS, String outputAS) {
-
- AnnotationSet inAS = inputAS == null || inputAS.trim().length() == 0 ? doc
- .getAnnotations() : doc.getAnnotations(inputAS);
+ AnnotationSet inAS =
+ inputAS == null || inputAS.trim().length() == 0
+ ? doc.getAnnotations()
+ : doc.getAnnotations(inputAS);
inAS = inAS.get(new HashSet<String>(annotationTypes));
AnnotationSet outAS = doc.getAnnotations(outputAS);
for(Annotation a : inAS) {
@@ -784,7 +672,7 @@
/**
* A method that calculates IAA between the given annotation sets
- *
+ *
* @param corpus
* @param keyAS
* @param responseAS
@@ -792,39 +680,33 @@
* @throws ExecutionException
*/
private void calculateIAA(Corpus corpus, String keyAS, String responseAS,
- String filePrefix) throws ExecutionException {
+ String filePrefix) throws ExecutionException {
try {
assurancePR.reInit();
- }
- catch(ResourceInstantiationException rie) {
+ } catch(ResourceInstantiationException rie) {
throw new ExecutionException(rie);
}
-
// lets set the params on qualityAssurancePR
assurancePR.setAnnotationTypes(annotationTypes);
assurancePR.setFeatureNames(featureNames == null
- ? new ArrayList<String>()
- : featureNames);
+ ? new ArrayList<String>()
+ : featureNames);
assurancePR.setMeasure(measure);
assurancePR.setOutputFolderUrl(outputFolderUrl);
-
// the remaining two params will be set later
assurancePR.setKeyASName(keyAS);
assurancePR.setResponseASName(responseAS);
-
controller.setCorpus(corpus);
controller.execute();
-
// QA PR produces two stats file, one for corpus and one for
// documents
// for QAForTeamware, we don't need corpus one
File corpusFile = new File(outputFolderUrl.getFile(), "corpus-stats.html");
corpusFile.delete();
-
- File documentFile = new File(outputFolderUrl.getFile(),
- "document-stats.html");
- File documentOutFile = new File(outputFolderUrl.getFile(), filePrefix
- + "-document-stats.html");
+ File documentFile =
+ new File(outputFolderUrl.getFile(), "document-stats.html");
+ File documentOutFile =
+ new File(outputFolderUrl.getFile(), filePrefix + "-document-stats.html");
documentFile.renameTo(documentOutFile);
}
@@ -839,7 +721,7 @@
/**
* Annotation types for which the stats should be calculated
- *
+ *
* @return
*/
public List<String> getAnnotationTypes() {
@@ -848,7 +730,7 @@
/**
* Annotation types for which the stats should be calculated
- *
+ *
* @param annotationTypes
*/
@RunTime
@@ -859,7 +741,7 @@
/**
* Features names for which the stats should be calculated
- *
+ *
* @return
*/
public List<String> getFeatureNames() {
@@ -868,7 +750,7 @@
/**
* Features names for which the stats should be calculated
- *
+ *
* @param featureNames
*/
@RunTime
@@ -880,7 +762,7 @@
/**
* Measure to use for stats calculation
- *
+ *
* @return
*/
public Measure getMeasure() {
@@ -889,7 +771,7 @@
/**
* Measure to use for stats calculation
- *
+ *
* @param measure
*/
@RunTime
@@ -900,7 +782,7 @@
/**
* URL of the folder to store output files into
- *
+ *
* @return
*/
public URL getOutputFolderUrl() {
@@ -909,7 +791,7 @@
/**
* URL of the folder to store output files into
- *
+ *
* @param outputFolderUrl
*/
@RunTime
@@ -919,8 +801,28 @@
}
/**
+ * annotators to ignore when computing IAA
+ * @return
+ */
+ public List<String> getAnnotatorsToIgnore() {
+ return annotatorsToIgnore;
+ }
+
+ /**
+ * annotators to ingore when computing IAA
+ * @param annotatorsToIgnore
+ */
+ @RunTime
+ @Optional
+ @CreoleParameter
+ public void setAnnotatorsToIgnore(List<String> annotatorsToIgnore) {
+ this.annotatorsToIgnore = annotatorsToIgnore;
+ }
+
+
+ /**
* Storing individual results for each pair of annotators
- *
+ *
* @author niraj
*/
class Result {
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Free Next-Gen Firewall Hardware Offer
Buy your Sophos next-gen firewall before the end March 2013
and get the hardware for free! Learn more.
http://p.sf.net/sfu/sophos-d2d-feb
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
