Revision: 17225
http://sourceforge.net/p/gate/code/17225
Author: valyt
Date: 2014-01-10 16:36:03 +0000 (Fri, 10 Jan 2014)
Log Message:
-----------
Implemented indexing of annotations in the new 5.0 live-indexing framework.
The indexes get built, but we're currently getting an AbstractMethodError when
closing the mention indexes. This sounds like a dereliction of duty on the part
of the compiler, but its something to look into some other time, whilst in
possession of a clear head.
Changed all ANT build files to target Java 7.
Modified Paths:
--------------
mimir/branches/5.0/mimir-client/build.xml
mimir/branches/5.0/mimir-core/build.xml
mimir/branches/5.0/mimir-core/src/gate/mimir/AbstractSemanticAnnotationHelper.java
mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
mimir/branches/5.0/mimir-core/src/gate/mimir/SemanticAnnotationHelper.java
mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicIndex.java
mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicTokenIndex.java
mimir/branches/5.0/mimir-core/src/gate/mimir/index/Indexer.java
mimir/branches/5.0/mimir-core/src/gate/mimir/index/mg4j/MentionsIndexBuilder.java
mimir/branches/5.0/mimir-core/src/gate/mimir/index/mg4j/zipcollection/DocumentCollectionWriter.java
mimir/branches/5.0/mimir-core/src/gate/mimir/util/DelegatingSemanticAnnotationHelper.java
mimir/branches/5.0/mimir-test/src/gate/mimir/test/Scratch.java
mimir/branches/5.0/plugins/db-h2/build.xml
mimir/branches/5.0/plugins/db-h2/src/gate/mimir/db/DBSemanticAnnotationHelper.java
mimir/branches/5.0/plugins/measurements/build.xml
mimir/branches/5.0/plugins/measurements/src/gate/mimir/measurements/MeasurementAnnotationHelper.java
mimir/branches/5.0/plugins/sesame/build.xml
mimir/branches/5.0/plugins/sesame/src/gate/mimir/sesame/SesameSemanticAnnotationHelper.java
mimir/branches/5.0/plugins/sparql/build.xml
mimir/branches/5.0/plugins/sparql/src/gate/mimir/sparql/SPARQLSemanticAnnotationHelper.java
Added Paths:
-----------
mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicMentionsIndex.java
Modified: mimir/branches/5.0/mimir-client/build.xml
===================================================================
--- mimir/branches/5.0/mimir-client/build.xml 2014-01-08 17:32:10 UTC (rev
17224)
+++ mimir/branches/5.0/mimir-client/build.xml 2014-01-10 16:36:03 UTC (rev
17225)
@@ -56,8 +56,8 @@
<javac destdir="${classes.dir}"
srcdir="${src.dir}"
debug="true"
- target="1.6"
- source="1.6"
+ target="1.7"
+ source="1.7"
encoding="UTF-8"
classpathref="compile.classpath" />
</target>
@@ -86,7 +86,7 @@
Windowtitle="${app.title} (${app.version}) JavaDoc"
docencoding="UTF-8"
charset="UTF-8"
- source="1.6"
+ source="1.7"
useexternalfile="yes"
breakiterator="true">
<fileset dir="${src.dir}" includes="**/*.java"/>
Modified: mimir/branches/5.0/mimir-core/build.xml
===================================================================
--- mimir/branches/5.0/mimir-core/build.xml 2014-01-08 17:32:10 UTC (rev
17224)
+++ mimir/branches/5.0/mimir-core/build.xml 2014-01-10 16:36:03 UTC (rev
17225)
@@ -53,8 +53,8 @@
<javac destdir="${classes.dir}"
srcdir="${src.dir}"
debug="true"
- target="1.6"
- source="1.6"
+ target="1.7"
+ source="1.7"
encoding="UTF-8"
classpathref="compile.classpath" />
</target>
@@ -84,7 +84,7 @@
Windowtitle="${app.title} (${app.version}) JavaDoc"
docencoding="UTF-8"
charset="UTF-8"
- source="1.6"
+ source="1.7"
useexternalfile="yes"
breakiterator="true">
<fileset dir="${src.dir}" includes="**/*.java"/>
Modified:
mimir/branches/5.0/mimir-core/src/gate/mimir/AbstractSemanticAnnotationHelper.java
===================================================================
---
mimir/branches/5.0/mimir-core/src/gate/mimir/AbstractSemanticAnnotationHelper.java
2014-01-08 17:32:10 UTC (rev 17224)
+++
mimir/branches/5.0/mimir-core/src/gate/mimir/AbstractSemanticAnnotationHelper.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -15,6 +15,7 @@
package gate.mimir;
import gate.Document;
+import gate.mimir.index.AtomicMentionsIndex;
import gate.mimir.index.Indexer;
import gate.mimir.index.Mention;
import gate.mimir.search.QueryEngine;
@@ -314,7 +315,7 @@
@Override
- public void init(Indexer indexer) {
+ public void init(AtomicMentionsIndex index) {
checkInit();
}
Modified: mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
2014-01-08 17:32:10 UTC (rev 17224)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -14,9 +14,12 @@
*/
package gate.mimir;
+import gate.Document;
import gate.Gate;
+import gate.mimir.IndexConfig.SemanticIndexerConfig;
import gate.mimir.IndexConfig.TokenIndexerConfig;
import gate.mimir.index.AtomicIndex;
+import gate.mimir.index.AtomicMentionsIndex;
import gate.mimir.index.AtomicTokenIndex;
import gate.mimir.index.IndexException;
import gate.mimir.index.mg4j.GATEDocument;
@@ -178,21 +181,31 @@
TokenIndexerConfig tokConfs[] = indexConfig.getTokenIndexers();
tokenIndexes = new AtomicTokenIndex[tokConfs.length];
for(int i = 0; i < tokConfs.length; i++) {
- TokenIndexerConfig tokConf = tokConfs[i];
String subIndexname = "token-" + i;
tokenIndexes[i] = new AtomicTokenIndex(
this,
subIndexname,
new File(indexDirectory, subIndexname),
- tokConf.isDirectIndexEnabled(),
+ tokConfs[i].isDirectIndexEnabled(),
new LinkedBlockingQueue<GATEDocument>(),
new LinkedBlockingQueue<GATEDocument>(),
- tokConf,
+ tokConfs[i],
i == 0);
}
- //TODO
- mentionIndexes = new AtomicIndex[0];
+ SemanticIndexerConfig sics[] = indexConfig.getSemanticIndexers();
+ mentionIndexes = new AtomicIndex[sics.length];
+ for(int i = 0; i < sics.length; i++) {
+ String subIndexname = "mention-" + i;
+ mentionIndexes[i] = new AtomicMentionsIndex(
+ this,
+ subIndexname,
+ new File(indexDirectory, subIndexname),
+ sics[i].isDirectIndexEnabled(),
+ new LinkedBlockingQueue<GATEDocument>(),
+ new LinkedBlockingQueue<GATEDocument>(),
+ sics[i]);
+ }
// construct the joint array of sub-indexes
subIndexes = new AtomicIndex[tokenIndexes.length + mentionIndexes.length];
@@ -212,11 +225,12 @@
* to all the input queues is interrupted.
* @throws IllegalStateException if the index has already been closed.
*/
- public void indexDocument(GATEDocument document) throws InterruptedException
{
+ public void indexDocument(Document document) throws InterruptedException {
if(closed) throw new IllegalStateException("This index has been closed, "
+ "no further documents can be indexed.");
+ GATEDocument gDocument = new GATEDocument(document, indexConfig);
for(AtomicIndex aSubIndex: subIndexes){
- aSubIndex.getInputQueue().put(document);
+ aSubIndex.getInputQueue().put(gDocument);
}
}
@@ -236,5 +250,17 @@
// wait for indexing to end
documentsCollectorThread.join();
}
+
+ public IndexConfig getIndexConfig() {
+ return indexConfig;
+ }
+ public QueryEngine getQueryEngine() {
+ return null;
+ }
+
+ public File getIndexDirectory() {
+ return indexDirectory;
+ }
+
}
Modified:
mimir/branches/5.0/mimir-core/src/gate/mimir/SemanticAnnotationHelper.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/SemanticAnnotationHelper.java
2014-01-08 17:32:10 UTC (rev 17224)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/SemanticAnnotationHelper.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -16,6 +16,7 @@
import gate.Annotation;
import gate.Document;
+import gate.mimir.index.AtomicMentionsIndex;
import gate.mimir.index.Indexer;
import gate.mimir.index.Mention;
import gate.mimir.search.QueryEngine;
@@ -74,7 +75,7 @@
* when doing indexing.
* @param indexer
*/
- public void init(Indexer indexer);
+ public void init(AtomicMentionsIndex index);
/**
* Called by the containing {@link QueryEngine} when this helper is first
@@ -91,7 +92,7 @@
* @return the URIs for the mention (created in the triple store) that are
* associated with the input annotation.
*/
- public String[] getMentionUris(Annotation annotation, int length, Indexer
indexer);
+ public String[] getMentionUris(Annotation annotation, int length,
AtomicMentionsIndex indexer);
/**
* Prepares this helper for running on a new document.
@@ -167,7 +168,7 @@
* Closes this annotation helper. Implementers should perform maintenance
* operations (such as closing connections to ORDI, etc) on this call.
*/
- public void close(Indexer indexer);
+ public void close(AtomicMentionsIndex index);
/**
Modified: mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicIndex.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicIndex.java
2014-01-08 17:32:10 UTC (rev 17224)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicIndex.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -757,10 +757,7 @@
// now try to just open it as an on-disk index
newIndex = Index.getInstance(mg4jBasename, true, true);
}
- } catch(ConfigurationException | ClassNotFoundException | SecurityException
- | InstantiationException | IllegalAccessException
- | InvocationTargetException | NoSuchMethodException
- | URISyntaxException e) {
+ } catch(Exception e) {
throw new IndexException("Could not open the index just written to " +
mg4jBasename , e);
}
@@ -909,10 +906,7 @@
// now try to just open it as an on-disk index
newIndex = Index.getInstance(mg4jBasename, true, true);
}
- } catch(ConfigurationException | ClassNotFoundException | SecurityException
- | InstantiationException | IllegalAccessException
- | InvocationTargetException | NoSuchMethodException
- | URISyntaxException e) {
+ } catch(Exception e) {
throw new IndexException("Could not open the sub-index at" +
mg4jBasename , e);
}
@@ -988,7 +982,7 @@
*/
protected void flush() throws IOException {
- }
+ };
/**
* Notifies this index to stop its indexing operations, and waits for
all data
@@ -1131,8 +1125,10 @@
return indexDirectory;
}
-
-
+ public MimirIndex getParent() {
+ return parent;
+ }
+
public BlockingQueue<GATEDocument> getInputQueue() {
return inputQueue;
}
Added:
mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicMentionsIndex.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicMentionsIndex.java
(rev 0)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicMentionsIndex.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -0,0 +1,206 @@
+/*
+ * AtomicMentionsIndex.java
+ *
+ * Copyright (c) 2007-2014, The University of Sheffield.
+ *
+ * This file is part of GATE MÃmir (see http://gate.ac.uk/family/mimir.html),
+ * and is free software, licenced under the GNU Lesser General Public License,
+ * Version 3, June 2007 (also included with this distribution as file
+ * LICENCE-LGPL3.html).
+ *
+ * Valentin Tablan, 10 Jan 2014
+ *
+ * $Id$
+ */
+package gate.mimir.index;
+
+import gate.Annotation;
+import gate.AnnotationSet;
+import gate.Document;
+import gate.mimir.IndexConfig;
+import gate.mimir.MimirIndex;
+import gate.mimir.SemanticAnnotationHelper;
+import gate.mimir.IndexConfig.SemanticIndexerConfig;
+import gate.mimir.index.mg4j.GATEDocument;
+import gate.util.OffsetComparator;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.index.NullTermProcessor;
+import it.unimi.dsi.lang.ObjectParser;
+
+import org.apache.log4j.Logger;
+
+/**
+ *
+ */
+public class AtomicMentionsIndex extends AtomicIndex {
+
+ private final static Logger logger =
Logger.getLogger(AtomicMentionsIndex.class);
+
+ /**
+ * The {@link IndexConfig} used by the {@link MimirIndex} that contains this
+ * mentions index.
+ */
+ protected IndexConfig indexConfig;
+
+ protected SemanticIndexerConfig semIdxConfid;
+ /**
+ * Helpers for each semantic annotation type.
+ */
+ protected Map<String, SemanticAnnotationHelper> annotationHelpers;
+
+ protected List<SemanticAnnotationHelper> documentHelpers;
+
+ /**
+ * An {@link OffsetComparator} used to sort the annotations by offset before
+ * indexing.
+ */
+ protected OffsetComparator offsetComparator;
+
+ /**
+ * @param parent
+ * @param name
+ * @param indexDirectory
+ * @param hasDirectIndex
+ * @param inputQueue
+ * @param outputQueue
+ */
+ public AtomicMentionsIndex(MimirIndex parent, String name,
+ File indexDirectory, boolean hasDirectIndex,
+ BlockingQueue<GATEDocument> inputQueue,
+ BlockingQueue<GATEDocument> outputQueue,
+ SemanticIndexerConfig siConfig) {
+ super(parent, name, indexDirectory, hasDirectIndex, inputQueue,
outputQueue);
+ this.semIdxConfid = siConfig;
+ indexConfig = parent.getIndexConfig();
+ //get the helpers
+ annotationHelpers = new HashMap<String, SemanticAnnotationHelper>(
+ siConfig.getAnnotationTypes().length);
+ documentHelpers = new LinkedList<SemanticAnnotationHelper>();
+ for(int i = 0; i < siConfig.getAnnotationTypes().length; i++){
+ SemanticAnnotationHelper theHelper = siConfig.getHelpers()[i];
+ if(theHelper.getMode() == SemanticAnnotationHelper.Mode.DOCUMENT) {
+ documentHelpers.add(theHelper);
+ } else {
+ annotationHelpers.put(siConfig.getAnnotationTypes()[i], theHelper);
+ }
+ theHelper.init(this);
+ }
+ offsetComparator = new OffsetComparator();
+ // create and save the term processor
+ termProcessor = NullTermProcessor.getInstance();
+ additionalProperties.setProperty(Index.PropertyKeys.TERMPROCESSOR,
+ ObjectParser.toSpec(termProcessor));
+ // start the indexing thread
+ indexingThread = new Thread(this, "Mimir-" + name + " indexing thread");
+ indexingThread.start();
+ }
+
+ /* (non-Javadoc)
+ * @see
gate.mimir.index.AtomicIndex#getAnnotsToProcess(gate.mimir.index.mg4j.GATEDocument)
+ */
+ @Override
+ protected Annotation[] getAnnotsToProcess(GATEDocument gateDocument)
+ throws IndexException {
+ Document document = gateDocument.getDocument();
+ Annotation[] semanticAnnots;
+ AnnotationSet semAnnSet =
+ (indexConfig.getSemanticAnnotationSetName() == null ||
+ indexConfig.getSemanticAnnotationSetName().length() == 0) ?
+ document.getAnnotations() :
+ document.getAnnotations(indexConfig.getSemanticAnnotationSetName());
+ if(semAnnSet.size() > 0){
+ AnnotationSet semAnns = null;
+ synchronized(semAnnSet) {
+ semAnns = semAnnSet.get(annotationHelpers.keySet());
+ }
+ semanticAnnots = semAnns.toArray(new Annotation[semAnns.size()]);
+ Arrays.sort(semanticAnnots, offsetComparator);
+ }else{
+ semanticAnnots = new Annotation[0];
+ }
+ return semanticAnnots;
+ }
+
+ /* (non-Javadoc)
+ * @see
gate.mimir.index.AtomicIndex#calculateStartPositionForAnnotation(gate.Annotation,
gate.mimir.index.mg4j.GATEDocument)
+ */
+ @Override
+ protected void calculateStartPositionForAnnotation(Annotation ann,
+ GATEDocument gateDocument) throws IndexException {
+ if(ann == null) {
+ // we're supposed index the document metadata
+ tokenPosition = 0;
+ } else {
+ //calculate the term position for the current semantic annotation
+ while(tokenPosition < gateDocument.getTokenAnnots().length &&
+ gateDocument.getTokenAnnots()[tokenPosition].
+ getEndNode().getOffset().longValue() <=
+ ann.getStartNode().getOffset().longValue()){
+ tokenPosition++;
+ }
+ //check if lastTokenposition is valid
+ if(tokenPosition >= gateDocument.getTokenAnnots().length){
+ //malfunction
+ logger.error(
+ "Semantic annotation [Type:" + ann.getType() +
+ ", start: " + ann.getStartNode().getOffset().toString() +
+ ", end: " + ann.getEndNode().getOffset().toString() +
+ "] outside of the tokens area in document" +
+ " URI: " + gateDocument.uri() +
+ " Title: " + gateDocument.title());
+ }
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see
gate.mimir.index.AtomicIndex#calculateTermStringForAnnotation(gate.Annotation,
gate.mimir.index.mg4j.GATEDocument)
+ */
+ @Override
+ protected String[] calculateTermStringForAnnotation(Annotation ann,
+ GATEDocument gateDocument) throws IndexException {
+ if(ann == null) {
+ // obtain the URIs to be indexed for the *document* metadata
+ List<String> terms = new LinkedList<String>();
+ for(SemanticAnnotationHelper aHelper : documentHelpers) {
+ String[] someTerms = aHelper.getMentionUris(null, Mention.NO_LENGTH,
this);
+ if(someTerms != null) {
+ for(String aTerm : someTerms) {
+ terms.add(aTerm);
+ }
+ }
+ }
+ return terms.toArray(new String[terms.size()]);
+ } else {
+ //calculate the annotation length (as number of terms)
+ SemanticAnnotationHelper helper = annotationHelpers.get(ann.getType());
+ int length = 1;
+ while(tokenPosition + length < gateDocument.getTokenAnnots().length &&
+ gateDocument.getTokenAnnots()[tokenPosition + length].
+ getStartNode().getOffset().longValue() <
+ ann.getEndNode().getOffset().longValue()){
+ length++;
+ }
+ //get the annotation URI
+ return helper.getMentionUris(ann, length, this);
+ }
+ }
+
+ @Override
+ protected void flush() throws IOException {
+ for(SemanticAnnotationHelper sah : annotationHelpers.values()) {
+ sah.close(this);
+ }
+ }
+
+
+}
Property changes on:
mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicMentionsIndex.java
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+Id
\ No newline at end of property
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Modified:
mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicTokenIndex.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicTokenIndex.java
2014-01-08 17:32:10 UTC (rev 17224)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicTokenIndex.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -227,6 +227,5 @@
logger.info("Saving zipped collection");
collectionWriter.close();
}
- super.flush();
}
}
Modified: mimir/branches/5.0/mimir-core/src/gate/mimir/index/Indexer.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/index/Indexer.java
2014-01-08 17:32:10 UTC (rev 17224)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/index/Indexer.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -17,6 +17,7 @@
import gate.Document;
import gate.Gate;
import gate.mimir.IndexConfig;
+import gate.mimir.MimirIndex;
import gate.mimir.SemanticAnnotationHelper;
import gate.mimir.IndexConfig.SemanticIndexerConfig;
import gate.mimir.index.mg4j.GATEDocument;
@@ -101,7 +102,8 @@
config.getSemanticIndexers().length > 0) {
for(SemanticIndexerConfig sic : config.getSemanticIndexers()){
for(SemanticAnnotationHelper sah : sic.getHelpers()){
- sah.init(this);
+// sah.init(this);
+ sah.init((AtomicMentionsIndex)null);
}
}
}
@@ -175,7 +177,8 @@
if(aSIC.getHelpers() != null) {
for(int i = 0; i < aSIC.getHelpers().length; i++) {
SemanticAnnotationHelper aHelper = aSIC.getHelpers()[i];
- aHelper.close(this);
+// aHelper.close(this);
+ aHelper.close((AtomicMentionsIndex)null);
}
}
annHelpersClosingProgress += 1.0 / config.getSemanticIndexers().length;
Modified:
mimir/branches/5.0/mimir-core/src/gate/mimir/index/mg4j/MentionsIndexBuilder.java
===================================================================
---
mimir/branches/5.0/mimir-core/src/gate/mimir/index/mg4j/MentionsIndexBuilder.java
2014-01-08 17:32:10 UTC (rev 17224)
+++
mimir/branches/5.0/mimir-core/src/gate/mimir/index/mg4j/MentionsIndexBuilder.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -186,7 +186,8 @@
// obtain the URIs to be indexed for the *document* metadata
List<String> terms = new LinkedList<String>();
for(SemanticAnnotationHelper aHelper : documentHelpers) {
- String[] someTerms = aHelper.getMentionUris(null, Mention.NO_LENGTH,
indexer);
+// String[] someTerms = aHelper.getMentionUris(null, Mention.NO_LENGTH,
indexer);
+ String[] someTerms = aHelper.getMentionUris(null, Mention.NO_LENGTH,
null);
if(someTerms != null) {
for(String aTerm : someTerms) {
terms.add(aTerm);
@@ -205,7 +206,8 @@
length++;
}
//get the annotation URI
- return helper.getMentionUris(ann, length, indexer);
+// return helper.getMentionUris(ann, length, indexer);
+ return helper.getMentionUris(ann, length, null);
}
}
}
Modified:
mimir/branches/5.0/mimir-core/src/gate/mimir/index/mg4j/zipcollection/DocumentCollectionWriter.java
===================================================================
---
mimir/branches/5.0/mimir-core/src/gate/mimir/index/mg4j/zipcollection/DocumentCollectionWriter.java
2014-01-08 17:32:10 UTC (rev 17224)
+++
mimir/branches/5.0/mimir-core/src/gate/mimir/index/mg4j/zipcollection/DocumentCollectionWriter.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -158,7 +158,7 @@
* be opened for writing.
*/
protected void openZipFile() throws IndexException{
- zipFile = new File(new File(indexDir, Indexer.MG4J_INDEX_DIRNAME),
+ zipFile = new File(indexDir,
Indexer.MIMIR_COLLECTION_BASENAME +
"-" + zipFileId +
Indexer.MIMIR_COLLECTION_EXTENSION);
Modified:
mimir/branches/5.0/mimir-core/src/gate/mimir/util/DelegatingSemanticAnnotationHelper.java
===================================================================
---
mimir/branches/5.0/mimir-core/src/gate/mimir/util/DelegatingSemanticAnnotationHelper.java
2014-01-08 17:32:10 UTC (rev 17224)
+++
mimir/branches/5.0/mimir-core/src/gate/mimir/util/DelegatingSemanticAnnotationHelper.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -21,7 +21,9 @@
import gate.Document;
import gate.mimir.AbstractSemanticAnnotationHelper;
import gate.mimir.Constraint;
+import gate.mimir.MimirIndex;
import gate.mimir.SemanticAnnotationHelper;
+import gate.mimir.index.AtomicMentionsIndex;
import gate.mimir.index.Indexer;
import gate.mimir.index.Mention;
import gate.mimir.search.QueryEngine;
@@ -186,7 +188,7 @@
}
@Override
- public void init(Indexer indexer) {
+ public void init(AtomicMentionsIndex indexer) {
super.init(indexer);
delegate.init(indexer);
}
@@ -203,8 +205,8 @@
@Override
public String[] getMentionUris(Annotation annotation, int length,
- Indexer indexer) {
- return delegate.getMentionUris(annotation, length, indexer);
+ AtomicMentionsIndex index) {
+ return delegate.getMentionUris(annotation, length, index);
}
@Override
@@ -237,8 +239,8 @@
@Override
- public void close(Indexer indexer) {
- delegate.close(indexer);
+ public void close(AtomicMentionsIndex index) {
+ delegate.close(index);
}
@Override
Modified: mimir/branches/5.0/mimir-test/src/gate/mimir/test/Scratch.java
===================================================================
--- mimir/branches/5.0/mimir-test/src/gate/mimir/test/Scratch.java
2014-01-08 17:32:10 UTC (rev 17224)
+++ mimir/branches/5.0/mimir-test/src/gate/mimir/test/Scratch.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -221,6 +221,52 @@
true, Gate.getClassLoader()).asSubclass(
AbstractSemanticAnnotationHelper.class));
+
+ MimirIndex mainIndex = new MimirIndex(indexConfig);
+
+ // index some documents
+ File zipFile = new File(args[1]);
+ String fileURI = zipFile.toURI().toString();
+ ZipFile zip = new ZipFile(args[1]);
+ Enumeration<? extends ZipEntry> entries = zip.entries();
+
+ int copies = 1;
+ while(entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ if(entry.isDirectory()) {
+ continue;
+ }
+ URL url = new URL("jar:" + fileURI + "!/" + entry.getName());
+ Document doc = gate.Factory.newDocument(url, "UTF-8");
+ for(int i = 0; i < copies; i++) {
+ mainIndex.indexDocument(doc);
+ }
+ }
+
+ mainIndex.close();
+ }
+
+ /**
+ * Scratch code to exercise the 5.0 indexer framework
+ */
+ public static void mainAtomicTokenIndexer5(String[] args) throws Exception {
+ Gate.setGateHome(new File("gate-home"));
+ Gate.setUserConfigFile(new File("gate-home/user-gate.xml"));
+ Gate.init();
+ // load the tokeniser plugin
+ Gate.getCreoleRegister().registerDirectories(new
File("gate-home/plugins/ANNIE-tokeniser").toURI().toURL());
+ // load the DB plugin
+ Gate.getCreoleRegister().registerDirectories(new
File("../plugins/db-h2").toURI().toURL());
+ // load the measurements plugin
+ Gate.getCreoleRegister().registerDirectories(new
File("../plugins/measurements").toURI().toURL());
+
+ File indexDir = new File(args[0]);
+
+ IndexConfig indexConfig = TestUtils.getTestIndexConfig(indexDir,
+ Class.forName("gate.mimir.db.DBSemanticAnnotationHelper",
+ true, Gate.getClassLoader()).asSubclass(
+ AbstractSemanticAnnotationHelper.class));
+
MimirIndex mainIndex = new MimirIndex(indexDir);
// build a token indexer
Modified: mimir/branches/5.0/plugins/db-h2/build.xml
===================================================================
--- mimir/branches/5.0/plugins/db-h2/build.xml 2014-01-08 17:32:10 UTC (rev
17224)
+++ mimir/branches/5.0/plugins/db-h2/build.xml 2014-01-10 16:36:03 UTC (rev
17225)
@@ -64,8 +64,8 @@
<javac destdir="${classes.dir}"
srcdir="${src.dir}"
debug="true"
- target="1.6"
- source="1.6"
+ target="1.7"
+ source="1.7"
encoding="UTF-8"
classpathref="compile.classpath" />
</target>
Modified:
mimir/branches/5.0/plugins/db-h2/src/gate/mimir/db/DBSemanticAnnotationHelper.java
===================================================================
---
mimir/branches/5.0/plugins/db-h2/src/gate/mimir/db/DBSemanticAnnotationHelper.java
2014-01-08 17:32:10 UTC (rev 17224)
+++
mimir/branches/5.0/plugins/db-h2/src/gate/mimir/db/DBSemanticAnnotationHelper.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -21,7 +21,9 @@
import gate.mimir.Constraint;
import gate.mimir.ConstraintType;
import gate.mimir.IndexConfig;
+import gate.mimir.MimirIndex;
import gate.mimir.SemanticAnnotationHelper;
+import gate.mimir.index.AtomicMentionsIndex;
import gate.mimir.index.Indexer;
import gate.mimir.index.Mention;
import gate.mimir.search.QueryEngine;
@@ -358,8 +360,8 @@
private transient FeatureMap documentFeatures;
@Override
- public void init(Indexer indexer) {
- super.init(indexer);
+ public void init(AtomicMentionsIndex index) {
+ super.init(index);
if(getUriFeatures() != null && getUriFeatures().length > 0) {
logger.warn(
"This helper type does not fully support URI features, "
@@ -375,23 +377,23 @@
// calculate the basename
// to avoid inter-locking between the multiple SB-based indexers, they
each
- // create their ow database.
+ // create their own database.
tableBaseName = annotationType.replaceAll("[^\\p{Alnum}_]", "_");
- List<String> baseNames = (List<String>)indexer.getIndexConfig()
+ List<String> baseNames = (List<String>)index.getParent().getIndexConfig()
.getContext().get(DB_NAMES_CONTEXT_KEY);
if(baseNames == null) {
baseNames = new LinkedList<String>();
- indexer.getIndexConfig().getContext().put(DB_NAMES_CONTEXT_KEY,
baseNames);
+
index.getParent().getIndexConfig().getContext().put(DB_NAMES_CONTEXT_KEY,
baseNames);
}
while(baseNames.contains(tableBaseName)) {
tableBaseName += "_";
}
baseNames.add(tableBaseName);
- File dbDir = new File(indexer.getIndexDir(), DB_DIR_NAME);
+ File dbDir = new File(index.getIndexDirectory(), DB_DIR_NAME);
try {
Class.forName("org.h2.Driver");
- String cacheSizeStr = indexer.getIndexConfig().getOptions().get(
+ String cacheSizeStr =
index.getParent().getIndexConfig().getOptions().get(
DB_CACHE_SIZE_OPTIONS_KEY);
// default to 100 MiB, if not provided
if(cacheSizeStr == null) cacheSizeStr = Integer.toString(100 *1024);
@@ -400,7 +402,7 @@
"/" + tableBaseName + ";CACHE_SIZE=" + cacheSizeStr, "sa", "");
dbConnection.setAutoCommit(true);
dbConnection.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED);
- createDb(indexer);
+ createDb(index);
} catch(SQLException e) {
throw new RuntimeException("Error while initialising the database", e);
} catch(ClassNotFoundException e) {
@@ -539,7 +541,7 @@
*
* @throws SQLException
*/
- protected void createDb(Indexer indexer) throws SQLException {
+ protected void createDb(AtomicMentionsIndex indexer) throws SQLException {
Statement stmt = dbConnection.createStatement();
// ////////////////////////////////
// create the Level 1 table
@@ -748,7 +750,7 @@
@Override
public String[] getMentionUris(Annotation ann, int length,
- Indexer indexer) {
+ AtomicMentionsIndex index) {
FeatureMap featuresToIndex;
if(getMode() == Mode.DOCUMENT) {
length = -1;
@@ -1289,7 +1291,7 @@
}
@Override
- public void close(Indexer indexer) {
+ public void close(AtomicMentionsIndex indexer) {
closeDB();
}
Modified: mimir/branches/5.0/plugins/measurements/build.xml
===================================================================
--- mimir/branches/5.0/plugins/measurements/build.xml 2014-01-08 17:32:10 UTC
(rev 17224)
+++ mimir/branches/5.0/plugins/measurements/build.xml 2014-01-10 16:36:03 UTC
(rev 17225)
@@ -61,8 +61,8 @@
<javac destdir="${classes.dir}"
srcdir="${src.dir}"
debug="true"
- target="1.6"
- source="1.6"
+ target="1.7"
+ source="1.7"
encoding="UTF-8"
classpathref="compile.classpath" />
</target>
@@ -101,7 +101,7 @@
Windowtitle="${app.title} (${app.version}) JavaDoc"
docencoding="UTF-8"
charset="UTF-8"
- source="1.6"
+ source="1.7"
useexternalfile="yes"
breakiterator="true">
<fileset dir="${src.dir}" includes="**/*.java"/>
Modified:
mimir/branches/5.0/plugins/measurements/src/gate/mimir/measurements/MeasurementAnnotationHelper.java
===================================================================
---
mimir/branches/5.0/plugins/measurements/src/gate/mimir/measurements/MeasurementAnnotationHelper.java
2014-01-08 17:32:10 UTC (rev 17224)
+++
mimir/branches/5.0/plugins/measurements/src/gate/mimir/measurements/MeasurementAnnotationHelper.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -21,6 +21,7 @@
import gate.mimir.Constraint;
import gate.mimir.ConstraintType;
import gate.mimir.SemanticAnnotationHelper;
+import gate.mimir.index.AtomicMentionsIndex;
import gate.mimir.index.Indexer;
import gate.mimir.index.Mention;
import gate.mimir.search.QueryEngine;
@@ -172,7 +173,7 @@
}
@Override
- public void init(Indexer indexer) {
+ public void init(AtomicMentionsIndex indexer) {
// create the delegate - needs to happen before super.init
if(delegateHelperType == null) { throw new IllegalArgumentException(
"No value provided for delegateHelperType"); }
Modified: mimir/branches/5.0/plugins/sesame/build.xml
===================================================================
--- mimir/branches/5.0/plugins/sesame/build.xml 2014-01-08 17:32:10 UTC (rev
17224)
+++ mimir/branches/5.0/plugins/sesame/build.xml 2014-01-10 16:36:03 UTC (rev
17225)
@@ -63,8 +63,8 @@
<javac destdir="${classes.dir}"
srcdir="${src.dir}"
debug="true"
- target="1.6"
- source="1.6"
+ target="1.7"
+ source="1.7"
encoding="UTF-8"
classpathref="compile.classpath" />
</target>
@@ -105,7 +105,7 @@
Windowtitle="${app.title} (${app.version}) JavaDoc"
docencoding="UTF-8"
charset="UTF-8"
- source="1.6"
+ source="1.7"
useexternalfile="yes"
breakiterator="true">
<fileset dir="${src.dir}" includes="**/*.java"/>
Modified:
mimir/branches/5.0/plugins/sesame/src/gate/mimir/sesame/SesameSemanticAnnotationHelper.java
===================================================================
---
mimir/branches/5.0/plugins/sesame/src/gate/mimir/sesame/SesameSemanticAnnotationHelper.java
2014-01-08 17:32:10 UTC (rev 17224)
+++
mimir/branches/5.0/plugins/sesame/src/gate/mimir/sesame/SesameSemanticAnnotationHelper.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -85,7 +85,9 @@
import gate.mimir.Constraint;
import gate.mimir.ConstraintType;
import gate.mimir.IndexConfig;
+import gate.mimir.MimirIndex;
import gate.mimir.SemanticAnnotationHelper;
+import gate.mimir.index.AtomicMentionsIndex;
import gate.mimir.index.IndexException;
import gate.mimir.index.Indexer;
import gate.mimir.index.Mention;
@@ -270,7 +272,7 @@
private static final long serialVersionUID = 1396623744631385943L;
@Override
- public void init(Indexer indexer) {
+ public void init(AtomicMentionsIndex indexer) {
super.init(indexer);
setFloatFeatures(concatenateArrays(getIntegerFeatures(),
getFloatFeatures()));
setIntegerFeatures(new String[0]);
@@ -284,7 +286,7 @@
this.uriFeaturePredicates = new URI[this.uriFeatureNames.length];
try {
- connection =
getRepositoryConnection(indexer.getIndexConfig());
+ connection =
getRepositoryConnection(indexer.getParent().getIndexConfig());
connection.setAutoCommit(true);
parser = new SPARQLParser();
// ensure static initialization is done.
@@ -339,7 +341,7 @@
@Override
public String[] getMentionUris(Annotation ann, int length,
- Indexer indexer) {
+ AtomicMentionsIndex indexer) {
if (!isInited()) init(indexer);
FeatureMap featuresToIndex;
@@ -532,7 +534,7 @@
}
@Override
- public void close(Indexer indexer) {
+ public void close(AtomicMentionsIndex index) {
if (isInited()) {
logger.info("Closing Sesame Repository Connection");
try {
@@ -545,21 +547,20 @@
connection = null;
parser = null;
// if we're the last helper, shutdown the ORDI source
- int clientCount = (Integer)
indexer.getIndexConfig().getContext()
+ int clientCount = (Integer)
index.getParent().getIndexConfig().getContext()
.get(SESAME_CONNECTION_COUNT_KEY);
clientCount--;
- indexer.getIndexConfig()
- .getContext()
+ index.getParent().getIndexConfig().getContext()
.put(SESAME_CONNECTION_COUNT_KEY,
Integer.valueOf(clientCount));
if (clientCount == 0) {
// shutting down the ORDI Source
- RepositoryManager manager = (RepositoryManager)
indexer
+ RepositoryManager manager = (RepositoryManager)
index.getParent()
.getIndexConfig().getContext().get(SESAME_RMANAGER_KEY);
if (manager != null) {
logger.info("Shutting down Sesame
repositoryManager");
manager.shutDown();
- indexer.getIndexConfig().getContext()
+
index.getParent().getIndexConfig().getContext()
.remove(SESAME_RMANAGER_KEY);
}
}
Modified: mimir/branches/5.0/plugins/sparql/build.xml
===================================================================
--- mimir/branches/5.0/plugins/sparql/build.xml 2014-01-08 17:32:10 UTC (rev
17224)
+++ mimir/branches/5.0/plugins/sparql/build.xml 2014-01-10 16:36:03 UTC (rev
17225)
@@ -52,8 +52,8 @@
<javac destdir="${classes.dir}"
srcdir="${src.dir}"
debug="true"
- target="1.6"
- source="1.6"
+ target="1.7"
+ source="1.7"
encoding="UTF-8"
classpathref="compile.classpath" />
</target>
@@ -89,7 +89,7 @@
Windowtitle="${app.title} (${app.version}) JavaDoc"
docencoding="UTF-8"
charset="UTF-8"
- source="1.6"
+ source="1.7"
useexternalfile="yes"
breakiterator="true">
<fileset dir="${src.dir}" includes="**/*.java"/>
Modified:
mimir/branches/5.0/plugins/sparql/src/gate/mimir/sparql/SPARQLSemanticAnnotationHelper.java
===================================================================
---
mimir/branches/5.0/plugins/sparql/src/gate/mimir/sparql/SPARQLSemanticAnnotationHelper.java
2014-01-08 17:32:10 UTC (rev 17224)
+++
mimir/branches/5.0/plugins/sparql/src/gate/mimir/sparql/SPARQLSemanticAnnotationHelper.java
2014-01-10 16:36:03 UTC (rev 17225)
@@ -41,6 +41,7 @@
import gate.mimir.Constraint;
import gate.mimir.ConstraintType;
import gate.mimir.SemanticAnnotationHelper;
+import gate.mimir.index.AtomicMentionsIndex;
import gate.mimir.index.Indexer;
import gate.mimir.index.Mention;
import gate.mimir.search.QueryEngine;
@@ -234,7 +235,7 @@
}
@Override
- public void init(Indexer indexer) {
+ public void init(AtomicMentionsIndex indexer) {
super.init(indexer);
// calculate authHeader value
if(sparqlEndpointUser != null && sparqlEndpointUser.length() > 0){
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
CenturyLink Cloud: The Leader in Enterprise Cloud Services.
Learn Why More Businesses Are Choosing CenturyLink Cloud For
Critical Workloads, Development Environments & Everything In Between.
Get a Quote or Start a Free Trial Today.
http://pubads.g.doubleclick.net/gampad/clk?id=119420431&iu=/4140/ostg.clktrk
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
