Revision: 15935
http://gate.svn.sourceforge.net/gate/?rev=15935&view=rev
Author: valyt
Date: 2012-07-13 15:15:56 +0000 (Fri, 13 Jul 2012)
Log Message:
-----------
Started work on term queries:
- first term query type: AnnotationTermQuery, implemented
- changed AnnotationQuery to use the new AnnotationTermQuery to get the terms
for the underlying disjunction.
Modified Paths:
--------------
mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/TermQuery.java
Added Paths:
-----------
mimir/trunk/mimir-core/src/gate/mimir/search/terms/
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsResultSet.java
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
2012-07-13 12:37:22 UTC (rev 15934)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
2012-07-13 15:15:56 UTC (rev 15935)
@@ -26,6 +26,7 @@
import gate.mimir.index.mg4j.TokenIndexBuilder;
import gate.mimir.index.mg4j.zipcollection.DocumentCollection;
import gate.mimir.index.mg4j.zipcollection.DocumentData;
+import gate.mimir.search.query.AnnotationQuery;
import gate.mimir.search.query.Binding;
import gate.mimir.search.query.QueryExecutor;
import gate.mimir.search.query.QueryNode;
@@ -427,6 +428,26 @@
}
/**
+ * Get the {@link SemanticAnnotationHelper} corresponding to a query's
+ * annotation type.
+ * @throws IllegalArgumentException if the annotation helper for this
+ * type cannot be found.
+ */
+ public SemanticAnnotationHelper getAnnotationHelper(AnnotationQuery query) {
+ for(SemanticIndexerConfig semConfig : indexConfig.getSemanticIndexers()){
+ for(int i = 0; i < semConfig.getAnnotationTypes().length; i++){
+ if(query.getAnnotationType().equals(
+ semConfig.getAnnotationTypes()[i])){
+ return semConfig.getHelpers()[i];
+ }
+ }
+ }
+ throw new IllegalArgumentException("Semantic annotation type \""
+ + query.getAnnotationType() + "\" not known to this query
engine.");
+ }
+
+
+ /**
* Obtains a query executor for a given {@link QueryNode}.
*
* @param query
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
2012-07-13 12:37:22 UTC (rev 15934)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
2012-07-13 15:15:56 UTC (rev 15935)
@@ -17,16 +17,14 @@
import gate.mimir.Constraint;
import gate.mimir.ConstraintType;
-import gate.mimir.IndexConfig;
-import gate.mimir.IndexConfig.SemanticIndexerConfig;
import gate.mimir.SemanticAnnotationHelper;
-import gate.mimir.index.Mention;
import gate.mimir.search.QueryEngine;
-import it.unimi.dsi.fastutil.ints.IntList;
-import it.unimi.dsi.fastutil.objects.ReferenceSet;
-import it.unimi.dsi.fastutil.objects.ReferenceSets;
+import gate.mimir.search.terms.AnnotationTermQuery;
+import gate.mimir.search.terms.TermsResultSet;
import it.unimi.dsi.big.mg4j.index.Index;
import it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor;
+import it.unimi.dsi.fastutil.objects.ReferenceSet;
+import it.unimi.dsi.fastutil.objects.ReferenceSets;
import java.io.IOException;
import java.util.ArrayList;
@@ -79,55 +77,26 @@
* Build the underlying OrQuery executor that this annotation query uses.
*/
protected void buildQuery() throws IOException {
- // find the semantic annotation helper for the right annotation type
- SemanticAnnotationHelper helper =
getAnnotationHelper(engine.getIndexConfig());
- isInDocumentMode = (helper.getMode() ==
SemanticAnnotationHelper.Mode.DOCUMENT);
- // ask the helper for the mentions that correspond to this query
- long start = System.currentTimeMillis();
- List<Mention> mentions =
helper.getMentions(query.getAnnotationType(),
- query.getConstraints(), engine);
- logger.debug(mentions.size() + " mentions obtained in " +
- (System.currentTimeMillis() - start) + " ms");
- // now create a big OrQuery of all the possible mentions with
- // appropriate gaps
- if(mentions.size() > 0) {
- QueryNode[] disjuncts = new QueryNode[mentions.size()];
- int index = 0;
- for(Mention m : mentions) {
+ SemanticAnnotationHelper helper = engine.getAnnotationHelper(query);
+ isInDocumentMode = (helper.getMode() ==
+ SemanticAnnotationHelper.Mode.DOCUMENT);
+ // get the mention URIs
+ TermsResultSet trs = new AnnotationTermQuery(query, engine).execute();
+ if(trs.terms != null && trs.terms.length > 0 &&
+ trs.termLengths != null) {
+ QueryNode[] disjuncts = new QueryNode[trs.terms.length];
+ for(int index = 0; index < trs.terms.length; index++) {
// create a term query for the mention URI
disjuncts[index] = new TermQuery(query.annotationType,
- m.getUri(), m.getLength());
- index++;
+ trs.termIds[index], trs.termLengths[index]);
}
-
QueryNode underlyingQuery = new OrQuery(disjuncts);
- underlyingExecutor = underlyingQuery.getQueryExecutor(engine);
+ underlyingExecutor = underlyingQuery.getQueryExecutor(engine);
} else {
// no results from the helper => no results from us
latestDocument = -1;
}
}
-
- /**
- * Get the {@link SemanticAnnotationHelper} corresponding to this query's
- * annotation type.
- * @param indexConfig the index configuration
- * @throws IllegalArgumentException if the annotation helper for this
- * type is not a {@link PlainAnnotationHelper}.
- */
- protected SemanticAnnotationHelper getAnnotationHelper(
- IndexConfig indexConfig) {
- for(SemanticIndexerConfig semConfig : indexConfig.getSemanticIndexers()){
- for(int i = 0; i < semConfig.getAnnotationTypes().length; i++){
- if(query.getAnnotationType().equals(
- semConfig.getAnnotationTypes()[i])){
- return semConfig.getHelpers()[i];
- }
- }
- }
- throw new IllegalArgumentException("Semantic annotation type \""
- + query.getAnnotationType() + "\" not known to this query
engine.");
- }
/* (non-Javadoc)
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/query/TermQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/query/TermQuery.java
2012-07-13 12:37:22 UTC (rev 15934)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/query/TermQuery.java
2012-07-13 15:15:56 UTC (rev 15935)
@@ -66,6 +66,13 @@
private String term;
/**
+ * The term ID for this query. If not known,
+ * {@link DocumentIterator#END_OF_LIST} is used.
+ */
+ private long termId = DocumentIterator.END_OF_LIST;
+
+
+ /**
* The name of the index to search.
*/
private String indexName;
@@ -122,30 +129,20 @@
public TermQueryExecutor(TermQuery node, QueryEngine engine) throws
IOException {
super(engine, node);
this.query = node;
- switch(this.query.indexType){
- case TOKENS:
- if(query.indexName == null){
- //token query, with no index name provided -> use the default
- indexReaderPool = engine.getIndexes()[0];
- }else{
- indexReaderPool = engine.getTokenIndex(query.indexName);
- }
- break;
- case ANNOTATIONS:
- indexReaderPool = engine.getAnnotationIndex(query.indexName);
- break;
- default:
- throw new IllegalArgumentException("Indexes of type " +
- this.query.indexType + " are not supported!");
- }
+ indexReaderPool = query.getIndex(engine);
if(indexReaderPool == null) throw new IllegalArgumentException(
"No index provided for field " + node.getIndexName() + "!");
- //use the term processor for the query term
- MutableString mutableString = new MutableString(query.getTerm());
- indexReader = indexReaderPool.borrowReader();
- indexReaderPool.getIndex().termProcessor.processTerm(mutableString);
- this.indexIterator = indexReader.documents(mutableString.toString());
+ indexReader = indexReaderPool.borrowReader();
+ // if we have the term ID, use that
+ if(query.termId != DocumentIterator.END_OF_LIST) {
+ this.indexIterator = indexReader.documents(query.termId);
+ } else {
+ //use the term processor for the query term
+ MutableString mutableString = new MutableString(query.getTerm());
+ indexReaderPool.getIndex().termProcessor.processTerm(mutableString);
+ this.indexIterator = indexReader.documents(mutableString.toString());
+ }
positionsIterator = null;
}
@@ -325,14 +322,44 @@
public CharSequence getTerm() {
return term;
}
+
/**
+ * @return the termId
+ */
+ public long getTermId() {
+ return termId;
+ }
+
+ /**
* @return the indexName
*/
public String getIndexName() {
return indexName;
}
+ /**
+ * Gets the index for this query in a given {@link QueryEngine}.
+ * @param engine
+ * @return
+ */
+ public IndexReaderPool getIndex(QueryEngine engine) {
+ switch(this.indexType){
+ case TOKENS:
+ if(indexName == null){
+ //token query, with no index name provided -> use the default
+ return engine.getIndexes()[0];
+ }else{
+ return engine.getTokenIndex(indexName);
+ }
+ case ANNOTATIONS:
+ return engine.getAnnotationIndex(indexName);
+ default:
+ throw new IllegalArgumentException("Indexes of type " +
+ indexType + " are not supported!");
+ }
+ }
+
/**
* Creates a new term query, for searching over the document text.
*
@@ -348,7 +375,20 @@
this(IndexType.TOKENS, indexName, term, 1);
}
-
+ /**
+ * Creates a new term query, for searching over the document text.
+ *
+ * @param indexName the name of the index to be searched. This should be one
+ * of the annotation feature names used for indexing tokens (see
+ * {@link IndexConfig.TokenIndexerConfig}).
+ *
+ * @param termId the term ID for the term to be searched for.
+ *
+ * @see IndexConfig.TokenIndexerConfig
+ */
+ public TermQuery(String indexName, long termId) {
+ this(IndexType.TOKENS, indexName, termId, 1);
+ }
/**
* Creates a new term query, for searching over semantic annotations.
@@ -362,14 +402,25 @@
* @param length the length of the mention sought.
*/
public TermQuery(String annotationType, String mentionURI, int length) {
- this.indexType = IndexType.ANNOTATIONS;
- this.indexName = annotationType;
- this.term = mentionURI;
- this.length = length;
+ this(IndexType.ANNOTATIONS, annotationType, mentionURI, length);
}
-
/**
+ * Creates a new term query, for searching over semantic annotations.
+ *
+ * @param annotationType the type of annotation sought. This should one of
the
+ * annotation types used when indexing semantic annotations (see
+ * {@link IndexConfig.SemanticIndexerConfig}).
+ *
+ * @param mentionTermid the term ID for the mentionURI sought.
+ *
+ * @param length the length of the mention sought.
+ */
+ public TermQuery(String annotationType, long mentionTermid, int length) {
+ this(IndexType.ANNOTATIONS, annotationType, mentionTermid, length);
+ }
+
+ /**
* Creates a new term query. This constructor is part of a low-level API.
see
* the other constructors of this class, which may be more suitable!
*
@@ -394,6 +445,32 @@
}
/**
+ * Creates a new term query. This constructor is part of a low-level API.
see
+ * the other constructors of this class, which may be more suitable!
+ *
+ * @param indexType The type of index to be searched.
+ *
+ * @param indexName the name of the index to be searched. If the indexType is
+ * {@link IndexType#TOKENS}, then the name is interpreted as the feature
name
+ * for the document tokens, if the indexType is {@link
IndexType#ANNOTATIONS},
+ * then the name is interpreted as annotation type.
+ *
+ * @param length the length of the hits (useful in the case of annotation
+ * indexes, where the length of each mention is stored external to the
actual
+ * index).
+ *
+ * @param termId the term ID for sought term.
+ */
+ public TermQuery(IndexType indexType, String indexName, long termId, int
length) {
+ this.indexType = indexType;
+ this.indexName = indexName;
+ this.termId = termId;
+ this.length = length;
+ }
+
+
+
+ /**
* Gets a new query executor for this {@link TermQuery}.
* @param indexes the set of indexes running on.
* @return an appropriate {@link QueryExecutor} (in this case, an instance of
Added:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractTermsQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractTermsQuery.java
(rev 0)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractTermsQuery.java
2012-07-13 15:15:56 UTC (rev 15935)
@@ -0,0 +1,39 @@
+/*
+ * AbstractTermsQuery.java
+ *
+ * Copyright (c) 2007-2011, The University of Sheffield.
+ *
+ * This file is part of GATE Mímir (see http://gate.ac.uk/family/mimir.html),
+ * and is free software, licenced under the GNU Lesser General Public License,
+ * Version 3, June 2007 (also included with this distribution as file
+ * LICENCE-LGPL3.html).
+ *
+ * Valentin Tablan, 13 Jul 2012
+ *
+ * $Id$
+ */
+package gate.mimir.search.terms;
+
+/**
+ * Base class for term queries.
+ */
+public abstract class AbstractTermsQuery implements TermQuery {
+
+ protected boolean stringsEnabled;
+
+ protected boolean idsEnabled;
+
+ protected boolean countsEnabled;
+
+ public AbstractTermsQuery(boolean idsEnabled, boolean stringsEnabled,
+ boolean countsEnabled) {
+ this.idsEnabled = idsEnabled;
+ this.stringsEnabled = stringsEnabled;
+ this.countsEnabled = countsEnabled;
+ }
+
+ public AbstractTermsQuery() {
+ this(true, false, false);
+ }
+
+}
Property changes on:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractTermsQuery.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
Added:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermQuery.java
(rev 0)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermQuery.java
2012-07-13 15:15:56 UTC (rev 15935)
@@ -0,0 +1,106 @@
+/*
+ * AnnotationTermQuery.java
+ *
+ * Copyright (c) 2007-2011, The University of Sheffield.
+ *
+ * This file is part of GATE Mímir (see http://gate.ac.uk/family/mimir.html),
+ * and is free software, licenced under the GNU Lesser General Public License,
+ * Version 3, June 2007 (also included with this distribution as file
+ * LICENCE-LGPL3.html).
+ *
+ * Valentin Tablan, 13 Jul 2012
+ *
+ * $Id$
+ */
+package gate.mimir.search.terms;
+
+import it.unimi.dsi.big.mg4j.index.BitStreamIndex;
+import it.unimi.dsi.big.mg4j.index.Index;
+import it.unimi.dsi.big.util.StringMap;
+import it.unimi.dsi.lang.MutableString;
+
+import java.util.List;
+
+import org.apache.log4j.Logger;
+
+import gate.mimir.SemanticAnnotationHelper;
+import gate.mimir.index.Mention;
+import gate.mimir.search.QueryEngine;
+import gate.mimir.search.query.AnnotationQuery;
+import gate.mimir.search.query.OrQuery;
+import gate.mimir.search.query.QueryNode;
+import gate.mimir.search.query.TermQuery;
+
+/**
+ * Given an {@link AnnotationQuery}, this finds the set of terms that satisfy
+ * it.
+ */
+public class AnnotationTermQuery extends AbstractTermsQuery {
+
+ public AnnotationTermQuery(boolean idsEnabled, boolean stringsEnabled,
+ boolean countsEnabled,
+ AnnotationQuery annotationQuery) {
+ super(idsEnabled, stringsEnabled, countsEnabled);
+ this.annotationQuery = annotationQuery;
+ }
+
+ public AnnotationTermQuery(AnnotationQuery annotationQuery, QueryEngine
engine) {
+ super();
+ this.annotationQuery = annotationQuery;
+ this.engine = engine;
+ }
+
+ protected AnnotationQuery annotationQuery;
+
+ protected QueryEngine engine;
+
+ private static final Logger logger =
Logger.getLogger(AnnotationTermQuery.class);
+
+ /* (non-Javadoc)
+ * @see gate.mimir.search.terms.TermQuery#execute()
+ */
+ @Override
+ public TermsResultSet execute() {
+ // find the semantic annotation helper for the right annotation type
+ SemanticAnnotationHelper helper =
+ engine.getAnnotationHelper(annotationQuery);
+ // ask the helper for the mentions that correspond to this query
+ long start = System.currentTimeMillis();
+ List<Mention> mentions = helper.getMentions(
+ annotationQuery.getAnnotationType(),
+ annotationQuery.getConstraints(), engine);
+ logger.debug(mentions.size() + " mentions obtained in " +
+ (System.currentTimeMillis() - start) + " ms");
+ StringMap<? extends CharSequence> termMap = null;
+ Index mg4jIndex = engine.getAnnotationIndex(
+ annotationQuery.getAnnotationType()).getIndex();
+ if(mg4jIndex instanceof BitStreamIndex) {
+ termMap = ((BitStreamIndex)mg4jIndex).termMap;
+ } else {
+ // this indicates major changes in the underlying MG4J implementation
+ throw new IllegalStateException(
+ "Underlying MG4J index is not bitstream based!");
+ }
+
+ if(mentions.size() > 0) {
+ long[] termIds = new long[mentions.size()];
+ String[] terms = new String[mentions.size()];
+ int[] lengths = new int[mentions.size()];
+ int index = 0;
+ for(Mention m : mentions) {
+ terms[index] = m.getUri();
+ lengths[index] = m.getLength();
+ // find the term ID
+ //use the term processor for the query term
+ MutableString mutableString = new MutableString(m.getUri());
+ mg4jIndex.termProcessor.processTerm(mutableString);
+ // TODO use toString() if not working
+ termIds[index] = termMap.getLong( mutableString);
+ index++;
+ }
+ return new TermsResultSet(termIds, terms, lengths, null);
+ } else {
+ return TermsResultSet.EMPTY;
+ }
+ }
+}
Property changes on:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermQuery.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
Added: mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermQuery.java
(rev 0)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermQuery.java
2012-07-13 15:15:56 UTC (rev 15935)
@@ -0,0 +1,28 @@
+/*
+ * TermQuery.java
+ *
+ * Copyright (c) 2007-2011, The University of Sheffield.
+ *
+ * This file is part of GATE Mímir (see http://gate.ac.uk/family/mimir.html),
+ * and is free software, licenced under the GNU Lesser General Public License,
+ * Version 3, June 2007 (also included with this distribution as file
+ * LICENCE-LGPL3.html).
+ *
+ * Valentin Tablan, 13 Jul 2012
+ *
+ * $Id$
+ */
+package gate.mimir.search.terms;
+
+/**
+ * A query that returns terms.
+ * Term queries are fast, so they run synchronously.
+ */
+public interface TermQuery {
+ /**
+ * Runs the term query (in the calling thread) and returns the matched terms.
+ * @return a {@link TermsResultSet} containing the matched terms.
+ */
+ public TermsResultSet execute();
+
+}
Property changes on:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermQuery.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
Added: mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsResultSet.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsResultSet.java
(rev 0)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsResultSet.java
2012-07-13 15:15:56 UTC (rev 15935)
@@ -0,0 +1,60 @@
+/*
+ * TermsResultSet.java
+ *
+ * Copyright (c) 2007-2011, The University of Sheffield.
+ *
+ * This file is part of GATE Mímir (see http://gate.ac.uk/family/mimir.html),
+ * and is free software, licenced under the GNU Lesser General Public License,
+ * Version 3, June 2007 (also included with this distribution as file
+ * LICENCE-LGPL3.html).
+ *
+ * Valentin Tablan, 13 Jul 2012
+ *
+ * $Id$
+ */
+package gate.mimir.search.terms;
+
+/**
+ * Class representing the results of a {@link TermQuery}.
+ */
+public class TermsResultSet {
+
+ /**
+ * The term IDs, as retrieved from the index. Array parallel with
+ * {@link #terms} and {@link #counts}.
+ */
+ public final long[] termIds;
+
+ /**
+ * The lengths (number of tokens) for the terms.
+ */
+ public final int[] termLengths;
+
+ /**
+ * The strings for the terms. Array parallel with
+ * {@link #termIds} and {@link #counts}.
+ */
+ public final String[] terms;
+
+
+ /**
+ * The counts (numbers of occurrences) for the terms. Array parallel with
+ * {@link #terms} and {@link #termIds}.
+ */
+ public final int[] counts;
+
+ public TermsResultSet(long[] termIds, String[] terms,int[] termLengths,
int[] counts) {
+ super();
+ this.termIds = termIds;
+ this.terms = terms;
+ this.termLengths = termLengths;
+ this.counts = counts;
+ }
+
+ /**
+ * Constant representing the empty result set.
+ */
+ public static final TermsResultSet EMPTY = new TermsResultSet(
+ new long[] {}, new String[]{}, new int[] {}, new int[]{});
+
+}
Property changes on:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsResultSet.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Live Security Virtual Conference
Exclusive live event will cover all the ways today's security and
threat landscape has changed and how IT managers can respond. Discussions
will include endpoint security, mobile security and the latest in malware
threats. http://www.accelacomm.com/jaw/sfrnl04242012/114/50122263/
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
