Revision: 16341
http://gate.svn.sourceforge.net/gate/?rev=16341&view=rev
Author: valyt
Date: 2012-11-27 16:22:45 +0000 (Tue, 27 Nov 2012)
Log Message:
-----------
Changes to the TermsQuery implementations:
- termIDs are not provided any more (as they are not portable between sessions/
machines);
- term Strings are now always provided;
- changed all the existing implementations accordingly.
None of these changes are yet tested.
Modified Paths:
--------------
mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractIndexTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AndTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsAndTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsOrTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/LimitTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/OrTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/SortedTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsResultSet.java
Added Paths:
-----------
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractWrapperTermsQuery.java
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -82,14 +82,13 @@
SemanticAnnotationHelper.Mode.DOCUMENT);
// get the mention URIs
TermsResultSet trs = new AnnotationTermsQuery(query).execute(engine);
- if(trs.termIds != null && trs.termIds.length > 0 &&
+ if(trs.termStrings != null && trs.termStrings.length > 0 &&
trs.termLengths != null) {
QueryNode[] disjuncts = new QueryNode[trs.termStrings.length];
for(int index = 0; index < trs.termStrings.length; index++) {
// create a term query for the mention URI
disjuncts[index] = new TermQuery(query.annotationType,
- trs.termStrings[index], trs.termIds[index],
- trs.termLengths[index]);
+ trs.termStrings[index], trs.termLengths[index]);
}
QueryNode underlyingQuery = new OrQuery(disjuncts);
underlyingExecutor = underlyingQuery.getQueryExecutor(engine);
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractIndexTermsQuery.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractIndexTermsQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractIndexTermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -23,7 +23,6 @@
import it.unimi.dsi.big.mg4j.search.visitor.CounterSetupVisitor;
import it.unimi.dsi.big.mg4j.search.visitor.TermCollectionVisitor;
import it.unimi.dsi.fastutil.ints.IntArrayList;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOException;
@@ -95,7 +94,16 @@
*/
protected transient QueryEngine engine;
+
+ protected final boolean countsEnabled;
/**
+ * @return the countsEnabled
+ */
+ public boolean isCountsEnabled() {
+ return countsEnabled;
+ }
+
+ /**
* The default set of stop words.
*/
public static final String[] DEFAULT_STOP_WORDS = new String[] {
@@ -160,8 +168,9 @@
* @param limit the maximum number of terms to return.
*/
public AbstractIndexTermsQuery(String indexName, IndexType indexType,
- boolean stringsEnabled, boolean countsEnabled, int limit) {
- super(stringsEnabled, countsEnabled, limit);
+ boolean countsEnabled, int limit) {
+ super(limit);
+ this.countsEnabled = countsEnabled;
this.indexName = indexName;
this.indexType = indexType;
}
@@ -203,10 +212,7 @@
protected TermsResultSet buildResultSet(DocumentIterator documentIterator)
throws IOException {
// prepare local data
- LongArrayList termIds = new LongArrayList();
- ObjectArrayList<String> termStrings = stringsEnabled ||
- (indexType == IndexType.ANNOTATIONS && describeAnnotations) ?
- new ObjectArrayList<String>() : null;
+ ObjectArrayList<String> termStrings = new ObjectArrayList<String>();
IntArrayList termCounts = countsEnabled ? new IntArrayList() : null;
TermCollectionVisitor termCollectionVisitor = null;
CounterSetupVisitor counterSetupVisitor = null;
@@ -227,7 +233,7 @@
long termId = documentIterator.nextDocument();
terms:while(termId != DocumentIterator.END_OF_LIST && termId != -1 &&
- termIds.size() < limit) {
+ termStrings.size() < limit) {
int termCount = -1;
if(countsEnabled){
counterSetupVisitor.clear();
@@ -236,16 +242,8 @@
for (int aCount : counterSetupVisitor.count ) termCount += aCount;
}
String termString = null;
- // get the term string, if required
- if(// if stop words are blocked, we need to check the term string
- isStopWordsBlocked() ||
- // if strings enabled, we need the term string so we can return it
- stringsEnabled ||
- // if annotation index, we need the term to check for the right
- // annotation type inside the index (which may include multiple types)
- indexType == IndexType.ANNOTATIONS) {
- termString = indirectIndexPool.getTerm(termId);
- }
+ // get the term string
+ termString = indirectIndexPool.getTerm(termId);
if(indexType == IndexType.ANNOTATIONS) {
if(!annotationHelper.isMentionUri(termString)){
@@ -253,7 +251,7 @@
termId = documentIterator.nextDocument();
continue terms;
}
- if(stringsEnabled && describeAnnotations) {
+ if(describeAnnotations) {
termString = annotationHelper.describeMention(termString);
// check if this term has the same description as a previous one
// which would make it appear as an indistinguishable duplicate
@@ -276,20 +274,18 @@
continue terms;
}
- termIds.add(termId);
+ termStrings.add(termString);
if(countsEnabled){
termCounts.add(termCount);
}
- if(termStrings != null){
- termStrings.add(termString);
- }
+
termId = documentIterator.nextDocument();
}
// construct the result
- return new TermsResultSet(termIds.toLongArray(),
- stringsEnabled ? termStrings.toArray(new String[termStrings.size()]) :
null,
- null,
- countsEnabled ? termCounts.toIntArray() : null);
+ return new TermsResultSet(
+ termStrings.toArray(new String[termStrings.size()]),
+ null,
+ countsEnabled ? termCounts.toIntArray() : null);
}
/**
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractTermsQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractTermsQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractTermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -14,7 +14,11 @@
*/
package gate.mimir.search.terms;
+import it.unimi.dsi.fastutil.Arrays;
+import it.unimi.dsi.fastutil.Swapper;
+import it.unimi.dsi.fastutil.ints.AbstractIntComparator;
+
/**
* Base class for term queries.
*/
@@ -24,45 +28,49 @@
* Serialization ID.
*/
private static final long serialVersionUID = -8448110711378800097L;
-
- protected final boolean stringsEnabled;
- protected final boolean countsEnabled;
-
-
/**
* The maximum number of results to be returned.
*/
protected final int limit;
- public AbstractTermsQuery(boolean stringsEnabled, boolean countsEnabled,
- int limit) {
- this.stringsEnabled = stringsEnabled;
- this.countsEnabled = countsEnabled;
+ public AbstractTermsQuery(int limit) {
this.limit = limit;
}
- public AbstractTermsQuery(boolean stringsEnabled, boolean countsEnabled) {
- this(stringsEnabled, countsEnabled, NO_LIMIT);
- }
-
public AbstractTermsQuery() {
- this(false, false, NO_LIMIT);
+ this(NO_LIMIT);
}
-
+
+
/**
- * @return the stringsEnabled
+ * Sorts the arrays inside a {@link TermsResultSet} using the termString for
+ * comparison.
+ * @param trs
*/
- public boolean isStringsEnabled() {
- return stringsEnabled;
+ public static void sortTermsResultSetByTermString(final TermsResultSet trs) {
+ Arrays.quickSort(0, trs.termStrings.length, new AbstractIntComparator() {
+ @Override
+ public int compare(int k1, int k2) {
+ return trs.termStrings[k1].compareTo(trs.termStrings[k2]);
+ }
+ }, new Swapper() {
+ @Override
+ public void swap(int a, int b) {
+ String termString = trs.termStrings[a];
+ trs.termStrings[a] = trs.termStrings[b];
+ trs.termStrings[b] = termString;
+ if(trs.termCounts != null) {
+ int termCount = trs.termCounts[a];
+ trs.termCounts[a] = trs.termCounts[b];
+ trs.termCounts[b] = termCount;
+ }
+ if(trs.termLengths != null) {
+ int termLength = trs.termLengths[a];
+ trs.termLengths[a] = trs.termLengths[b];
+ trs.termLengths[b] = termLength;
+ }
+ }
+ });
}
-
- /**
- * @return the countsEnabled
- */
- public boolean isCountsEnabled() {
- return countsEnabled;
- }
-
-
}
Added:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractWrapperTermsQuery.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractWrapperTermsQuery.java
(rev 0)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractWrapperTermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -0,0 +1,52 @@
+/*
+ * AbstractWrapperTermsQuery.java
+ *
+ * Copyright (c) 2007-2012, The University of Sheffield.
+ *
+ * This file is part of GATE MÃmir (see http://gate.ac.uk/family/mimir.html),
+ * and is free software, licenced under the GNU Lesser General Public License,
+ * Version 3, June 2007 (also included with this distribution as file
+ * LICENCE-LGPL3.html).
+ *
+ * Valentin Tablan, 27 Nov 2012
+ *
+ * $Id$
+ */
+package gate.mimir.search.terms;
+
+import gate.creole.annic.apache.lucene.search.TermQuery;
+
+/**
+ * Abstract base class for {@link TermQuery} implementations that wrap another
+ * {@link TermQuery} instance.
+ */
+public abstract class AbstractWrapperTermsQuery extends AbstractTermsQuery {
+
+ /**
+ * Serialization ID.
+ */
+ private static final long serialVersionUID = -6582029649819116753L;
+
+ public AbstractWrapperTermsQuery(TermsQuery wrappedQuery) {
+ this(wrappedQuery, NO_LIMIT);
+ }
+
+ public AbstractWrapperTermsQuery(TermsQuery wrappedQuery, int limit) {
+ super(limit);
+ this.wrappedQuery = wrappedQuery;
+ }
+
+ /* (non-Javadoc)
+ * @see gate.mimir.search.terms.TermsQuery#isCountsEnabled()
+ */
+ @Override
+ public boolean isCountsEnabled() {
+ return wrappedQuery != null && wrappedQuery.isCountsEnabled();
+ }
+
+ /**
+ * The wrapped wrappedQuery
+ */
+ protected TermsQuery wrappedQuery;
+
+}
Property changes on:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractWrapperTermsQuery.java
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/terms/AndTermsQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/AndTermsQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/AndTermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -18,6 +18,7 @@
import it.unimi.dsi.fastutil.Arrays;
import it.unimi.dsi.fastutil.Swapper;
+import it.unimi.dsi.fastutil.ints.AbstractIntComparator;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntComparator;
import it.unimi.dsi.fastutil.longs.LongArrayList;
@@ -41,6 +42,12 @@
protected TermsQuery[] subQueries;
/**
+ * A boolean flag that is set to true if all sub-queries support counts,
which
+ * allows this query to also provide them.
+ */
+ protected boolean countsAvailable;
+
+ /**
* Constructs a new AND term query.
*
* @param stringsEnabled should terms strings be returned.
@@ -50,10 +57,16 @@
* @param limit the maximum number of terms to be returned.
* @param subQueries the term queries that form the disjunction.
*/
- public AndTermsQuery(boolean stringsEnabled, boolean countsEnabled,
- int limit, TermsQuery... subQueries) {
- super(stringsEnabled, countsEnabled, limit);
+ public AndTermsQuery(int limit, TermsQuery... subQueries) {
+ super(limit);
this.subQueries = subQueries;
+ countsAvailable = true;
+ for(TermsQuery aQuery : subQueries) {
+ if(!aQuery.isCountsEnabled()) {
+ countsAvailable = false;
+ break;
+ }
+ }
}
/* (non-Javadoc)
@@ -62,9 +75,15 @@
@Override
public TermsResultSet execute(QueryEngine engine) throws IOException {
final TermsResultSet[] resSets = new TermsResultSet[subQueries.length];
+ boolean lengthsAvailable = false;
for(int i = 0; i < subQueries.length; i++) {
resSets[i] = subQueries[i].execute(engine);
- if(resSets[i].termIds.length == 0) return TermsResultSet.EMPTY;
+ if(resSets[i].termStrings.length == 0) return TermsResultSet.EMPTY;
+ // this implementation requires that all sub-queries return terms in a
+ // consistent order, so we sort them lexicographically by termString
+ sortTermsResultSetByTermString(resSets[i]);
+ // at least one sub-query must provide lengths
+ if(resSets[i].termLengths != null) lengthsAvailable = true;
}
// optimisation: sort sub-runners by increasing sizes
Arrays.quickSort(0, resSets.length, new IntComparator() {
@@ -74,7 +93,7 @@
}
@Override
public int compare(int k1, int k2) {
- return resSets[k1].termIds.length - resSets[k2].termIds.length;
+ return resSets[k1].termStrings.length -
resSets[k2].termStrings.length;
}
}, new Swapper() {
@Override
@@ -86,35 +105,25 @@
});
// prepare local data
- LongArrayList termIds = new LongArrayList();
- ObjectArrayList<String> termStrings = stringsEnabled ?
- new ObjectArrayList<String>() : null;
- IntArrayList termCounts = countsEnabled ? new IntArrayList() : null;
+ ObjectArrayList<String> termStrings = new ObjectArrayList<String>();
+ IntArrayList termCounts = countsAvailable ? new IntArrayList() : null;
+ IntArrayList termLengths = lengthsAvailable ? new IntArrayList() : null;
// merge the inputs
int[] indexes = new int[subQueries.length]; // initialised with 0s
int currRunner = 0;
- long termId = resSets[currRunner].termIds[indexes[currRunner]];
+ String termString = resSets[currRunner].termStrings[indexes[currRunner]];
top:while(currRunner < subQueries.length) {
currRunner++;
while(currRunner < subQueries.length &&
- resSets[currRunner].termIds[indexes[currRunner]] == termId) {
+
resSets[currRunner].termStrings[indexes[currRunner]].equals(termString)) {
currRunner++;
}
if(currRunner == subQueries.length) {
- // all heads agree
- termIds.add(termId);
- if(stringsEnabled) {
- String termString = null;
- for(int i = 0;
- i < subQueries.length && termString == null;
- i++) {
- if(resSets[i].termStrings != null){
- termString = resSets[i].termStrings[indexes[i]];
- }
- }
- termStrings.add(termString);
- }
- if(countsEnabled) {
+ // all heads agree:
+ // store the term string
+ termStrings.add(termString);
+ // calculate the term count
+ if(countsAvailable) {
int count = 0;
for(int i = 0; i < subQueries.length; i++) {
if(resSets[i].termCounts != null) {
@@ -123,27 +132,41 @@
}
termCounts.add(count);
}
+ // calculate the term length
+ if(lengthsAvailable) {
+ int termLength = -1;
+ for(int i = 0;
+ i < subQueries.length && termLength == -1;
+ i++) {
+ if(resSets[i].termLengths != null){
+ termLength = resSets[i].termLengths[indexes[i]];
+ }
+ }
+ termLengths.add(termLength);
+ }
+
+
// and start fresh
currRunner = 0;
indexes[currRunner]++;
- if(indexes[currRunner] == resSets[currRunner].termIds.length) {
+ if(indexes[currRunner] == resSets[currRunner].termStrings.length) {
// we're out
break top;
} else {
- termId = resSets[currRunner].termIds[indexes[currRunner]];
+ termString = resSets[currRunner].termStrings[indexes[currRunner]];
continue top;
}
} else {
// current runner is wrong
- while(resSets[currRunner].termIds[indexes[currRunner]] < termId) {
+
while(resSets[currRunner].termStrings[indexes[currRunner]].compareTo(termString)
< 0) {
indexes[currRunner]++;
- if(indexes[currRunner] == resSets[currRunner].termIds.length) {
+ if(indexes[currRunner] == resSets[currRunner].termStrings.length) {
// this runner has run out
break top;
} else {
- if(resSets[currRunner].termIds[indexes[currRunner]] > termId) {
+
if(resSets[currRunner].termStrings[indexes[currRunner]].compareTo(termString)
> 0) {
// new term ID
- termId = resSets[currRunner].termIds[indexes[currRunner]];
+ termString =
resSets[currRunner].termStrings[indexes[currRunner]];
currRunner = -1;
continue top;
}
@@ -152,10 +175,17 @@
}
} // top while
// construct the result
- return new TermsResultSet(termIds.toLongArray(),
- stringsEnabled ? termStrings.toArray(new String[termStrings.size()]) :
null,
- null,
- countsEnabled ? termCounts.toIntArray() : null);
+ return new TermsResultSet(
+ termStrings.toArray(new String[termStrings.size()]),
+ lengthsAvailable? termLengths.toIntArray() : null,
+ countsAvailable ? termCounts.toIntArray() : null);
}
-
+
+ /* (non-Javadoc)
+ * @see gate.mimir.search.terms.TermsQuery#isCountsEnabled()
+ */
+ @Override
+ public boolean isCountsEnabled() {
+ return countsAvailable;
+ }
}
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermsQuery.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermsQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -14,11 +14,8 @@
*/
package gate.mimir.search.terms;
-import it.unimi.dsi.big.mg4j.index.BitStreamIndex;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.util.StringMap;
-import it.unimi.dsi.lang.MutableString;
-
+import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import org.apache.log4j.Logger;
@@ -40,8 +37,8 @@
private static final long serialVersionUID = 777418229209857720L;
public AnnotationTermsQuery(AnnotationQuery annotationQuery,
- boolean stringsEnabled, boolean countsEnabled, int limit) {
- super(stringsEnabled, countsEnabled, limit);
+ boolean countsEnabled, int limit) {
+ super(limit);
this.annotationQuery = annotationQuery;
}
@@ -69,35 +66,31 @@
annotationQuery.getConstraints(), engine);
logger.debug(mentions.size() + " mentions obtained in " +
(System.currentTimeMillis() - start) + " ms");
- StringMap<? extends CharSequence> termMap = null;
- Index mg4jIndex = engine.getAnnotationIndex(
- annotationQuery.getAnnotationType()).getIndex();
- if(mg4jIndex instanceof BitStreamIndex) {
- termMap = ((BitStreamIndex)mg4jIndex).termMap;
- } else {
- // this indicates major changes in the underlying MG4J implementation
- throw new IllegalStateException(
- "Underlying MG4J index is not a bitstream index.");
- }
if(mentions.size() > 0) {
- long[] termIds = new long[mentions.size()];
String[] terms = new String[mentions.size()];
int[] lengths = new int[mentions.size()];
int index = 0;
for(Mention m : mentions) {
terms[index] = m.getUri();
lengths[index] = m.getLength();
- // find the term ID
- //use the term processor for the query term
- MutableString mutableString = new MutableString(m.getUri());
- mg4jIndex.termProcessor.processTerm(mutableString);
- termIds[index] = termMap.getLong( mutableString);
index++;
}
- return new TermsResultSet(termIds, terms, lengths, null);
+ return new TermsResultSet(terms, lengths, null);
} else {
return TermsResultSet.EMPTY;
}
}
+
+ /**
+ * This type of terms query does not use the index, all the results are
+ * obtained from the semantic annotation helpers. Because of this, counts are
+ * not available: this method always returns <code>false</code>.
+ */
+ @Override
+ public boolean isCountsEnabled() {
+ return false;
+ }
+
+
}
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentTermsQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentTermsQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentTermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -57,9 +57,9 @@
* @param limit the maximum number of results to be returned.
*/
public DocumentTermsQuery(String indexName,
- IndexType indexType, boolean stringsEnabled, boolean countsEnabled,
+ IndexType indexType, boolean countsEnabled,
int limit, long documentId) {
- super(indexName, indexType, stringsEnabled, countsEnabled, limit);
+ super(indexName, indexType, countsEnabled, limit);
this.documentId = documentId;
}
@@ -83,7 +83,7 @@
*/
public DocumentTermsQuery(String indexName, IndexType indexType,
long documentId) {
- this(indexName, indexType, false, true, NO_LIMIT, documentId);
+ this(indexName, indexType, true, NO_LIMIT, documentId);
}
/**
@@ -107,7 +107,7 @@
*/
public DocumentTermsQuery(String indexName, IndexType indexType, int limit,
long documentId) {
- this(indexName, indexType, false, true, limit, documentId);
+ this(indexName, indexType, true, limit, documentId);
}
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsAndTermsQuery.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsAndTermsQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsAndTermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -39,20 +39,20 @@
protected long[] documentIds;
public DocumentsAndTermsQuery(String indexName, IndexType indexType,
- boolean stringsEnabled, boolean countsEnabled,
+ boolean countsEnabled,
int limit, long... documentIds) {
- super(indexName, indexType, stringsEnabled, countsEnabled, limit);
+ super(indexName, indexType, countsEnabled, limit);
this.documentIds = documentIds;
}
public DocumentsAndTermsQuery(String indexName, IndexType indexType,
long... documentIds) {
- this(indexName, indexType, false, false, NO_LIMIT, documentIds);
+ this(indexName, indexType, false, NO_LIMIT, documentIds);
}
public DocumentsAndTermsQuery(String indexName, IndexType indexType,
int limit, long... documentIds) {
- this(indexName, indexType, false, false, limit, documentIds);
+ this(indexName, indexType, false, limit, documentIds);
}
/* (non-Javadoc)
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsOrTermsQuery.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsOrTermsQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsOrTermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -41,21 +41,21 @@
protected long[] documentIds;
public DocumentsOrTermsQuery(String indexName, IndexType indexType,
- boolean stringsEnabled, boolean countsEnabled,
+ boolean countsEnabled,
int limit, long... documentIds) {
- super(indexName, indexType, stringsEnabled, countsEnabled, limit);
+ super(indexName, indexType, countsEnabled, limit);
this.documentIds = documentIds;
}
public DocumentsOrTermsQuery(String indexName, IndexType indexType,
long... documentIds) {
- this(indexName, indexType, false, false, NO_LIMIT, documentIds);
+ this(indexName, indexType, false, NO_LIMIT, documentIds);
}
public DocumentsOrTermsQuery(String indexName, IndexType indexType, int
limit,
long... documentIds) {
- this(indexName, indexType, false, false, limit, documentIds);
+ this(indexName, indexType, false, limit, documentIds);
}
/* (non-Javadoc)
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/LimitTermsQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/LimitTermsQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/LimitTermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -29,19 +29,15 @@
* {@link SortedTermsQuery} (to change the results order) and
* <strong>then</strong> limit the number of results.
*/
-public class LimitTermsQuery extends AbstractTermsQuery {
+public class LimitTermsQuery extends AbstractWrapperTermsQuery {
/**
* Serialization ID.
*/
private static final long serialVersionUID = -2853628566995944376L;
- protected TermsQuery query;
-
-
public LimitTermsQuery(TermsQuery query, int limit) {
- super(query.isStringsEnabled(), query.isCountsEnabled(), limit);
- this.query = query;
+ super(query, limit);
}
@@ -50,26 +46,24 @@
*/
@Override
public TermsResultSet execute(QueryEngine engine) throws IOException {
- TermsResultSet trs = query.execute(engine);
- if(trs.termIds.length > limit) {
- long[] termIds = new long[limit];
- System.arraycopy(trs.termIds, 0, termIds, 0, limit);
+ TermsResultSet trs = wrappedQuery.execute(engine);
+ if(trs.termStrings != null && trs.termStrings.length > limit) {
+
+ String[] termStrings = new String[limit];
+ System.arraycopy(trs.termStrings, 0, termStrings, 0, limit);
+
int[] termCounts = null;
if(trs.termCounts != null) {
termCounts = new int[limit];
System.arraycopy(trs.termCounts, 0, termCounts, 0, limit);
}
- String[] termStrings = null;
- if(trs.termStrings != null) {
- termStrings = new String[limit];
- System.arraycopy(trs.termStrings, 0, termStrings, 0, limit);
- }
+
int[] termLengths = null;
if(trs.termLengths != null) {
termLengths = new int[limit];
System.arraycopy(trs.termLengths, 0, termLengths, 0, limit);
}
- return new TermsResultSet(termIds, termStrings, termLengths, termCounts);
+ return new TermsResultSet(termStrings, termLengths, termCounts);
} else {
return trs;
}
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/terms/OrTermsQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/OrTermsQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/OrTermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -19,6 +19,7 @@
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongHeapSemiIndirectPriorityQueue;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import it.unimi.dsi.fastutil.objects.ObjectHeapSemiIndirectPriorityQueue;
import java.io.IOException;
@@ -37,6 +38,9 @@
*/
protected TermsQuery[] subQueries;
+ protected boolean countsAvailable;
+
+
/**
* Constructs a new OR terms query.
* @param stringsEnabled should terms strings be returned.
@@ -46,10 +50,16 @@
* @param limit the maximum number of terms to be returned.
* @param subQueries the term queries that form the disjunction.
*/
- public OrTermsQuery(boolean stringsEnabled, boolean countsEnabled,
- int limit, TermsQuery... subQueries) {
- super(stringsEnabled, countsEnabled, limit);
+ public OrTermsQuery(int limit, TermsQuery... subQueries) {
+ super(limit);
this.subQueries = subQueries;
+ countsAvailable = true;
+ for(TermsQuery aQuery : subQueries) {
+ if(!aQuery.isCountsEnabled()) {
+ countsAvailable = false;
+ break;
+ }
+ }
}
/* (non-Javadoc)
@@ -58,68 +68,72 @@
@Override
public TermsResultSet execute(QueryEngine engine) throws IOException {
TermsResultSet[] resSets = new TermsResultSet[subQueries.length];
- long[] currentTerm = new long[resSets.length];
- LongHeapSemiIndirectPriorityQueue queue =
- new LongHeapSemiIndirectPriorityQueue(currentTerm);
+ String[] currentTerm = new String[resSets.length];
+ ObjectHeapSemiIndirectPriorityQueue<String> queue =
+ new ObjectHeapSemiIndirectPriorityQueue<String>(currentTerm);
int[] termIndex = new int[resSets.length];
+ boolean lengthsAvailable = true;
for(int i = 0; i < subQueries.length; i++) {
resSets[i] = subQueries[i].execute(engine);
- if(resSets[i].termIds.length > 0){
+ // this implementation requires that all sub-queries return terms in a
+ // consistent order, so we sort them lexicographically by termString
+ sortTermsResultSetByTermString(resSets[i]);
+ if(resSets[i].termStrings.length > 0){
termIndex[i] = 0;
- currentTerm[i] = resSets[i].termIds[termIndex[i]];
+ currentTerm[i] = resSets[i].termStrings[termIndex[i]];
queue.enqueue(i);
}
+ // we need *all* sub-queries to provide lengths, because we don't know
+ // which one will provide any of the results.
+ if(resSets[i].termLengths == null) lengthsAvailable = false;
}
// prepare local data
- LongArrayList termIds = new LongArrayList();
- ObjectArrayList<String> termStrings = stringsEnabled ?
- new ObjectArrayList<String>() : null;
- IntArrayList termCounts = countsEnabled ? new IntArrayList() : null;
- int front[] = null;
- if(stringsEnabled || countsEnabled) front = new int[resSets.length];
+ ObjectArrayList<String> termStrings = new ObjectArrayList<String>();
+ IntArrayList termLengths = lengthsAvailable ? new IntArrayList() : null;
+ IntArrayList termCounts = countsAvailable ? new IntArrayList() : null;
+ int front[] = new int[resSets.length];
// enumerate all terms
top:while(!queue.isEmpty()) {
int first = queue.first();
- long termId = resSets[first].termIds[termIndex[first]];
- termIds.add(termId);
- if(countsEnabled || stringsEnabled) {
+ String termString = resSets[first].termStrings[termIndex[first]];
+ termStrings.add(termString);
+ if(countsAvailable) {
int frontSize = queue.front(front);
- String termString = null;
int count = 0;
for(int i = 0; i < frontSize; i++) {
int subRunnerId = front[i];
- if(stringsEnabled &&
- termString == null &&
- resSets[subRunnerId].termStrings != null) {
- termString =
resSets[subRunnerId].termStrings[termIndex[subRunnerId]];
- }
- if(resSets[subRunnerId].termCounts != null) {
- count += resSets[subRunnerId].termCounts[termIndex[subRunnerId]];
- }
+ count += resSets[subRunnerId].termCounts[termIndex[subRunnerId]];
}
- if(stringsEnabled) termStrings.add(termString);
- if(countsEnabled) termCounts.add(count);
+ termCounts.add(count);
}
// consume all equal terms
- while(resSets[first].termIds[termIndex[first]] == termId) {
+ while(resSets[first].termStrings[termIndex[first]].equals(termString)) {
// advance this subRunner
termIndex[first]++;
- if(termIndex[first] == resSets[first].termIds.length) {
+ if(termIndex[first] == resSets[first].termStrings.length) {
// 'first' is out
queue.dequeue();
if(queue.isEmpty()) break top;
} else {
- currentTerm[first] = resSets[first].termIds[termIndex[first]];
+ currentTerm[first] = resSets[first].termStrings[termIndex[first]];
queue.changed();
}
first = queue.first();
}
}
// construct the result
- return new TermsResultSet(termIds.toLongArray(),
- stringsEnabled ? termStrings.toArray(new String[termStrings.size()]) :
null,
- null,
- countsEnabled ? termCounts.toIntArray() : null);
+ return new TermsResultSet(
+ termStrings.toArray(new String[termStrings.size()]),
+ lengthsAvailable ? termLengths.toIntArray() : null,
+ countsAvailable ? termCounts.toIntArray() : null);
}
+
+ /* (non-Javadoc)
+ * @see gate.mimir.search.terms.TermsQuery#isCountsEnabled()
+ */
+ @Override
+ public boolean isCountsEnabled() {
+ return countsAvailable;
+ }
}
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/SortedTermsQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/SortedTermsQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/SortedTermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -26,7 +26,7 @@
* A wrapper for another terms query that simply sorts the returned terms based
* on some criteria.
*/
-public class SortedTermsQuery extends AbstractTermsQuery {
+public class SortedTermsQuery extends AbstractWrapperTermsQuery {
/**
* Serialization ID.
@@ -35,12 +35,7 @@
public static enum SortOrder {
- /** Sort by ID, ascending. */
- ID,
- /** Sort by ID (descending) */
- ID_DESC,
-
/** Sort by counts */
COUNT,
@@ -52,8 +47,6 @@
STRING_DESC
}
- protected TermsQuery query;
-
protected SortOrder[] criteria;
/**
@@ -65,7 +58,7 @@
* </ul>
*/
public static final SortOrder[] DEFAULT_SORT_CRITERIA = new SortOrder[]
- { SortOrder.COUNT_DESC, SortOrder.STRING, SortOrder.ID };
+ { SortOrder.COUNT_DESC, SortOrder.STRING};
/**
* Creates a new sorted terms query, wrapping the provided query, and using
@@ -74,8 +67,7 @@
* @param criteria
*/
public SortedTermsQuery(TermsQuery query, SortOrder... criteria) {
- super(query.isStringsEnabled(), query.isCountsEnabled(), NO_LIMIT);
- this.query = query;
+ super(query);
this.criteria = criteria;
}
@@ -93,8 +85,8 @@
*/
@Override
public TermsResultSet execute(QueryEngine engine) throws IOException {
- final TermsResultSet trs = query.execute(engine);
- Arrays.quickSort(0, trs.termIds.length, new IntComparator() {
+ final TermsResultSet trs = wrappedQuery.execute(engine);
+ Arrays.quickSort(0, trs.termStrings.length, new IntComparator() {
@Override
public int compare(Integer o1, Integer o2) {
return compare(o1.intValue(), o2.intValue());
@@ -105,12 +97,6 @@
int retval = 0;
for(SortOrder crit: criteria) {
switch(crit) {
- case ID:
- retval = Long.signum(trs.termIds[k1] - trs.termIds[k2]);
- break;
- case ID_DESC:
- retval = -Long.signum(trs.termIds[k1] - trs.termIds[k2]);
- break;
case COUNT:
if(trs.termCounts != null){
retval = trs.termCounts[k1] - trs.termCounts[k2];
@@ -143,9 +129,6 @@
}, new Swapper() {
@Override
public void swap(int a, int b) {
- long termId = trs.termIds[a];
- trs.termIds[a] = trs.termIds[b];
- trs.termIds[b] = termId;
if(trs.termStrings != null) {
String termString = trs.termStrings[a];
trs.termStrings[a] = trs.termStrings[b];
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsQuery.java
2012-11-27 16:06:51 UTC (rev 16340)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsQuery.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -20,11 +20,14 @@
import gate.mimir.search.QueryEngine;
/**
- * A query that returns terms. The terms returned must be sorted in ascending
- * order of their term ID.
+ * A query that returns terms rather than documents.
*/
public interface TermsQuery extends Serializable {
+ /**
+ * Constant used to mark that no limit is applied to the number of returned
+ * results.
+ */
public static final int NO_LIMIT = Integer.MAX_VALUE;
/**
@@ -37,15 +40,9 @@
public TermsResultSet execute(QueryEngine engine) throws IOException;
/**
- * Does this query return term strings?
+ * Does this terms query return term counts?
* @return
*/
- public boolean isStringsEnabled();
-
- /**
- * Does this query return term counts?
- * @return
- */
public boolean isCountsEnabled();
}
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsResultSet.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsResultSet.java
2012-11-27 16:06:51 UTC (rev 16340)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/TermsResultSet.java
2012-11-27 16:22:45 UTC (rev 16341)
@@ -18,6 +18,9 @@
/**
* Class representing the results of a {@link TermsQuery}.
+ * A terms result set is a set of terms, represented by their
+ * {@link #termStrings}. Optionally {@link #termCounts}, and
+ * {@link #termLengths} may also be available.
*/
public class TermsResultSet implements Serializable {
@@ -26,11 +29,6 @@
*/
private static final long serialVersionUID = -7722325563637139625L;
- /**
- * The term IDs, as retrieved from the index. Array parallel with
- * {@link #termStrings} and {@link #termCounts}.
- */
- public final long[] termIds;
/**
* The lengths (number of tokens) for the terms.
@@ -50,10 +48,9 @@
*/
public final int[] termCounts;
- public TermsResultSet(long[] termIds, String[] termStrings,int[]
termLengths,
+ public TermsResultSet(String[] termStrings,int[] termLengths,
int[] termCounts) {
super();
- this.termIds = termIds;
this.termStrings = termStrings;
this.termLengths = termLengths;
this.termCounts = termCounts;
@@ -63,6 +60,6 @@
* Constant representing the empty result set.
*/
public static final TermsResultSet EMPTY = new TermsResultSet(
- new long[] {}, new String[]{}, new int[] {}, new int[]{});
+ new String[]{}, new int[] {}, new int[]{});
}
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Monitor your physical, virtual and cloud infrastructure from a single
web console. Get in-depth insight into apps, servers, databases, vmware,
SAP, cloud infrastructure, etc. Download 30-day Free Trial.
Pricing starts from $795 for 25 servers or applications!
http://p.sf.net/sfu/zoho_dev2dev_nov
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
