Revision: 14589
http://gate.svn.sourceforge.net/gate/?rev=14589&view=rev
Author: valyt
Date: 2011-11-22 17:57:48 +0000 (Tue, 22 Nov 2011)
Log Message:
-----------
- new QueryRunner API,
- RankingQueryRunnerImpl(*) now works,
- old QueryRunner interface renamed to QueryRunnerMk1 for now.
(*) this is a working name; it will probably revert to being called simply
QueryRunnerImpl.
Modified Paths:
--------------
mimir/trunk/mimir-core/src/gate/mimir/search/FederatedQueryRunner.java
mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunner.java
mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunnerImpl.java
mimir/trunk/mimir-core/src/gate/mimir/search/RankingQueryRunnerImpl.java
Added Paths:
-----------
mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunnerMk1.java
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/FederatedQueryRunner.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/FederatedQueryRunner.java
2011-11-22 13:54:51 UTC (rev 14588)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/FederatedQueryRunner.java
2011-11-22 17:57:48 UTC (rev 14589)
@@ -31,14 +31,14 @@
/**
* QueryRunner implementation for querying federated indexes.
*/
-public class FederatedQueryRunner implements QueryRunner {
+public class FederatedQueryRunner implements QueryRunnerMk1 {
/**
* The QueryRunners running the sub-queries of this federated
* query. Typically this would be one runner for each sub-index,
* all running the same query string.
*/
- protected QueryRunner[] subRunners;
+ protected QueryRunnerMk1[] subRunners;
/**
* A pointer to the most recent DocumentData object in the
@@ -124,7 +124,7 @@
* Create a federated query runner that delegates to the specified
* sub-query runners.
*/
- public FederatedQueryRunner(QueryRunner[] subRunners) {
+ public FederatedQueryRunner(QueryRunnerMk1[] subRunners) {
super();
this.subRunners = subRunners;
// Java doesn't allow you to create an array of generic elements
@@ -453,7 +453,7 @@
* Tell all the sub-query runners to getMoreHits.
*/
public void getMoreHits() throws IOException {
- for(QueryRunner r : subRunners) {
+ for(QueryRunnerMk1 r : subRunners) {
r.getMoreHits();
}
}
@@ -495,7 +495,7 @@
* Pass on the stage timeout to all the sub-runners.
*/
public void setStageTimeout(int timeout) throws IOException {
- for(QueryRunner r : subRunners) {
+ for(QueryRunnerMk1 r : subRunners) {
r.setStageTimeout(timeout);
}
}
@@ -504,7 +504,7 @@
* Close all the sub-query runners.
*/
public void close() throws IOException {
- for(QueryRunner r : subRunners) {
+ for(QueryRunnerMk1 r : subRunners) {
r.close();
}
}
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
2011-11-22 13:54:51 UTC (rev 14588)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
2011-11-22 17:57:48 UTC (rev 14589)
@@ -182,7 +182,7 @@
* runners when the query engine itself is closed (thus releasing all open
* files).
*/
- private List<QueryRunner> activeQueryRunners;
+ private List<QueryRunnerMk1> activeQueryRunners;
/**
* @return the indexDir
@@ -363,7 +363,7 @@
readDeletedDocs();
activeQueryRunners = Collections.synchronizedList(
- new ArrayList<QueryRunner>());
+ new ArrayList<QueryRunnerMk1>());
} catch(FileNotFoundException e) {
throw new IndexException("File not found!", e);
} catch(IOException e) {
@@ -383,10 +383,10 @@
* @throws IOException
* if the index files cannot be accessed.
*/
- public QueryRunner getQueryRunner(QueryNode query) throws IOException {
+ public QueryRunnerMk1 getQueryRunner(QueryNode query) throws IOException {
logger.info("Executing query: " + query.toString());
QueryExecutor qExecutor = query.getQueryExecutor(this);
- QueryRunner qRunner = new QueryRunnerImpl(qExecutor);
+ QueryRunnerMk1 qRunner = new QueryRunnerImpl(qExecutor);
activeQueryRunners.add(qRunner);
return qRunner;
}
@@ -395,7 +395,7 @@
* Notifies the QueryEngine that the given QueryRunner has been closed.
* @param qRunner
*/
- public void releaseQueryRunner(QueryRunner qRunner) {
+ public void releaseQueryRunner(QueryRunnerMk1 qRunner) {
activeQueryRunners.remove(qRunner);
}
@@ -411,7 +411,7 @@
* @throws ParseException
* if the string provided for the query cannot be parsed.
*/
- public QueryRunner getQueryRunner(String query) throws IOException,
+ public QueryRunnerMk1 getQueryRunner(String query) throws IOException,
ParseException {
logger.info("Executing query: " + query.toString());
QueryNode qNode =
@@ -643,8 +643,8 @@
*/
public void close() {
// close all active query runners
- List<QueryRunner> runnersCopy = new
ArrayList<QueryRunner>(activeQueryRunners);
- for(QueryRunner aRunner : runnersCopy) {
+ List<QueryRunnerMk1> runnersCopy = new
ArrayList<QueryRunnerMk1>(activeQueryRunners);
+ for(QueryRunnerMk1 aRunner : runnersCopy) {
try {
logger.debug("Closing query runner: " + aRunner.toString());
aRunner.close();
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunner.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunner.java
2011-11-22 13:54:51 UTC (rev 14588)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunner.java
2011-11-22 17:57:48 UTC (rev 14589)
@@ -8,7 +8,7 @@
* Version 3, June 2007 (also included with this distribution as file
* LICENCE-LGPL3.html).
*
- * Valentin tablan, 16 Dec 2009
+ * Valentin Tablan, 22 Nov 2011
*
* $Id$
*/
@@ -25,241 +25,157 @@
import java.util.Map;
import java.util.Set;
+
/**
- * A QueryRunner object can be used to execute a query in a separate thread.
The
- * query execution will be performed in stages, each being limited to a given
- * number of maximum hits or a set timeout. During the query execution
- * statistics are made available about the number of hits currently obtained
for
- * each document. The hits already collected can be accessed in a random
- * fashion.
+ * A QueryRunner is used to manage the execution of the query (supplied as a
+ * {@link QueryExecutor}). Implementations may use a background thread to
+ * pre-fetch data which they then make available through the public API.
*
- * N.B. Documents are referred to in two different manners:
- * <ul>
- * <li>by documentId: the ID associated with the document at indexing time,
- * which never changes</li>
- * <li>by index: the index in the list of documents found to contain
hits.</li>
- * </ul>
- * You can tell which addressing scheme is used based on the of the parameter.
+ * All references to documents are made by rank, i.e. the position of the
+ * document in the list of results.
*
- * Implementations of this interface must be thread-safe!
+ * Unless there is a good reason not to (e.g. results ranking), the documents
+ * will be returned in increasing documentID order.
+ *
+ * QueryRunners that perform ranking will re-order the result list so that
+ * documents are returned in decreasing score order.
*/
public interface QueryRunner {
-
/**
- * The default number of hits obtained in one search stage.
+ * Gets the number of result documents.
+ * @return <code>-1</code> if the search has not yet completed, the total
+ * number of result document otherwise.
*/
- public static final int DEFAULT_MAX_HITS = 1000000;
-
-
+ public int getDocumentsCount();
+
/**
- * The default maximum amount of time (in milliseconds) used for one search
- * stage.
+ * Gets the number of result documents found so far. After the search
+ * completes, the result returned by this call is identical to that of
+ * {@link #getDocumentsCount()}.
+ * @return the number of result documents known so far.
*/
- public static final int DEFAULT_TIMEOUT = 30000;
-
+ public int getCurrentDocumentsCount();
+
/**
- * Starts a new search stage, obtaining more results. Executes the query
- * asynchronously until a set number of hits are obtained, or a given
- * period is elapsed.
- *
- * If the query execution is currently in progress (i.e. a previous search
- * stage has not yet finished), this call will have no effect. If a previous
- * query execution stage has ended, this call will cause the search to
- * restart, accumulating more hits. If the search had finished, this call
- * will have no effect.
- *
- * @see #setMaxHits(int)
- * @see #setTimeout(int)
- * @see #isActive()
- * @see #isComplete()
- *
- * @throws IOException
- */
- public void getMoreHits() throws IOException;
-
- /**
- * Sets the maximum number of hits to be obtain in one search stage. If not
- * set, the {@link #DEFAULT_MAX_HITS default value} is used.
- * If the value provided is negative, then no limit is imposed on the number
- * of hits obtained.
- *
- * @param the maximum number of hits to be obtained by the next call to
- * {@link #getMoreHits()}.
- * @throws IOException if the communication with the query runner
- * implementation fails.
- */
- public void setStageMaxHits(int maxHits) throws IOException;
-
- /**
- * Sets the maximum amount of time to be used for one search stage. If not
- * set, the {@link #DEFAULT_TIMEOUT default value} is used. If the value
- * provided is negative, then no limit is imposed on the amount of time
spent.
- *
- * @param timeout the maximum amount of time (in milliseconds) to be used by
- * the next call to {@link #getMoreHits()}.
- * @throws IOException if the communication with the query runner
- * implementation fails.
- */
- public void setStageTimeout(int timeout) throws IOException;
-
- /**
- * Gets the number of hits obtained so far.
- * This number may increase at any time if the query is currently
- * {@link #isActive() active}.
- * @return an int value, representing the number of hits.
- */
- public int getHitsCount();
-
- /**
- * Gets the number of distinct documents found to contain hits so far.
- * This number may increase at any time if the query is currently
- * {@link #isActive() active}.
- * @return an int value, representing the number of distinct documents.
- */
- public int getDocumentsCount();
-
- /**
- * Gets the ID of a document found to contain hits.
- * @param index the index of the desired document in the list of documents.
+ * Gets the ID of a result document.
+ * @param rank the index of the desired document in the list of documents.
* This should be a value between 0 and {@link #getDocumentsCount()} -1.
*
+ * If the requested document position has not yet been ranked (i.e. we know
+ * there is a document at that position, but we don't yet know which one)
then
+ * the necessary ranking is performed before this method returns.
+ *
* @return an int value, representing the ID of the requested document.
* @throws IndexOutOfBoundsException is the index provided is less than
zero,
* or greater than {@link #getDocumentsCount()} -1.
+ * @throws IOException
*/
- public int getDocumentID(int index) throws IndexOutOfBoundsException;
-
+ public int getDocumentID(int rank) throws IndexOutOfBoundsException,
+ IOException;
+
/**
- * Gets the number of hits for one of the documents found to contain hits.
- * Note that for the <i>last</i> document this number is a lower bound -
there
- * may still be more hits to be found in this document (unless
- * {@link #isComplete()} returns true).
- *
- * @param index the index of the desired document in the list of documents.
+ * Retrieves the hits within a given result document.
+ * @param rank the index of the desired document in the list of documents.
* This should be a value between 0 and {@link #getDocumentsCount()} -1.
- *
- * @return an int value, representing the number of hits on the requested
- * document.
- * @throws IndexOutOfBoundsException is the index provided is less than
zero,
- * or greater than {@link #getDocumentsCount()} -1.
+ *
+ * This method call waits until the requested data is available before
+ * returning (document hits are being collected by a background thread).
+ *
+ * @return
+ * @throws IOException
+ * @throws IndexOutOfBoundsException
*/
- public int getDocumentHitsCount(int index) throws IndexOutOfBoundsException;
-
+ public List<Binding> getDocumentHits(int rank)
+ throws IndexOutOfBoundsException, IOException;
+
/**
- * Gets a subset of the hits obtained so far. If the set of hits requested
- * does not exist (e.g. if the startIndex is too large) then an empty
- * list will be returned.
+ * Gets a segment of the document text for a given document.
+ * @param rank the rank of the requested document.
+ * @param termPosition the first term requested.
+ * @param length the number of terms requested.
+ * @return two parallel String arrays, one containing term text, the other
+ * containing the spaces in between. The first term is results[0][0], the
+ * space following it is results[1][0], etc.
*
- * @param startIndex the index of the first requested hit.
- * @param hitCount the maximum number of hits to be returned (fewer hits
will
- * be returned if there aren't enough available).
- * @return a list of maximum hitCount hits.
- * @throws IndexOutOfBoundsException if startIndex is negative.
+ * @throws IndexException
+ * @throws IndexOutOfBoundsException
+ * @throws IOException
*/
- public List<Binding> getHits(int startIndex, int hitCount)
- throws IndexOutOfBoundsException;
-
+ public String[][] getDocumentText(int rank, int termPosition, int length)
+ throws IndexException, IndexOutOfBoundsException, IOException;
+
/**
- * Gets all the hits for a given document.
- * @param documentId the ID of the document for which the hits are being
- * requested.
- * @return a list of hits
+ * Obtains the URI for a given document.
+ * @param rank the rank for the requested document.
+ * @return the URI provided at indexing time for the document.
+ * @throws IndexException
* @throws IndexOutOfBoundsException
+ * @throws IOException
*/
- public List<Binding> getHitsForDocument(int documentId)
- throws IndexOutOfBoundsException;
-
+ public String getDocumentURI(int rank) throws IndexException,
+ IndexOutOfBoundsException, IOException;
+
/**
- * Render the content of the given document, with the hits for this query
- * highlighted.
- * @param documentId
- * @param out
- * @throws IOException if the output cannot be written to.
- * @throws IndexException if no document renderer is available.
+ * Obtains the title for a given document.
+ * @param rank the rank of the requested document.
+ * @return the document title (provided at indexing time).
+ * @throws IndexException
+ * @throws IndexOutOfBoundsException
+ * @throws IOException
*/
- public void renderDocument(int documentId, Appendable out) throws
IOException,
- IndexException;
-
-
+ public String getDocumentTitle(int rank) throws IndexException,
+ IndexOutOfBoundsException, IOException;
+
/**
- * Obtains the URI for a given document (specified by its ID).
- * @param documentID
- * @return
- * @throws IndexException if the document URI cannot be retrieved from the
- * index.
- */
- public String getDocumentURI(int documentID) throws IndexException;
-
- /**
- * Obtains the title for a given document (specified by its ID).
- * @param documentID
- * @return
- * @throws IndexException if the document title cannot be retrieved from the
- * index.
- */
- public String getDocumentTitle(int documentID) throws IndexException;
-
- /**
* Obtains an arbitrary document metadata field from the stored document
data.
* {@link DocumentMetadataHelper}s used at indexing time can add arbitrary
* {@link Serializable} values as metadata fields for the documents being
- * indexed. This method is used at search time to retrieve those values.
- *
- * @param docID the ID of document for which the metadata is sought.
- * @param fieldName the name of the metadata fields to be obtained
- * @return the de-serialised value stored at indexing time for the given
- * field name and document.
+ * indexed. This method is used at search time to retrieve those values.
+ *
+ * @param rank the rank for the requested document.
+ * @param fieldName the field name for which the value is sought.
+ * @return
* @throws IndexException
- */
- public Serializable getDocumentMetadataField(int docID, String fieldName)
- throws IndexException;
-
+ * @throws IndexOutOfBoundsException
+ * @throws IOException
+ */
+ public Serializable getDocumentMetadataField(int rank, String fieldName)
+ throws IndexException, IndexOutOfBoundsException, IOException;
+
/**
* Obtains a set of arbitrary document metadata fields from the stored
* document data.
* {@link DocumentMetadataHelper}s used at indexing time can add arbitrary
* {@link Serializable} values as metadata fields for the documents being
- * indexed. This method is used at search time to retrieve those values.
- *
- * @param docID the ID of document for which the metadata is sought.
- * @param fieldNames the names of the metadata fields to be obtained
- * @return the de-serialised values stored at indexing time for the given
- * field names and document (as a Map from field name to filed value).
+ * indexed. This method is used at search time to retrieve those values.
+ *
+ * @param rank the rank for the requested document.
+ * @param fieldNames the names of the metadata fields for which the values
are
+ * requested.
+ * @return a {@link Map} linking field names with their values.
* @throws IndexException
- */
- public Map<String, Serializable> getDocumentMetadataFields(int docID,
- Set<String> fieldNames) throws IndexException;
-
- /**
- * Gets a segment of the document text for a given document.
- * @param documentID
- * @param startToken
- * @param length
- * @return
- * @throws IndexException if the document text cannot be retrieved from the
- * index.
+ * @throws IndexOutOfBoundsException
+ * @throws IOException
*/
- public String[][] getDocumentText(int documentID, int termPosition,
- int length) throws IndexException;
-
+ public Map<String, Serializable> getDocumentMetadataFields(int rank,
+ Set<String> fieldNames) throws IndexException,
+ IndexOutOfBoundsException, IOException;
+
/**
- * Checks whether a search stage is currently active.
- * @return <code>true</code> iff more hits are currently being sought (a
- * search stage has started and not finished yet)
+ * Render the content of the given document, with the hits for this query
+ * highlighted.
+ *
+ * @param rank the rank for the requested document.
+ * @param out an {@link Appendable} to which the output is written.
+ * @throws IOException
+ * @throws IndexException
*/
- public boolean isActive();
-
+ public void renderDocument(int rank, Appendable out) throws IOException,
+ IndexException;
+
/**
- * Checks whether all the available hits have been obtained. When this
returns
- * <code>true</code>, further calls to {@link #getMoreHits()} will have no
- * effect.
- * @return <code>true</code> after all the possible hits have been obtained.
- */
- public boolean isComplete();
-
- /**
* Closes this {@link QueryExecutor} and releases all resources used.
- * @throws IOException if the index files cannot be accessed.
+ * @throws IOException
*/
public void close() throws IOException;
-}
+}
\ No newline at end of file
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunnerImpl.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunnerImpl.java
2011-11-22 13:54:51 UTC (rev 14588)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunnerImpl.java
2011-11-22 17:57:48 UTC (rev 14589)
@@ -41,7 +41,7 @@
* @author valyt
*
*/
-public class QueryRunnerImpl implements QueryRunner {
+public class QueryRunnerImpl implements QueryRunnerMk1 {
private class SearchStageRunner implements Runnable{
Copied: mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunnerMk1.java (from
rev 14588, mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunner.java)
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunnerMk1.java
(rev 0)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/QueryRunnerMk1.java
2011-11-22 17:57:48 UTC (rev 14589)
@@ -0,0 +1,265 @@
+/*
+ * QueryRunner.java
+ *
+ * Copyright (c) 2007-2011, The University of Sheffield.
+ *
+ * This file is part of GATE MÃmir (see http://gate.ac.uk/family/mimir.html),
+ * and is free software, licenced under the GNU Lesser General Public License,
+ * Version 3, June 2007 (also included with this distribution as file
+ * LICENCE-LGPL3.html).
+ *
+ * Valentin tablan, 16 Dec 2009
+ *
+ * $Id$
+ */
+package gate.mimir.search;
+
+import gate.mimir.DocumentMetadataHelper;
+import gate.mimir.index.IndexException;
+import gate.mimir.search.query.Binding;
+import gate.mimir.search.query.QueryExecutor;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A QueryRunner object can be used to execute a query in a separate thread.
The
+ * query execution will be performed in stages, each being limited to a given
+ * number of maximum hits or a set timeout. During the query execution
+ * statistics are made available about the number of hits currently obtained
for
+ * each document. The hits already collected can be accessed in a random
+ * fashion.
+ *
+ * N.B. Documents are referred to in two different manners:
+ * <ul>
+ * <li>by documentId: the ID associated with the document at indexing time,
+ * which never changes</li>
+ * <li>by index: the index in the list of documents found to contain
hits.</li>
+ * </ul>
+ * You can tell which addressing scheme is used based on the of the parameter.
+ *
+ * Implementations of this interface must be thread-safe!
+ */
+public interface QueryRunnerMk1 {
+
+ /**
+ * The default number of hits obtained in one search stage.
+ */
+ public static final int DEFAULT_MAX_HITS = 1000000;
+
+
+ /**
+ * The default maximum amount of time (in milliseconds) used for one search
+ * stage.
+ */
+ public static final int DEFAULT_TIMEOUT = 30000;
+
+ /**
+ * Starts a new search stage, obtaining more results. Executes the query
+ * asynchronously until a set number of hits are obtained, or a given
+ * period is elapsed.
+ *
+ * If the query execution is currently in progress (i.e. a previous search
+ * stage has not yet finished), this call will have no effect. If a previous
+ * query execution stage has ended, this call will cause the search to
+ * restart, accumulating more hits. If the search had finished, this call
+ * will have no effect.
+ *
+ * @see #setMaxHits(int)
+ * @see #setTimeout(int)
+ * @see #isActive()
+ * @see #isComplete()
+ *
+ * @throws IOException
+ */
+ public void getMoreHits() throws IOException;
+
+ /**
+ * Sets the maximum number of hits to be obtain in one search stage. If not
+ * set, the {@link #DEFAULT_MAX_HITS default value} is used.
+ * If the value provided is negative, then no limit is imposed on the number
+ * of hits obtained.
+ *
+ * @param the maximum number of hits to be obtained by the next call to
+ * {@link #getMoreHits()}.
+ * @throws IOException if the communication with the query runner
+ * implementation fails.
+ */
+ public void setStageMaxHits(int maxHits) throws IOException;
+
+ /**
+ * Sets the maximum amount of time to be used for one search stage. If not
+ * set, the {@link #DEFAULT_TIMEOUT default value} is used. If the value
+ * provided is negative, then no limit is imposed on the amount of time
spent.
+ *
+ * @param timeout the maximum amount of time (in milliseconds) to be used by
+ * the next call to {@link #getMoreHits()}.
+ * @throws IOException if the communication with the query runner
+ * implementation fails.
+ */
+ public void setStageTimeout(int timeout) throws IOException;
+
+ /**
+ * Gets the number of hits obtained so far.
+ * This number may increase at any time if the query is currently
+ * {@link #isActive() active}.
+ * @return an int value, representing the number of hits.
+ */
+ public int getHitsCount();
+
+ /**
+ * Gets the number of distinct documents found to contain hits so far.
+ * This number may increase at any time if the query is currently
+ * {@link #isActive() active}.
+ * @return an int value, representing the number of distinct documents.
+ */
+ public int getDocumentsCount();
+
+ /**
+ * Gets the ID of a document found to contain hits.
+ * @param index the index of the desired document in the list of documents.
+ * This should be a value between 0 and {@link #getDocumentsCount()} -1.
+ *
+ * @return an int value, representing the ID of the requested document.
+ * @throws IndexOutOfBoundsException is the index provided is less than
zero,
+ * or greater than {@link #getDocumentsCount()} -1.
+ */
+ public int getDocumentID(int index) throws IndexOutOfBoundsException;
+
+ /**
+ * Gets the number of hits for one of the documents found to contain hits.
+ * Note that for the <i>last</i> document this number is a lower bound -
there
+ * may still be more hits to be found in this document (unless
+ * {@link #isComplete()} returns true).
+ *
+ * @param index the index of the desired document in the list of documents.
+ * This should be a value between 0 and {@link #getDocumentsCount()} -1.
+ *
+ * @return an int value, representing the number of hits on the requested
+ * document.
+ * @throws IndexOutOfBoundsException is the index provided is less than
zero,
+ * or greater than {@link #getDocumentsCount()} -1.
+ */
+ public int getDocumentHitsCount(int index) throws IndexOutOfBoundsException;
+
+ /**
+ * Gets a subset of the hits obtained so far. If the set of hits requested
+ * does not exist (e.g. if the startIndex is too large) then an empty
+ * list will be returned.
+ *
+ * @param startIndex the index of the first requested hit.
+ * @param hitCount the maximum number of hits to be returned (fewer hits
will
+ * be returned if there aren't enough available).
+ * @return a list of maximum hitCount hits.
+ * @throws IndexOutOfBoundsException if startIndex is negative.
+ */
+ public List<Binding> getHits(int startIndex, int hitCount)
+ throws IndexOutOfBoundsException;
+
+ /**
+ * Gets all the hits for a given document.
+ * @param documentId the ID of the document for which the hits are being
+ * requested.
+ * @return a list of hits
+ * @throws IndexOutOfBoundsException
+ */
+ public List<Binding> getHitsForDocument(int documentId)
+ throws IndexOutOfBoundsException;
+
+ /**
+ * Render the content of the given document, with the hits for this query
+ * highlighted.
+ * @param documentId
+ * @param out
+ * @throws IOException if the output cannot be written to.
+ * @throws IndexException if no document renderer is available.
+ */
+ public void renderDocument(int documentId, Appendable out) throws
IOException,
+ IndexException;
+
+
+ /**
+ * Obtains the URI for a given document (specified by its ID).
+ * @param documentID
+ * @return
+ * @throws IndexException if the document URI cannot be retrieved from the
+ * index.
+ */
+ public String getDocumentURI(int documentID) throws IndexException;
+
+ /**
+ * Obtains the title for a given document (specified by its ID).
+ * @param documentID
+ * @return
+ * @throws IndexException if the document title cannot be retrieved from the
+ * index.
+ */
+ public String getDocumentTitle(int documentID) throws IndexException;
+
+ /**
+ * Obtains an arbitrary document metadata field from the stored document
data.
+ * {@link DocumentMetadataHelper}s used at indexing time can add arbitrary
+ * {@link Serializable} values as metadata fields for the documents being
+ * indexed. This method is used at search time to retrieve those values.
+ *
+ * @param docID the ID of document for which the metadata is sought.
+ * @param fieldName the name of the metadata fields to be obtained
+ * @return the de-serialised value stored at indexing time for the given
+ * field name and document.
+ * @throws IndexException
+ */
+ public Serializable getDocumentMetadataField(int docID, String fieldName)
+ throws IndexException;
+
+ /**
+ * Obtains a set of arbitrary document metadata fields from the stored
+ * document data.
+ * {@link DocumentMetadataHelper}s used at indexing time can add arbitrary
+ * {@link Serializable} values as metadata fields for the documents being
+ * indexed. This method is used at search time to retrieve those values.
+ *
+ * @param docID the ID of document for which the metadata is sought.
+ * @param fieldNames the names of the metadata fields to be obtained
+ * @return the de-serialised values stored at indexing time for the given
+ * field names and document (as a Map from field name to filed value).
+ * @throws IndexException
+ */
+ public Map<String, Serializable> getDocumentMetadataFields(int docID,
+ Set<String> fieldNames) throws IndexException;
+
+ /**
+ * Gets a segment of the document text for a given document.
+ * @param documentID
+ * @param startToken
+ * @param length
+ * @return
+ * @throws IndexException if the document text cannot be retrieved from the
+ * index.
+ */
+ public String[][] getDocumentText(int documentID, int termPosition,
+ int length) throws IndexException;
+
+ /**
+ * Checks whether a search stage is currently active.
+ * @return <code>true</code> iff more hits are currently being sought (a
+ * search stage has started and not finished yet)
+ */
+ public boolean isActive();
+
+ /**
+ * Checks whether all the available hits have been obtained. When this
returns
+ * <code>true</code>, further calls to {@link #getMoreHits()} will have no
+ * effect.
+ * @return <code>true</code> after all the possible hits have been obtained.
+ */
+ public boolean isComplete();
+
+ /**
+ * Closes this {@link QueryExecutor} and releases all resources used.
+ * @throws IOException if the index files cannot be accessed.
+ */
+ public void close() throws IOException;
+}
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/RankingQueryRunnerImpl.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/RankingQueryRunnerImpl.java
2011-11-22 13:54:51 UTC (rev 14588)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/RankingQueryRunnerImpl.java
2011-11-22 17:57:48 UTC (rev 14589)
@@ -15,6 +15,8 @@
*/
package gate.mimir.search;
+import gate.mimir.DocumentMetadataHelper;
+import gate.mimir.index.IndexException;
import gate.mimir.search.query.Binding;
import gate.mimir.search.query.QueryExecutor;
import gate.mimir.search.query.QueryNode;
@@ -27,9 +29,13 @@
import it.unimi.dsi.fastutil.objects.ObjectList;
import java.io.IOException;
+import java.io.Serializable;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.SortedMap;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Future;
@@ -43,21 +49,32 @@
* A QueryRunner implementation that can perform ranking.
* This query runner has two modes of functioning: ranking and non-ranking,
* depending on whether a {@link MimirScorer} is provided during construction
- * or not.
+ * or not.
+ * All documents are referred to using their rank (i.e. position in the list
of
+ * results). When working in non-ranking mode, ranking order is the same as
+ * document ID order.
*/
-public class RankingQueryRunnerImpl {
+public class RankingQueryRunnerImpl implements QueryRunner {
- private static final Runnable NO_MORE_JOBS = new Runnable(){
+
+ /**
+ * Constant used as a flag to mark then of a list of tasks.
+ */
+ private static final Runnable NO_MORE_TASKS = new Runnable(){
public void run() {}
};
+ /**
+ * The background thread implementation: simply collects {@link Runnable}s
+ * from the {@link RankingQueryRunnerImpl#backgroundTasks} queue and runs
them.
+ */
protected class BackgroundRunner implements Runnable {
@Override
public void run() {
try {
while(true) {
Runnable job = backgroundTasks.take();
- if(job == NO_MORE_JOBS) break;
+ if(job == NO_MORE_TASKS) break;
else job.run();
}
} catch(InterruptedException e) {
@@ -107,7 +124,7 @@
// we need to 'scroll back' the executor: get a new executor
QueryExecutor oldExecutor = queryExecutor;
queryExecutor = queryExecutor.getQueryNode().getQueryExecutor(
- queryExecutor.getQueryEngine());
+ queryEngine);
oldExecutor.close();
}
for(int i = start; i < end; i++) {
@@ -217,7 +234,10 @@
}
}
- protected Logger logger = Logger.getLogger(RankingQueryRunnerImpl.class);
+ /**
+ * Shared logger instance.
+ */
+ protected static Logger logger =
Logger.getLogger(RankingQueryRunnerImpl.class);
/**
* The {@link QueryExecutor} for the query being run.
@@ -225,6 +245,11 @@
protected QueryExecutor queryExecutor;
/**
+ * The QueryEngine we run inside.
+ */
+ protected QueryEngine queryEngine;
+
+ /**
* The {@link MimirScorer} to be used for ranking documents.
*/
protected MimirScorer scorer;
@@ -277,6 +302,9 @@
*/
protected BlockingQueue<Runnable> backgroundTasks;
+ /**
+ * Flag used to mark that all results documents have been counted.
+ */
protected volatile boolean allDocIdsCollected = false;
/**
@@ -289,7 +317,8 @@
public RankingQueryRunnerImpl(QueryExecutor executor, MimirScorer scorer)
throws IOException {
this.queryExecutor = executor;
this.scorer = scorer;
- docBlockSize = queryExecutor.getQueryEngine().getRankingDocCount();
+ queryEngine = queryExecutor.getQueryEngine();
+ docBlockSize = queryEngine.getRankingDocCount();
documentIds = new IntArrayList();
documentHits = new ObjectArrayList<List<Binding>>();
if(scorer != null) {
@@ -305,9 +334,9 @@
backgroundTasks = new LinkedBlockingQueue<Runnable>();
Runnable backgroundRunner = new BackgroundRunner();
//get a thread from the executor, if one exists
- if(queryExecutor.getQueryEngine().getExecutor() != null){
+ if(queryEngine.getExecutor() != null){
try {
- queryExecutor.getQueryEngine().getExecutor().execute(backgroundRunner);
+ queryEngine.getExecutor().execute(backgroundRunner);
} catch(RejectedExecutionException e) {
logger.warn("Could not allocate a new background thread", e);
throw new RejectedExecutionException(
@@ -328,56 +357,35 @@
}
}
- /**
- * Gets the number of result documents. If the search has not yet completed,
- * then -1 is returned.
- * @return
+ /* (non-Javadoc)
+ * @see gate.mimir.search.QueryRunner#getDocumentsCount()
*/
+ @Override
public int getDocumentsCount() {
if(allDocIdsCollected) return documentIds.size();
else return -1;
}
- /**
- * Gets the number of documents found to contain hits so far. After the
search
- * completes, the result returned by this call is identical to that of
- * {@link #getDocumentsCount()}.
- * @return
+ /* (non-Javadoc)
+ * @see gate.mimir.search.QueryRunner#getCurrentDocumentsCount()
*/
+ @Override
public int getCurrentDocumentsCount() {
return documentIds.size();
}
- /**
- * Gets the ID of a result document.
- * @param rank the index of the desired document in the list of documents.
- * This should be a value between 0 and {@link #getDocumentsCount()} -1.
- *
- * If the requested document position has not yet been ranked (i.e. we know
- * there is a document at that position, but we don't yet know which one)
then
- * the necessary ranking is performed before this method returns.
- *
- * @return an int value, representing the ID of the requested document.
- * @throws IndexOutOfBoundsException is the index provided is less than
zero,
- * or greater than {@link #getDocumentsCount()} -1.
- * @throws IOException
+ /* (non-Javadoc)
+ * @see gate.mimir.search.QueryRunner#getDocumentID(int)
*/
+ @Override
public int getDocumentID(int rank) throws IndexOutOfBoundsException,
IOException {
return documentIds.getInt(getDocumentIndex(rank));
}
- /**
- * Retrieves the hits withing a given result document.
- * @param rank the index of the desired document in the list of documents.
- * This should be a value between 0 and {@link #getDocumentsCount()} -1.
- *
- * This method call waits until the requested data is available before
- * returning (document hits are being collected by a background thread).
- *
- * @return
- * @throws IOException
- * @throws IndexOutOfBoundsException
+ /* (non-Javadoc)
+ * @see gate.mimir.search.QueryRunner#getDocumentHits(int)
*/
+ @Override
public List<Binding> getDocumentHits(int rank) throws
IndexOutOfBoundsException, IOException {
int documentIndex = getDocumentIndex(rank);
List<Binding> hits = documentHits.get(documentIndex);
@@ -442,8 +450,7 @@
int rankRangeEnd = index;
if((rankRangeEnd - rankRangeStart) < (docBlockSize -1)) {
// extend the size of the chunk of documents to be ranked
- rankRangeEnd = rankRangeStart +
- queryExecutor.getQueryEngine().getRankingDocCount();
+ rankRangeEnd = rankRangeStart + docBlockSize;
}
// the document with the minimum score already ranked.
int smallestOldScoreDocId = rankRangeStart > 0 ?
@@ -520,7 +527,8 @@
else if (midVal < documentScore) end = mid - 1;
else {
// we found a doc with exactly the same score: scan to the right
- while(documentScores.getDouble(documentsOrder.getInt(mid)) ==
+ while(documentsOrder.size() < mid &&
+ documentScores.getDouble(documentsOrder.getInt(mid)) ==
documentScore){
mid++;
}
@@ -574,12 +582,79 @@
}
}
+ /* (non-Javadoc)
+ * @see gate.mimir.search.QueryRunner#getDocumentText(int, int, int)
+ */
+ @Override
+ public String[][] getDocumentText(int rank, int termPosition, int length)
+ throws IndexException, IndexOutOfBoundsException, IOException {
+ return queryEngine.getText(getDocumentID(rank), termPosition, length);
+ }
+
+ /* (non-Javadoc)
+ * @see gate.mimir.search.QueryRunner#getDocumentURI(int)
+ */
+ @Override
+ public String getDocumentURI(int rank) throws IndexException,
+ IndexOutOfBoundsException, IOException {
+ return queryEngine.getDocumentURI(getDocumentID(rank));
+ }
+
+ /* (non-Javadoc)
+ * @see gate.mimir.search.QueryRunner#getDocumentTitle(int)
+ */
+ @Override
+ public String getDocumentTitle(int rank) throws IndexException,
+ IndexOutOfBoundsException, IOException {
+ return queryEngine.getDocumentTitle(getDocumentID(rank));
+ }
+
+ /* (non-Javadoc)
+ * @see gate.mimir.search.QueryRunner#getDocumentMetadataField(int,
java.lang.String)
+ */
+ @Override
+ public Serializable getDocumentMetadataField(int rank, String fieldName)
+ throws IndexException, IndexOutOfBoundsException, IOException {
+ return queryEngine.getDocumentMetadataField(getDocumentID(rank),
fieldName);
+ }
+
+ /* (non-Javadoc)
+ * @see gate.mimir.search.QueryRunner#getDocumentMetadataFields(int,
java.util.Set)
+ */
+ @Override
+ public Map<String, Serializable> getDocumentMetadataFields(int rank,
+ Set<String> fieldNames) throws IndexException,
IndexOutOfBoundsException,
+ IOException {
+ Map<String, Serializable> res = new HashMap<String, Serializable>();
+ int docId = getDocumentID(rank);
+ for(String fieldName : fieldNames) {
+ Serializable value = getDocumentMetadataField(docId, fieldName);
+ if(value != null) res.put(fieldName, value);
+ }
+ return res;
+ }
+
+ /* (non-Javadoc)
+ * @see gate.mimir.search.QueryRunner#renderDocument(int,
java.lang.Appendable)
+ */
+ @Override
+ public void renderDocument(int rank, Appendable out) throws IOException,
+ IndexException {
+ queryEngine.renderDocument(getDocumentID(rank),
+ getDocumentHits(rank), out);
+ }
+
+ /* (non-Javadoc)
+ * @see gate.mimir.search.QueryRunner#close()
+ */
+ @Override
public void close() throws IOException {
- // TODO give back the borrowed thread
queryExecutor.close();
scorer = null;
try {
- backgroundTasks.put(NO_MORE_JOBS);
+ // stop the background tasks runnable,
+ // which will return the thread to the pool
+ backgroundTasks.put(NO_MORE_TASKS);
} catch(InterruptedException e) {
// ignore
}
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
All the data continuously generated in your IT infrastructure
contains a definitive record of customers, application performance,
security threats, fraudulent activity, and more. Splunk takes this
data and makes sense of it. IT sense. And common sense.
http://p.sf.net/sfu/splunk-novd2d
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
