Author: cstamas
Date: Thu Mar 31 13:52:09 2011
New Revision: 1087301
URL: http://svn.apache.org/viewvc?rev=1087301&view=rev
Log:
MINDEXER-14 and others.
* re-introduced "count" on AbstractSearchRequest to overcome problem of Group
searches being "wide open" for OOM and no way to limit them.
* cleanup the API wrt "search paging": there was paging support in indexer, but
it never worked for grouped and flat searches. Only pageable search was much
later implemented Iterator search. Now, the API properly reflects this fact,
unlike before, when it suggested that flat and grouped supports paging but
actually they did not.
* Javadoc improvements, clearing up the meaning of "count" and hits.
* Some UTs commented out -- those that were testing (and were disabled) paging
on flat search, but actually did nothing, see MINDEXER-22 for more.
Modified:
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchPageableRequest.java
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchRequest.java
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchResponse.java
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultIteratorResultSet.java
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultSearchEngine.java
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/FlatSearchRequest.java
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/IteratorSearchRequest.java
maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/AbstractRepoNexusIndexerTest.java
maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Index20081108RegressionTest.java
Modified:
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchPageableRequest.java
URL:
http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchPageableRequest.java?rev=1087301&r1=1087300&r2=1087301&view=diff
==============================================================================
---
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchPageableRequest.java
(original)
+++
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchPageableRequest.java
Thu Mar 31 13:52:09 2011
@@ -5,32 +5,25 @@ import java.util.List;
import org.apache.lucene.search.Query;
import org.apache.maven.index.context.IndexingContext;
+/**
+ * Pageable search request. Adds "start" point, to skip wanted number of
records, to implement paging. Use "count" of
+ * AbstractSearchRequest to set page size.
+ *
+ * @author cstamas
+ */
public class AbstractSearchPageableRequest
extends AbstractSearchRequest
{
/**
- * Constant for denoting undefined value for result count.
- */
- protected static final int UNDEFINED = -1;
-
- /**
* The number of hit we want to skip from result set. Defaults to 0.
*/
private int start;
- /**
- * The page size, actually count of items in one page. Different than
limit, because this will _cut_ the response to
- * the requested count.
- */
- private int count;
-
public AbstractSearchPageableRequest( Query query )
{
super( query, null );
this.start = 0;
-
- this.count = UNDEFINED;
}
public AbstractSearchPageableRequest( Query query, List<IndexingContext>
contexts )
@@ -38,8 +31,6 @@ public class AbstractSearchPageableReque
super( query, contexts );
this.start = 0;
-
- this.count = UNDEFINED;
}
/**
@@ -66,29 +57,4 @@ public class AbstractSearchPageableReque
this.start = start;
}
-
- /**
- * Returns the "count" of wanted results. See {@link #UNDEFINED}.
- *
- * @return
- */
- public int getCount()
- {
- return count;
- }
-
- /**
- * Sets the "count" of wanted results. See {@link #UNDEFINED}.
- *
- * @param count
- */
- public void setCount( int count )
- {
- if ( UNDEFINED != count && count < 1 )
- {
- throw new IllegalArgumentException( "Count cannot be less than 1!"
);
- }
-
- this.count = count;
- }
}
Modified:
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchRequest.java
URL:
http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchRequest.java?rev=1087301&r1=1087300&r2=1087301&view=diff
==============================================================================
---
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchRequest.java
(original)
+++
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchRequest.java
Thu Mar 31 13:52:09 2011
@@ -26,11 +26,23 @@ import org.apache.maven.index.context.In
public class AbstractSearchRequest
{
+ /**
+ * Constant for denoting undefined value for result count.
+ */
+ public static final int UNDEFINED = -1;
+
private Query query;
private List<IndexingContext> contexts;
/**
+ * The maximum count of results expected to have delivered, actually count
of items (AIs). More precisely, with this
+ * setting we LIMIT the number of Lucene Documents for total set of hits
to be processed. If set to anything other
+ * than {@link #UNDEFINED}, search will stop upon processing this count of
AIs (that correspond to Lucene Document).
+ */
+ private int count;
+
+ /**
* The filter to be used while executing the search request.
*/
private ArtifactInfoFilter artifactInfoFilter;
@@ -65,6 +77,8 @@ public class AbstractSearchRequest
{
getContexts().addAll( contexts );
}
+
+ this.count = UNDEFINED;
}
public Query getQuery()
@@ -93,6 +107,31 @@ public class AbstractSearchRequest
}
/**
+ * Returns the "count" of wanted results. See {@link #UNDEFINED} and
{@link #count}.
+ *
+ * @return
+ */
+ public int getCount()
+ {
+ return count;
+ }
+
+ /**
+ * Sets the "count" of wanted results. See {@link #UNDEFINED} and {@link
#count}.
+ *
+ * @param count
+ */
+ public void setCount( int count )
+ {
+ if ( UNDEFINED != count && count < 1 )
+ {
+ throw new IllegalArgumentException( "Count cannot be less than 1!"
);
+ }
+
+ this.count = count;
+ }
+
+ /**
* Returns true if hits are limited.
*
* @return
Modified:
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchResponse.java
URL:
http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchResponse.java?rev=1087301&r1=1087300&r2=1087301&view=diff
==============================================================================
---
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchResponse.java
(original)
+++
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/AbstractSearchResponse.java
Thu Mar 31 13:52:09 2011
@@ -60,7 +60,7 @@ public class AbstractSearchResponse
/**
* Returns the number of total hits found by this query (total number of
potential hits as reported by Lucene
* index). This is the number of existing AIs matching your query, and
does not represent the count of hits
- * delivered, which is returned by {@link #getHitsCount()}.
+ * delivered, which is returned by {@link #getReturnedHitsCount()}.
*
* @return
*/
Modified:
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultIteratorResultSet.java
URL:
http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultIteratorResultSet.java?rev=1087301&r1=1087300&r2=1087301&view=diff
==============================================================================
---
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultIteratorResultSet.java
(original)
+++
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultIteratorResultSet.java
Thu Mar 31 13:52:09 2011
@@ -109,7 +109,7 @@ public class DefaultIteratorResultSet
this.from = request.getStart();
this.count =
- ( request.getCount() == AbstractSearchPageableRequest.UNDEFINED ?
hits.scoreDocs.length : Math.min(
+ ( request.getCount() == AbstractSearchRequest.UNDEFINED ?
hits.scoreDocs.length : Math.min(
request.getCount(), hits.scoreDocs.length ) );
this.pointer = from;
Modified:
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultSearchEngine.java
URL:
http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultSearchEngine.java?rev=1087301&r1=1087300&r2=1087301&view=diff
==============================================================================
---
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultSearchEngine.java
(original)
+++
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultSearchEngine.java
Thu Mar 31 13:52:09 2011
@@ -305,9 +305,9 @@ public class DefaultSearchEngine
final IndexSearcher
indexSearcher, final Query query )
throws IOException
{
- int topHitCount = getTopDocsCollectorHitNum( request,
AbstractSearchPageableRequest.UNDEFINED );
+ int topHitCount = getTopDocsCollectorHitNum( request,
AbstractSearchRequest.UNDEFINED );
- if ( AbstractSearchPageableRequest.UNDEFINED != topHitCount )
+ if ( AbstractSearchRequest.UNDEFINED != topHitCount )
{
// count is set, simply just execute it as-is
final TopScoreDocCollector hits = TopScoreDocCollector.create(
topHitCount, true );
@@ -321,13 +321,11 @@ public class DefaultSearchEngine
// set something reasonable as 1k
topHitCount = 1000;
- // count is not set, start with something reasonable as 1k
- TopScoreDocCollector hits = TopScoreDocCollector.create(
topHitCount, true );
-
// perform search
+ TopScoreDocCollector hits = TopScoreDocCollector.create(
topHitCount, true );
indexSearcher.search( query, hits );
- // check total hits against
+ // check total hits against, does it fit?
if ( topHitCount < hits.getTotalHits() )
{
topHitCount = hits.getTotalHits();
@@ -431,12 +429,20 @@ public class DefaultSearchEngine
{
final AbstractSearchPageableRequest prequest =
(AbstractSearchPageableRequest) request;
- if ( AbstractSearchPageableRequest.UNDEFINED !=
prequest.getCount() )
+ if ( AbstractSearchRequest.UNDEFINED != prequest.getCount() )
{
// easy, user knows and tells us how many results he want
return prequest.getCount() + prequest.getStart();
}
}
+ else
+ {
+ if ( AbstractSearchRequest.UNDEFINED != request.getCount() )
+ {
+ // easy, user knows and tells us how many results he want
+ return request.getCount();
+ }
+ }
return ceiling;
}
Modified:
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/FlatSearchRequest.java
URL:
http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/FlatSearchRequest.java?rev=1087301&r1=1087300&r2=1087301&view=diff
==============================================================================
---
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/FlatSearchRequest.java
(original)
+++
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/FlatSearchRequest.java
Thu Mar 31 13:52:09 2011
@@ -30,7 +30,7 @@ import org.apache.maven.index.context.In
* @see NexusIndexer#searchFlat(FlatSearchRequest)
*/
public class FlatSearchRequest
- extends AbstractSearchPageableRequest
+ extends AbstractSearchRequest
{
private Comparator<ArtifactInfo> artifactInfoComparator;
Modified:
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/IteratorSearchRequest.java
URL:
http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/IteratorSearchRequest.java?rev=1087301&r1=1087300&r2=1087301&view=diff
==============================================================================
---
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/IteratorSearchRequest.java
(original)
+++
maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/IteratorSearchRequest.java
Thu Mar 31 13:52:09 2011
@@ -25,7 +25,8 @@ import org.apache.lucene.search.Query;
import org.apache.maven.index.context.IndexingContext;
/**
- * A Search Request that will perform the new "iterator-like" type of search.
+ * A Search Request that will perform the new "iterator-like" type of search.
It is pageable as plus, but on downside,
+ * it's corresponding result does not result size in advance.
*
* @author cstamas
*/
Modified:
maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/AbstractRepoNexusIndexerTest.java
URL:
http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/AbstractRepoNexusIndexerTest.java?rev=1087301&r1=1087300&r2=1087301&view=diff
==============================================================================
---
maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/AbstractRepoNexusIndexerTest.java
(original)
+++
maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/AbstractRepoNexusIndexerTest.java
Thu Mar 31 13:52:09 2011
@@ -80,7 +80,9 @@ public abstract class AbstractRepoNexusI
FlatSearchRequest request =
new FlatSearchRequest( nexusIndexer.constructQuery(
MAVEN.GROUP_ID, "org", SearchType.SCORED ) );
- request.setStart( 0 );
+ // See MINDEXER-22
+ // Flat search is not pageable
+ // request.setStart( 0 );
request.setCount( 50 );
@@ -424,56 +426,58 @@ public abstract class AbstractRepoNexusI
}
// Paging is currently disabled
- public void donttestPaging()
- throws Exception
- {
- // we have 22 artifact for this search
- int total = 22;
-
- int pageSize = 4;
-
- Query q = nexusIndexer.constructQuery( MAVEN.GROUP_ID, "org",
SearchType.SCORED );
-
- FlatSearchRequest req = new FlatSearchRequest( q );
-
- // have page size of 4, that will make us 4 pages
- req.setCount( pageSize );
-
- List<ArtifactInfo> constructedPageList = new ArrayList<ArtifactInfo>();
-
- int offset = 0;
-
- while ( true )
- {
- req.setStart( offset );
-
- FlatSearchResponse resp = nexusIndexer.searchFlat( req );
-
- Collection<ArtifactInfo> p = resp.getResults();
-
- assertEquals( p.toString(), total, resp.getTotalHits() );
-
- assertEquals( Math.min( pageSize, total - offset ), p.size() );
-
- constructedPageList.addAll( p );
-
- offset += pageSize;
-
- if ( offset > total )
- {
- break;
- }
- }
-
- //
- FlatSearchResponse response = nexusIndexer.searchFlat( new
FlatSearchRequest( q ) );
- Collection<ArtifactInfo> onePage = response.getResults();
-
- List<ArtifactInfo> onePageList = new ArrayList<ArtifactInfo>( onePage
);
-
- // onePage and constructedPage should hold equal elems in the same
order
- assertTrue( resultsAreEqual( onePageList, constructedPageList ) );
- }
+ // See MINDEXER-22
+ // Flat search is not pageable
+// public void donttestPaging()
+// throws Exception
+// {
+// // we have 22 artifact for this search
+// int total = 22;
+//
+// int pageSize = 4;
+//
+// Query q = nexusIndexer.constructQuery( MAVEN.GROUP_ID, "org",
SearchType.SCORED );
+//
+// FlatSearchRequest req = new FlatSearchRequest( q );
+//
+// // have page size of 4, that will make us 4 pages
+// req.setCount( pageSize );
+//
+// List<ArtifactInfo> constructedPageList = new
ArrayList<ArtifactInfo>();
+//
+// int offset = 0;
+//
+// while ( true )
+// {
+// req.setStart( offset );
+//
+// FlatSearchResponse resp = nexusIndexer.searchFlat( req );
+//
+// Collection<ArtifactInfo> p = resp.getResults();
+//
+// assertEquals( p.toString(), total, resp.getTotalHits() );
+//
+// assertEquals( Math.min( pageSize, total - offset ), p.size() );
+//
+// constructedPageList.addAll( p );
+//
+// offset += pageSize;
+//
+// if ( offset > total )
+// {
+// break;
+// }
+// }
+//
+// //
+// FlatSearchResponse response = nexusIndexer.searchFlat( new
FlatSearchRequest( q ) );
+// Collection<ArtifactInfo> onePage = response.getResults();
+//
+// List<ArtifactInfo> onePageList = new ArrayList<ArtifactInfo>(
onePage );
+//
+// // onePage and constructedPage should hold equal elems in the same
order
+// assertTrue( resultsAreEqual( onePageList, constructedPageList ) );
+// }
public void testPurge()
throws Exception
Modified:
maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Index20081108RegressionTest.java
URL:
http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Index20081108RegressionTest.java?rev=1087301&r1=1087300&r2=1087301&view=diff
==============================================================================
---
maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Index20081108RegressionTest.java
(original)
+++
maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Index20081108RegressionTest.java
Thu Mar 31 13:52:09 2011
@@ -151,7 +151,9 @@ public class Index20081108RegressionTest
{
FlatSearchRequest request = new FlatSearchRequest(
nexusIndexer.constructQuery( MAVEN.GROUP_ID, "org", SearchType.SCORED ) );
- request.setStart( 0 );
+ // See MINDEXER-22
+ // Flat search is not pageable
+ // request.setStart( 0 );
request.setCount( 50 );
@@ -206,57 +208,59 @@ public class Index20081108RegressionTest
}
}
- @Override
- public void donttestPaging()
- throws Exception
- {
- // we have 15 artifact for this search
- int total = 15;
-
- int pageSize = 4;
-
- Query q = nexusIndexer.constructQuery( MAVEN.GROUP_ID, "org",
SearchType.SCORED );
-
- FlatSearchRequest req = new FlatSearchRequest( q );
-
- // have page size of 4, that will make us 4 pages
- req.setCount( pageSize );
-
- List<ArtifactInfo> constructedPageList = new ArrayList<ArtifactInfo>();
-
- int offset = 0;
-
- while ( true )
- {
- req.setStart( offset );
-
- FlatSearchResponse resp = nexusIndexer.searchFlat( req );
-
- Collection<ArtifactInfo> p = resp.getResults();
-
- assertEquals( p.toString(), total, resp.getTotalHits() );
-
- assertEquals( Math.min( pageSize, total - offset ), p.size() );
-
- constructedPageList.addAll( p );
-
- offset += pageSize;
-
- if ( offset > total )
- {
- break;
- }
- }
-
- //
- FlatSearchResponse response = nexusIndexer.searchFlat( new
FlatSearchRequest( q ) );
- Collection<ArtifactInfo> onePage = response.getResults();
-
- List<ArtifactInfo> onePageList = new ArrayList<ArtifactInfo>( onePage
);
-
- // onePage and constructedPage should hold equal elems in the same
order
- assertTrue( resultsAreEqual( onePageList, constructedPageList ) );
- }
+ // See MINDEXER-22
+ // Flat search is not pageable
+// @Override
+// public void donttestPaging()
+// throws Exception
+// {
+// // we have 15 artifact for this search
+// int total = 15;
+//
+// int pageSize = 4;
+//
+// Query q = nexusIndexer.constructQuery( MAVEN.GROUP_ID, "org",
SearchType.SCORED );
+//
+// FlatSearchRequest req = new FlatSearchRequest( q );
+//
+// // have page size of 4, that will make us 4 pages
+// req.setCount( pageSize );
+//
+// List<ArtifactInfo> constructedPageList = new
ArrayList<ArtifactInfo>();
+//
+// int offset = 0;
+//
+// while ( true )
+// {
+// req.setStart( offset );
+//
+// FlatSearchResponse resp = nexusIndexer.searchFlat( req );
+//
+// Collection<ArtifactInfo> p = resp.getResults();
+//
+// assertEquals( p.toString(), total, resp.getTotalHits() );
+//
+// assertEquals( Math.min( pageSize, total - offset ), p.size() );
+//
+// constructedPageList.addAll( p );
+//
+// offset += pageSize;
+//
+// if ( offset > total )
+// {
+// break;
+// }
+// }
+//
+// //
+// FlatSearchResponse response = nexusIndexer.searchFlat( new
FlatSearchRequest( q ) );
+// Collection<ArtifactInfo> onePage = response.getResults();
+//
+// List<ArtifactInfo> onePageList = new ArrayList<ArtifactInfo>(
onePage );
+//
+// // onePage and constructedPage should hold equal elems in the same
order
+// assertTrue( resultsAreEqual( onePageList, constructedPageList ) );
+// }
@Override
public void testPurge()
