Author: cstamas Date: Fri Nov 26 17:20:50 2010 New Revision: 1039472 URL: http://svn.apache.org/viewvc?rev=1039472&view=rev Log: Fixed query creation in cases when actually a keyword is supplied as term search.
Before, QueryCreator (to be as versatile as possible) was creating too "wide" queries that were actually skewing results, and some hits were unrightfully been scored better as others. An attempt is made to fix it, and combination of KW and non-KW fields are used only, when input term is actually multi termed. For more and original discussion see https://issues.sonatype.org/browse/NEXUS-3881 Added: maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Nexus3881NexusIndexerTest.java (with props) maven/indexer/trunk/indexer-core/src/test/nexus-3881/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one.solution.this/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one.solution.this/1.0/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one.solution.this/1.0/one.solution.this-1.0.pom (with props) maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/solution.this/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/solution.this/1.0/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/solution.this/1.0/solution.this-1.0.pom (with props) maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/this_solution/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/this_solution/1.0/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/this_solution/1.0/this_solution-1.0.pom (with props) maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/solution/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/solution/one/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/solution/one/this/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/solution/one/this/1.0/ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/solution/one/this/1.0/this-1.0.pom (with props) Modified: maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultQueryCreator.java maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java maven/indexer/trunk/indexer-core/src/test/resources/testQueryCreatorNGSearch/case03.txt maven/indexer/trunk/indexer-core/src/test/resources/testQueryCreatorNGSearch/case04.txt Modified: maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultQueryCreator.java URL: http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultQueryCreator.java?rev=1039472&r1=1039471&r2=1039472&view=diff ============================================================================== --- maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultQueryCreator.java (original) +++ maven/indexer/trunk/indexer-core/src/main/java/org/apache/maven/index/DefaultQueryCreator.java Fri Nov 26 17:20:50 2010 @@ -18,16 +18,20 @@ */ package org.apache.maven.index; +import java.io.IOException; +import java.io.StringReader; + +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.Term; import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.queryParser.QueryParser.Operator; +import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.search.BooleanClause.Occur; import org.apache.maven.index.context.NexusAnalyzer; import org.apache.maven.index.creator.JarFileContentsIndexCreator; import org.apache.maven.index.creator.MinimalArtifactInfoIndexCreator; @@ -288,9 +292,12 @@ public class DefaultQueryCreator Query q2 = null; + int termCount = countTerms( indexerField, query ); + // try with KW only if the processed query in qpQuery does not have spaces! - if ( !query.contains( " " ) ) + if ( !query.contains( " " ) && termCount > 1 ) { + // get the KW field IndexerField keywordField = selectIndexerField( indexerField.getOntology(), SearchType.EXACT ); if ( keywordField.isKeyword() ) @@ -398,4 +405,30 @@ public class DefaultQueryCreator return new WildcardQuery( new Term( field, q ) ); } + + // == + + private NexusAnalyzer nexusAnalyzer = new NexusAnalyzer(); + + protected int countTerms( final IndexerField indexerField, final String query ) + { + try + { + TokenStream ts = nexusAnalyzer.reusableTokenStream( indexerField.getKey(), new StringReader( query ) ); + + int result = 0; + + while ( ts.next() != null ) + { + result++; + } + + return result; + } + catch ( IOException e ) + { + // will not happen + return 1; + } + } } Added: maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Nexus3881NexusIndexerTest.java URL: http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Nexus3881NexusIndexerTest.java?rev=1039472&view=auto ============================================================================== --- maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Nexus3881NexusIndexerTest.java (added) +++ maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Nexus3881NexusIndexerTest.java Fri Nov 26 17:20:50 2010 @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.maven.index; + +import java.io.File; + +import junit.framework.Assert; + +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; + +public class Nexus3881NexusIndexerTest + extends AbstractNexusIndexerTest +{ + protected File repo = new File( getBasedir(), "src/test/nexus-3881" ); + + @Override + protected void prepareNexusIndexer( NexusIndexer nexusIndexer ) + throws Exception + { + context = + nexusIndexer.addIndexingContext( "nexus-3881", "nexus-3881", repo, indexDir, null, null, DEFAULT_CREATORS ); + nexusIndexer.scan( context ); + } + + public void testRelevances() + throws Exception + { + Query q1 = nexusIndexer.constructQuery( MAVEN.GROUP_ID, "solution", SearchType.SCORED ); + Query q2 = nexusIndexer.constructQuery( MAVEN.ARTIFACT_ID, "solution", SearchType.SCORED ); + + BooleanQuery bq = new BooleanQuery(); + bq.add( q1, Occur.SHOULD ); + bq.add( q2, Occur.SHOULD ); + + IteratorSearchResponse response = nexusIndexer.searchIterator( new IteratorSearchRequest( bq ) ); + + Assert.assertEquals( "All artifacts has 'solution' in their GA!", 4, response.getTotalHits() ); + + float firstRel = response.getResults().next().getLuceneScore(); + + float lastRel = 0; + for ( ArtifactInfo ai : response ) + { + lastRel = ai.getLuceneScore(); + } + + Assert.assertTrue( + String.format( "The relevance span should be small! (%s)", + new Object[] { Float.valueOf( firstRel - lastRel ) } ), firstRel - lastRel < 0.35 ); + } +} Propchange: maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Nexus3881NexusIndexerTest.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/Nexus3881NexusIndexerTest.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Modified: maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java URL: http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java?rev=1039472&r1=1039471&r2=1039472&view=diff ============================================================================== --- maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java (original) +++ maven/indexer/trunk/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java Fri Nov 26 17:20:50 2010 @@ -39,20 +39,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.store.RAMDirectory; -import org.apache.maven.index.ArtifactContext; -import org.apache.maven.index.ArtifactInfo; -import org.apache.maven.index.ArtifactInfoFilter; -import org.apache.maven.index.ArtifactInfoGroup; -import org.apache.maven.index.Field; -import org.apache.maven.index.FlatSearchRequest; -import org.apache.maven.index.FlatSearchResponse; -import org.apache.maven.index.GroupedSearchRequest; -import org.apache.maven.index.GroupedSearchResponse; -import org.apache.maven.index.IteratorSearchRequest; -import org.apache.maven.index.IteratorSearchResponse; -import org.apache.maven.index.MAVEN; -import org.apache.maven.index.NexusIndexer; -import org.apache.maven.index.SearchType; import org.apache.maven.index.context.IndexCreator; import org.apache.maven.index.context.IndexingContext; import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException; @@ -60,7 +46,6 @@ import org.apache.maven.index.creator.Mi import org.apache.maven.index.packer.DefaultIndexPacker; import org.apache.maven.index.search.grouping.GAGrouping; import org.apache.maven.index.updater.DefaultIndexUpdater; -import org.codehaus.plexus.util.FileUtils; import org.codehaus.plexus.util.StringUtils; /** @author Jason van Zyl */ Added: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one.solution.this/1.0/one.solution.this-1.0.pom URL: http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one.solution.this/1.0/one.solution.this-1.0.pom?rev=1039472&view=auto ============================================================================== --- maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one.solution.this/1.0/one.solution.this-1.0.pom (added) +++ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one.solution.this/1.0/one.solution.this-1.0.pom Fri Nov 26 17:20:50 2010 @@ -0,0 +1,9 @@ +<project> + <modelVersion>4.0.0</modelVersion> + + <groupId>com</groupId> + <artifactId>one.solution.this</artifactId> + <version>1.0</version> + <packaging>pom</packaging> + +</project> Propchange: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one.solution.this/1.0/one.solution.this-1.0.pom ------------------------------------------------------------------------------ svn:eol-style = native Propchange: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one.solution.this/1.0/one.solution.this-1.0.pom ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/solution.this/1.0/solution.this-1.0.pom URL: http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/solution.this/1.0/solution.this-1.0.pom?rev=1039472&view=auto ============================================================================== --- maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/solution.this/1.0/solution.this-1.0.pom (added) +++ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/solution.this/1.0/solution.this-1.0.pom Fri Nov 26 17:20:50 2010 @@ -0,0 +1,9 @@ +<project> + <modelVersion>4.0.0</modelVersion> + + <groupId>com.one</groupId> + <artifactId>solution.this</artifactId> + <version>1.0</version> + <packaging>pom</packaging> + +</project> Propchange: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/solution.this/1.0/solution.this-1.0.pom ------------------------------------------------------------------------------ svn:eol-style = native Propchange: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/solution.this/1.0/solution.this-1.0.pom ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/this_solution/1.0/this_solution-1.0.pom URL: http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/this_solution/1.0/this_solution-1.0.pom?rev=1039472&view=auto ============================================================================== --- maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/this_solution/1.0/this_solution-1.0.pom (added) +++ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/this_solution/1.0/this_solution-1.0.pom Fri Nov 26 17:20:50 2010 @@ -0,0 +1,9 @@ +<project> + <modelVersion>4.0.0</modelVersion> + + <groupId>com.one</groupId> + <artifactId>this_solution</artifactId> + <version>1.0</version> + <packaging>pom</packaging> + +</project> Propchange: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/this_solution/1.0/this_solution-1.0.pom ------------------------------------------------------------------------------ svn:eol-style = native Propchange: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/one/this_solution/1.0/this_solution-1.0.pom ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/solution/one/this/1.0/this-1.0.pom URL: http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/solution/one/this/1.0/this-1.0.pom?rev=1039472&view=auto ============================================================================== --- maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/solution/one/this/1.0/this-1.0.pom (added) +++ maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/solution/one/this/1.0/this-1.0.pom Fri Nov 26 17:20:50 2010 @@ -0,0 +1,9 @@ +<project> + <modelVersion>4.0.0</modelVersion> + + <groupId>com.solution.one</groupId> + <artifactId>this</artifactId> + <version>1.0</version> + <packaging>pom</packaging> + +</project> Propchange: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/solution/one/this/1.0/this-1.0.pom ------------------------------------------------------------------------------ svn:eol-style = native Propchange: maven/indexer/trunk/indexer-core/src/test/nexus-3881/com/solution/one/this/1.0/this-1.0.pom ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Modified: maven/indexer/trunk/indexer-core/src/test/resources/testQueryCreatorNGSearch/case03.txt URL: http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/test/resources/testQueryCreatorNGSearch/case03.txt?rev=1039472&r1=1039471&r2=1039472&view=diff ============================================================================== --- maven/indexer/trunk/indexer-core/src/test/resources/testQueryCreatorNGSearch/case03.txt (original) +++ maven/indexer/trunk/indexer-core/src/test/resources/testQueryCreatorNGSearch/case03.txt Fri Nov 26 17:20:50 2010 @@ -1,6 +1,4 @@ -### Searched for field urn:maven#groupId (with 2 registered index fields) using query "commons" (QC create LQL "(g:commons g:commons*^0.8) (groupId:commons*)") -test :: commons-cli:commons-cli:1.0:null:jar -test :: commons-cli:commons-cli:1.0:sources:jar +### Searched for field urn:maven#groupId (with 2 registered index fields) using query "commons" (QC create LQL "groupId:commons*") test :: commons-logging:commons-logging:1.1:null:jar test :: commons-logging:commons-logging:1.1:sources:jar test :: commons-logging:commons-logging:1.0.4:null:jar @@ -14,4 +12,6 @@ test :: commons-logging:commons-logging: test :: commons-logging:commons-logging:1.0.1:javadoc:jar test :: commons-logging:commons-logging:1.0:null:jar test :: commons-logging:commons-logging:1.0:javadoc:jar +test :: commons-cli:commons-cli:1.0:null:jar +test :: commons-cli:commons-cli:1.0:sources:jar ### TOTAL:15 (response said 15) Modified: maven/indexer/trunk/indexer-core/src/test/resources/testQueryCreatorNGSearch/case04.txt URL: http://svn.apache.org/viewvc/maven/indexer/trunk/indexer-core/src/test/resources/testQueryCreatorNGSearch/case04.txt?rev=1039472&r1=1039471&r2=1039472&view=diff ============================================================================== --- maven/indexer/trunk/indexer-core/src/test/resources/testQueryCreatorNGSearch/case04.txt (original) +++ maven/indexer/trunk/indexer-core/src/test/resources/testQueryCreatorNGSearch/case04.txt Fri Nov 26 17:20:50 2010 @@ -1,4 +1,4 @@ -### Searched for field urn:maven#groupId (with 2 registered index fields) using query "log" (QC create LQL "(g:log g:log*^0.8) (groupId:log*)") +### Searched for field urn:maven#groupId (with 2 registered index fields) using query "log" (QC create LQL "groupId:log*") test :: commons-logging:commons-logging:1.1:null:jar test :: commons-logging:commons-logging:1.1:sources:jar test :: commons-logging:commons-logging:1.0.4:null:jar
