Add super query parser with tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d73a6ee9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d73a6ee9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d73a6ee9 Branch: refs/heads/0.2-dev Commit: d73a6ee9aabe0d3a7099f8dd493425ba1d921b11 Parents: b75a202 Author: Aaron McCurry <[email protected]> Authored: Sun Feb 17 15:26:48 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Sun Feb 17 15:26:48 2013 -0500 ---------------------------------------------------------------------- .../org/apache/blur/server/LayoutStrategy.java | 21 +++ .../org/apache/blur/server/QueryConverterImpl.java | 11 +- .../java/org/apache/blur/server/TableContext.java | 24 +++ .../blur/lucene/search/AbstractWrapperQuery.java | 38 ++++- .../org/apache/blur/lucene/search/SuperParser.java | 79 +++++++++ .../org/apache/blur/lucene/search/SuperQuery.java | 32 ++++- .../apache/blur/lucene/search/SuperParserTest.java | 131 +++++++++++++++ 7 files changed, 324 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-core/src/main/java/org/apache/blur/server/LayoutStrategy.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/server/LayoutStrategy.java b/src/blur-core/src/main/java/org/apache/blur/server/LayoutStrategy.java new file mode 100644 index 0000000..9cabe85 --- /dev/null +++ b/src/blur-core/src/main/java/org/apache/blur/server/LayoutStrategy.java @@ -0,0 +1,21 @@ +package org.apache.blur.server; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +public class LayoutStrategy { + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-core/src/main/java/org/apache/blur/server/QueryConverterImpl.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/server/QueryConverterImpl.java b/src/blur-core/src/main/java/org/apache/blur/server/QueryConverterImpl.java index 8b363f9..f9c8aff 100644 --- a/src/blur-core/src/main/java/org/apache/blur/server/QueryConverterImpl.java +++ b/src/blur-core/src/main/java/org/apache/blur/server/QueryConverterImpl.java @@ -20,8 +20,10 @@ import java.io.IOException; import org.apache.blur.analysis.BlurAnalyzer; import org.apache.blur.lucene.LuceneVersionConstant; +import org.apache.blur.lucene.search.ScoreType; +import org.apache.blur.lucene.search.SuperParser; +import org.apache.lucene.index.Term; import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.Query; public class QueryConverterImpl extends Configured implements QueryConverter { @@ -29,10 +31,15 @@ public class QueryConverterImpl extends Configured implements QueryConverter { @Override public Query convert(org.apache.blur.thrift.generated.Query query) throws IOException { TableContext tableContext = getTableContext(); + switch (query.getType()) { case STRING: + ScoreType defaultScoreType = tableContext.getDefaultScoreType(); + Term defaultPrimeDocTerm = tableContext.getDefaultPrimeDocTerm(); BlurAnalyzer analyzer = tableContext.getAnalyzer(); - QueryParser queryParser = new QueryParser(LuceneVersionConstant.LUCENE_VERSION, tableContext.getDefaultFieldName(), analyzer); + String defaultFieldName = tableContext.getDefaultFieldName(); + + SuperParser queryParser = new SuperParser(LuceneVersionConstant.LUCENE_VERSION, defaultFieldName, analyzer, defaultScoreType, defaultPrimeDocTerm); try { return queryParser.parse(query.getQueryString()); } catch (ParseException e) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java index 894f1c0..10285ef 100644 --- a/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java +++ b/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java @@ -28,12 +28,14 @@ import java.util.Map.Entry; import org.apache.blur.analysis.BlurAnalyzer; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; +import org.apache.blur.lucene.search.ScoreType; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.ReflectionUtils; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; +import org.apache.lucene.index.Term; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.Similarity; @@ -57,6 +59,10 @@ public class TableContext { private long timeBetweenRefreshs; private TypeManager typeChecker; + private ScoreType defaultScoreType; + + private Term defaultPrimeDocTerm; + public TypeManager getTypeManager() { return typeChecker; } @@ -86,6 +92,8 @@ public class TableContext { tableContext.timeBetweenCommits = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_COMMITS, 60000); tableContext.timeBetweenRefreshs = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_REFRESHS, 5000); tableContext.typeChecker = typeChecker; + tableContext.defaultPrimeDocTerm = new Term("_primedoc_"); + tableContext.defaultScoreType = ScoreType.SUPER; Class<?> c1 = configuration.getClass(BLUR_LUCENE_INDEX_DELETION_POLICY_CLASS, KeepOnlyLastCommitDeletionPolicy.class); tableContext.indexDeletionPolicy = (IndexDeletionPolicy) configure(ReflectionUtils.newInstance(c1, configuration), tableContext); @@ -198,4 +206,20 @@ public class TableContext { public void setDefaultFieldName(String defaultFieldName) { this.defaultFieldName = defaultFieldName; } + + public Term getDefaultPrimeDocTerm() { + return defaultPrimeDocTerm; + } + + public void setDefaultScoreType(ScoreType defaultScoreType) { + this.defaultScoreType = defaultScoreType; + } + + public void setDefaultPrimeDocTerm(Term defaultPrimeDocTerm) { + this.defaultPrimeDocTerm = defaultPrimeDocTerm; + } + + public ScoreType getDefaultScoreType() { + return defaultScoreType; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-query/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java ---------------------------------------------------------------------- diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java b/src/blur-query/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java index 9f31f3d..8f2b2cc 100644 --- a/src/blur-query/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java +++ b/src/blur-query/src/main/java/org/apache/blur/lucene/search/AbstractWrapperQuery.java @@ -51,10 +51,6 @@ public abstract class AbstractWrapperQuery extends Query { public abstract Weight createWeight(IndexSearcher searcher) throws IOException; - public boolean equals(Object obj) { - return _query.equals(obj); - } - public void extractTerms(Set<Term> terms) { _query.extractTerms(terms); } @@ -67,16 +63,42 @@ public abstract class AbstractWrapperQuery extends Query { return searcher.getSimilarity(); } - public int hashCode() { - return _query.hashCode(); - } - public abstract Query rewrite(IndexReader reader) throws IOException; public void setBoost(float b) { _query.setBoost(b); } + + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((_query == null) ? 0 : _query.hashCode()); + result = prime * result + (_rewritten ? 1231 : 1237); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (!super.equals(obj)) + return false; + if (getClass() != obj.getClass()) + return false; + AbstractWrapperQuery other = (AbstractWrapperQuery) obj; + if (_query == null) { + if (other._query != null) + return false; + } else if (!_query.equals(other._query)) + return false; + if (_rewritten != other._rewritten) + return false; + return true; + } + public abstract String toString(); public abstract String toString(String field); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java ---------------------------------------------------------------------- diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java new file mode 100644 index 0000000..80a53fa --- /dev/null +++ b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperParser.java @@ -0,0 +1,79 @@ +package org.apache.blur.lucene.search; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.Version; + +public class SuperParser extends QueryParser { + + private static final String MUST_NOT_STRING = "-"; + private static final String MUST_STRING = "+"; + private static final Pattern PATTERN = Pattern.compile("([-+]{0,1})\\s*?super\\s*?\\:\\s*?\\<(.*?)\\>"); + private static final Pattern CHECK = Pattern.compile("super\\s*?\\:\\s*?\\<"); + + private final Analyzer a; + private final String f; + private final Version matchVersion; + private final Term defaultPrimeDocTerm; + private final ScoreType defaultScoreType; + + public SuperParser(Version matchVersion, String f, Analyzer a, ScoreType defaultScoreType, Term defaultPrimeDocTerm) { + super(matchVersion, f, a); + this.matchVersion = matchVersion; + this.f = f; + this.a = a; + this.defaultPrimeDocTerm = defaultPrimeDocTerm; + this.defaultScoreType = defaultScoreType; + } + + @Override + public Query parse(String query) throws ParseException { + Matcher matcher = PATTERN.matcher(query); + BooleanQuery booleanQuery = null; + while (matcher.find()) { + int count = matcher.groupCount(); + for (int i = 0; i < count; i++) { + String occurString = matcher.group(i + 1); + i++; + String superQueryStr = matcher.group(i + 1); + Matcher matcherCheck = CHECK.matcher(superQueryStr); + if (matcherCheck.find()) { + throw new ParseException("Embedded super queries are not allowed [" + query + "]."); + } + + if (booleanQuery == null) { + booleanQuery = new BooleanQuery(); + } + + Occur occur = getOccur(occurString); + QueryParser parser = new QueryParser(matchVersion, f, a); + + Query superQuery = parser.parse(superQueryStr); + booleanQuery.add(new SuperQuery(superQuery, defaultScoreType, defaultPrimeDocTerm), occur); + } + } + if (booleanQuery == null) { + return super.parse(query); + } + return booleanQuery; + } + + private Occur getOccur(String occurString) { + if (occurString.equals(MUST_STRING)) { + return Occur.MUST; + } + if (occurString.equals(MUST_NOT_STRING)) { + return Occur.MUST_NOT; + } + return Occur.SHOULD; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java ---------------------------------------------------------------------- diff --git a/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java index 46f62b1..cc0d484 100644 --- a/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java +++ b/src/blur-query/src/main/java/org/apache/blur/lucene/search/SuperQuery.java @@ -74,11 +74,39 @@ public class SuperQuery extends AbstractWrapperQuery { } public String toString() { - return "super:{" + _query.toString() + "}"; + return "super:<" + _query.toString() + ">"; } public String toString(String field) { - return "super:{" + _query.toString(field) + "}"; + return "super:<" + _query.toString(field) + ">"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((primeDocTerm == null) ? 0 : primeDocTerm.hashCode()); + result = prime * result + ((scoreType == null) ? 0 : scoreType.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (!super.equals(obj)) + return false; + if (getClass() != obj.getClass()) + return false; + SuperQuery other = (SuperQuery) obj; + if (primeDocTerm == null) { + if (other.primeDocTerm != null) + return false; + } else if (!primeDocTerm.equals(other.primeDocTerm)) + return false; + if (scoreType != other.scoreType) + return false; + return true; } public static class SuperWeight extends Weight { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d73a6ee9/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java ---------------------------------------------------------------------- diff --git a/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java b/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java new file mode 100644 index 0000000..381ffe1 --- /dev/null +++ b/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java @@ -0,0 +1,131 @@ +package org.apache.blur.lucene.search; + +import static org.junit.Assert.*; + +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class SuperParserTest { + + private SuperParser parser; + + @Before + public void setup() { + parser = new SuperParser(Version.LUCENE_40, "", new WhitespaceAnalyzer(Version.LUCENE_40), ScoreType.SUPER, new Term("_primedoc_")); + } + + @Test + public void testParser1() throws ParseException { + Query query = parser.parse(" +super:<a:a d:e b:b> "); + + BooleanQuery booleanQuery = new BooleanQuery(); + booleanQuery.add(new TermQuery(new Term("a", "a")), Occur.SHOULD); + booleanQuery.add(new TermQuery(new Term("d", "e")), Occur.SHOULD); + booleanQuery.add(new TermQuery(new Term("b", "b")), Occur.SHOULD); + SuperQuery superQuery = new SuperQuery(booleanQuery, ScoreType.SUPER, new Term("_primedoc_")); + + BooleanQuery bq = new BooleanQuery(); + bq.add(superQuery, Occur.MUST); + + assertEquals(bq, query); + + } + + @Test + public void testParser2() throws ParseException { + Query query = parser.parse("super:<c:c d:d>"); + + BooleanQuery booleanQuery = new BooleanQuery(); + booleanQuery.add(new TermQuery(new Term("c", "c")), Occur.SHOULD); + booleanQuery.add(new TermQuery(new Term("d", "d")), Occur.SHOULD); + SuperQuery superQuery = new SuperQuery(booleanQuery, ScoreType.SUPER, new Term("_primedoc_")); + + BooleanQuery bq = new BooleanQuery(); + bq.add(superQuery, Occur.SHOULD); + + assertEquals(bq, query); + } + + @Test + public void testParser3() throws ParseException { + Query query = parser.parse("a:a d:e b:b"); + + BooleanQuery booleanQuery = new BooleanQuery(); + booleanQuery.add(new TermQuery(new Term("a", "a")), Occur.SHOULD); + booleanQuery.add(new TermQuery(new Term("d", "e")), Occur.SHOULD); + booleanQuery.add(new TermQuery(new Term("b", "b")), Occur.SHOULD); + + assertEquals(booleanQuery, query); + } + + @Test + public void testParser4() throws ParseException { + Query query = parser.parse("super:<a:a d:e b:b> - super:<c:c d:d>"); + + BooleanQuery booleanQuery1 = new BooleanQuery(); + booleanQuery1.add(new TermQuery(new Term("a", "a")), Occur.SHOULD); + booleanQuery1.add(new TermQuery(new Term("d", "e")), Occur.SHOULD); + booleanQuery1.add(new TermQuery(new Term("b", "b")), Occur.SHOULD); + + BooleanQuery booleanQuery2 = new BooleanQuery(); + booleanQuery2.add(new TermQuery(new Term("c", "c")), Occur.SHOULD); + booleanQuery2.add(new TermQuery(new Term("d", "d")), Occur.SHOULD); + + SuperQuery superQuery1 = new SuperQuery(booleanQuery1, ScoreType.SUPER, new Term("_primedoc_")); + SuperQuery superQuery2 = new SuperQuery(booleanQuery2, ScoreType.SUPER, new Term("_primedoc_")); + + BooleanQuery booleanQuery = new BooleanQuery(); + booleanQuery.add(superQuery1, Occur.SHOULD); + booleanQuery.add(superQuery2, Occur.MUST_NOT); + + assertEquals(booleanQuery, query); + } + + @Test + public void testParser5() throws ParseException { + + Query query = parser.parse("super:<a:a d:{e TO f} b:b test:hello\\<> - super:<c:c d:d>"); + + BooleanQuery booleanQuery1 = new BooleanQuery(); + booleanQuery1.add(new TermQuery(new Term("a", "a")), Occur.SHOULD); + booleanQuery1.add(new TermRangeQuery("d", new BytesRef("e"), new BytesRef("f"), false, false), Occur.SHOULD); + booleanQuery1.add(new TermQuery(new Term("b", "b")), Occur.SHOULD); + // std analyzer took the "<" out + booleanQuery1.add(new TermQuery(new Term("test", "hello<")), Occur.SHOULD); + + BooleanQuery booleanQuery2 = new BooleanQuery(); + booleanQuery2.add(new TermQuery(new Term("c", "c")), Occur.SHOULD); + booleanQuery2.add(new TermQuery(new Term("d", "d")), Occur.SHOULD); + + SuperQuery superQuery1 = new SuperQuery(booleanQuery1, ScoreType.SUPER, new Term("_primedoc_")); + SuperQuery superQuery2 = new SuperQuery(booleanQuery2, ScoreType.SUPER, new Term("_primedoc_")); + + BooleanQuery booleanQuery = new BooleanQuery(); + booleanQuery.add(superQuery1, Occur.SHOULD); + booleanQuery.add(superQuery2, Occur.MUST_NOT); + + assertEquals(booleanQuery, query); + } + + @Test + public void testParser6() throws ParseException { + SuperParser parser = new SuperParser(Version.LUCENE_40, "", new StandardAnalyzer(Version.LUCENE_40), ScoreType.SUPER, new Term("_primedoc_")); + try { + parser.parse("super : <a:a d:{e TO d} b:b super:<test:hello\\<>> super:<c:c d:d>"); + fail(); + } catch (ParseException e) { + // should throw an error + } + } +}
