Fixed BLUR-115
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/4252e8f6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/4252e8f6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/4252e8f6 Branch: refs/heads/master Commit: 4252e8f660e3ecf91e6cfb2c8d908cf71e0e5dda Parents: 412cb14 Author: Aaron McCurry <[email protected]> Authored: Tue Aug 13 09:42:26 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Aug 13 09:42:26 2013 -0400 ---------------------------------------------------------------------- .../apache/blur/analysis/BaseFieldManager.java | 46 ++++-- .../org/apache/blur/analysis/FieldManager.java | 34 +++- .../blur/analysis/FieldTypeDefinition.java | 9 +- .../apache/blur/analysis/HdfsFieldManager.java | 2 +- .../blur/analysis/ZooKeeperFieldManager.java | 2 +- .../type/DoubleFieldTypeDefinition.java | 2 +- .../type/FieldLessFieldTypeDefinition.java | 6 +- .../analysis/type/FloatFieldTypeDefinition.java | 2 +- .../analysis/type/IntFieldTypeDefinition.java | 2 +- .../analysis/type/LongFieldTypeDefinition.java | 2 +- .../type/NumericFieldTypeDefinition.java | 9 +- .../type/SpatialFieldTypeDefinition.java | 158 +++++++++++++++++++ .../type/StoredFieldTypeDefinition.java | 7 +- .../type/StringFieldTypeDefinition.java | 6 +- .../analysis/type/TextFieldTypeDefinition.java | 6 +- .../type/spatial/SpatialArgsParser.java | 145 +++++++++++++++++ .../blur/lucene/search/BlurQueryParser.java | 45 +++++- .../blur/lucene/search/SuperParserTest.java | 48 ++++-- blur-store/pom.xml | 5 + distribution/src/assemble/bin.xml | 2 + 20 files changed, 486 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java index 1b2ef0c..b29a3a5 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java @@ -30,6 +30,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.blur.analysis.type.DoubleFieldTypeDefinition; import org.apache.blur.analysis.type.FieldLessFieldTypeDefinition; import org.apache.blur.analysis.type.FloatFieldTypeDefinition; +import org.apache.blur.analysis.type.SpatialFieldTypeDefinition; import org.apache.blur.analysis.type.IntFieldTypeDefinition; import org.apache.blur.analysis.type.LongFieldTypeDefinition; import org.apache.blur.analysis.type.NumericFieldTypeDefinition; @@ -96,6 +97,7 @@ public abstract class BaseFieldManager extends FieldManager { _typeMap.put(LongFieldTypeDefinition.NAME, LongFieldTypeDefinition.class); _typeMap.put(DoubleFieldTypeDefinition.NAME, DoubleFieldTypeDefinition.class); _typeMap.put(FloatFieldTypeDefinition.NAME, FloatFieldTypeDefinition.class); + _typeMap.put(SpatialFieldTypeDefinition.NAME, SpatialFieldTypeDefinition.class); _fieldLessField = fieldLessField; _strict = strict; _defaultMissingFieldLessIndexing = defaultMissingFieldLessIndexing; @@ -276,7 +278,7 @@ public abstract class BaseFieldManager extends FieldManager { if (!tryToStore(fieldName, fieldLessIndexing, fieldType, props)) { return false; } - fieldTypeDefinition = newFieldTypeDefinition(fieldLessIndexing, fieldType, props); + fieldTypeDefinition = newFieldTypeDefinition(fieldName, fieldLessIndexing, fieldType, props); registerFieldTypeDefinition(fieldName, fieldTypeDefinition); return true; } @@ -309,7 +311,7 @@ public abstract class BaseFieldManager extends FieldManager { return fieldName.substring(0, indexOf); } - protected FieldTypeDefinition newFieldTypeDefinition(boolean fieldLessIndexing, String fieldType, + protected FieldTypeDefinition newFieldTypeDefinition(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props) { if (fieldType == null) { throw new IllegalArgumentException("Field type can not be null."); @@ -329,9 +331,9 @@ public abstract class BaseFieldManager extends FieldManager { throw new RuntimeException(e); } if (props == null) { - fieldTypeDefinition.configure(EMPTY_MAP); + fieldTypeDefinition.configure(fieldName, EMPTY_MAP); } else { - fieldTypeDefinition.configure(props); + fieldTypeDefinition.configure(fieldName, props); } fieldTypeDefinition.setFieldLessIndexing(fieldLessIndexing); return fieldTypeDefinition; @@ -374,6 +376,10 @@ public abstract class BaseFieldManager extends FieldManager { return fieldTypeDefinition.getAnalyzerForQuery(); } + public void addColumnDefinitionGis(String family, String columnName) throws IOException { + addColumnDefinition(family, columnName, null, false, SpatialFieldTypeDefinition.NAME, null); + } + public void addColumnDefinitionInt(String family, String columnName) throws IOException { addColumnDefinition(family, columnName, null, false, IntFieldTypeDefinition.NAME, null); } @@ -416,39 +422,48 @@ public abstract class BaseFieldManager extends FieldManager { public boolean isFieldLessIndexed(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); if (fieldTypeDefinition == null) { - throw new IOException("FieldTypeDefinition for field [" + field + "] is missing."); + return false; } return fieldTypeDefinition.isFieldLessIndexed(); } @Override - public boolean checkSupportForFuzzyQuery(String field) throws IOException { + public Boolean checkSupportForFuzzyQuery(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); if (fieldTypeDefinition == null) { - throw new IOException("FieldTypeDefinition for field [" + field + "] is missing."); + return null; } return fieldTypeDefinition.checkSupportForFuzzyQuery(); } @Override - public boolean checkSupportForPrefixQuery(String field) throws IOException { + public Boolean checkSupportForPrefixQuery(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); if (fieldTypeDefinition == null) { - throw new IOException("FieldTypeDefinition for field [" + field + "] is missing."); + return null; } return fieldTypeDefinition.checkSupportForPrefixQuery(); } @Override - public boolean checkSupportForWildcardQuery(String field) throws IOException { + public Boolean checkSupportForWildcardQuery(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); if (fieldTypeDefinition == null) { - throw new IOException("FieldTypeDefinition for field [" + field + "] is missing."); + return null; } return fieldTypeDefinition.checkSupportForWildcardQuery(); } @Override + public Boolean checkSupportForCustomQuery(String field) throws IOException { + FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); + if (fieldTypeDefinition == null) { + return null; + } + return fieldTypeDefinition.checkSupportForCustomQuery(); + } + + @Override public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); @@ -483,6 +498,15 @@ public abstract class BaseFieldManager extends FieldManager { } @Override + public Query getCustomQuery(String field, String text) throws IOException { + FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); + if (fieldTypeDefinition == null) { + throw new IOException("Field [" + field + "] is missing."); + } + return fieldTypeDefinition.getCustomQuery(text); + } + + @Override public String getFieldLessFieldName() { return _fieldLessField; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java index 519e397..ac95494 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java @@ -105,30 +105,40 @@ public abstract class FieldManager { * * @param field * the field name. - * @return boolean + * @return Boolean null if unknown. * @throws IOException */ - public abstract boolean checkSupportForFuzzyQuery(String field) throws IOException; + public abstract Boolean checkSupportForFuzzyQuery(String field) throws IOException; /** * Checks if this field supports prefix queries. * * @param field * the field name. - * @return boolean + * @return Boolean null if unknown. * @throws IOException */ - public abstract boolean checkSupportForPrefixQuery(String field) throws IOException; + public abstract Boolean checkSupportForPrefixQuery(String field) throws IOException; /** * Checks if this field supports wildcard queries. * * @param field * the field name. - * @return boolean + * @return Boolean null if unknown. + * @throws IOException + */ + public abstract Boolean checkSupportForWildcardQuery(String field) throws IOException; + + /** + * Checks to see if the field will deal with creating it's own query objects. + * + * @param field + * the field name. + * @return Boolean null if unknown. * @throws IOException */ - public abstract boolean checkSupportForWildcardQuery(String field) throws IOException; + public abstract Boolean checkSupportForCustomQuery(String field) throws IOException; /** * Gets a range query, if the method returns null the default term range query @@ -228,4 +238,16 @@ public abstract class FieldManager { */ public abstract boolean isStrict(); + /** + * Gets a custom query for the given field and text. + * + * @param field + * the field name. + * @param text + * the text to create the custom query. + * @return the query object. + * @throws IOException + */ + public abstract Query getCustomQuery(String field, String text) throws IOException; + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java index b6c511a..469decd 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java @@ -23,6 +23,7 @@ import org.apache.blur.thrift.generated.Column; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.search.Query; public abstract class FieldTypeDefinition { @@ -41,7 +42,7 @@ public abstract class FieldTypeDefinition { * @param properties * the properties. */ - public abstract void configure(Map<String, String> properties); + public abstract void configure(String fieldNameForThisInstance, Map<String, String> properties); /** * Gets the {@link Field}s for indexing from a single Column. @@ -147,4 +148,10 @@ public abstract class FieldTypeDefinition { public abstract boolean isNumeric(); + public abstract boolean checkSupportForCustomQuery(); + + public Query getCustomQuery(String text) { + throw new RuntimeException("Not supported."); + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java index e05e09e..2cbd9cb 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java @@ -140,7 +140,7 @@ public class HdfsFieldManager extends BaseFieldManager { inputStream.close(); boolean fieldLessIndexing = Boolean.parseBoolean(props.getProperty(FIELD_LESS_INDEXING)); String fieldType = props.getProperty(FIELD_TYPE); - FieldTypeDefinition fieldTypeDefinition = newFieldTypeDefinition(fieldLessIndexing, fieldType, toMap(props)); + FieldTypeDefinition fieldTypeDefinition = newFieldTypeDefinition(fieldName, fieldLessIndexing, fieldType, toMap(props)); registerFieldTypeDefinition(fieldName, fieldTypeDefinition); } finally { _lock.unlock(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java index ccbe4f5..65b3030 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java @@ -143,7 +143,7 @@ public class ZooKeeperFieldManager extends BaseFieldManager { inputStream.close(); boolean fieldLessIndexing = Boolean.parseBoolean(props.getProperty(FIELD_LESS_INDEXING)); String fieldType = props.getProperty(FIELD_TYPE); - FieldTypeDefinition fieldTypeDefinition = newFieldTypeDefinition(fieldLessIndexing, fieldType, toMap(props)); + FieldTypeDefinition fieldTypeDefinition = newFieldTypeDefinition(fieldName, fieldLessIndexing, fieldType, toMap(props)); registerFieldTypeDefinition(fieldName, fieldTypeDefinition); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/type/DoubleFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/DoubleFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/DoubleFieldTypeDefinition.java index 7bedbc8..c995c49 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/type/DoubleFieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/DoubleFieldTypeDefinition.java @@ -37,7 +37,7 @@ public class DoubleFieldTypeDefinition extends NumericFieldTypeDefinition { } @Override - public void configure(Map<String, String> properties) { + public void configure(String fieldNameForThisInstance, Map<String, String> properties) { String precisionStepStr = properties.get(NUMERIC_PRECISION_STEP); if (precisionStepStr != null) { _precisionStep = Integer.parseInt(precisionStepStr); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java index f6ca6f6..74aa4f9 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java @@ -43,7 +43,7 @@ public class FieldLessFieldTypeDefinition extends FieldTypeDefinition { } @Override - public void configure(Map<String, String> properties) { + public void configure(String fieldNameForThisInstance, Map<String, String> properties) { } @@ -101,4 +101,8 @@ public class FieldLessFieldTypeDefinition extends FieldTypeDefinition { return false; } + @Override + public boolean checkSupportForCustomQuery() { + return false; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/type/FloatFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/FloatFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/FloatFieldTypeDefinition.java index 452518b..c6e4f2a 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/type/FloatFieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/FloatFieldTypeDefinition.java @@ -37,7 +37,7 @@ public class FloatFieldTypeDefinition extends NumericFieldTypeDefinition { } @Override - public void configure(Map<String, String> properties) { + public void configure(String fieldNameForThisInstance, Map<String, String> properties) { String precisionStepStr = properties.get(NUMERIC_PRECISION_STEP); if (precisionStepStr != null) { _precisionStep = Integer.parseInt(precisionStepStr); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/type/IntFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/IntFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/IntFieldTypeDefinition.java index 7cd16f0..1229aab 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/type/IntFieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/IntFieldTypeDefinition.java @@ -37,7 +37,7 @@ public class IntFieldTypeDefinition extends NumericFieldTypeDefinition { } @Override - public void configure(Map<String, String> properties) { + public void configure(String fieldNameForThisInstance, Map<String, String> properties) { String precisionStepStr = properties.get(NUMERIC_PRECISION_STEP); if (precisionStepStr != null) { _precisionStep = Integer.parseInt(precisionStepStr); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/type/LongFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/LongFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/LongFieldTypeDefinition.java index bbfb688..14fe720 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/type/LongFieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/LongFieldTypeDefinition.java @@ -37,7 +37,7 @@ public class LongFieldTypeDefinition extends NumericFieldTypeDefinition { } @Override - public void configure(Map<String, String> properties) { + public void configure(String fieldNameForThisInstance, Map<String, String> properties) { String precisionStepStr = properties.get(NUMERIC_PRECISION_STEP); if (precisionStepStr != null) { _precisionStep = Integer.parseInt(precisionStepStr); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/type/NumericFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/NumericFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/NumericFieldTypeDefinition.java index 86f9417..69f016f 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/type/NumericFieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/NumericFieldTypeDefinition.java @@ -23,9 +23,9 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.NumericUtils; public abstract class NumericFieldTypeDefinition extends FieldTypeDefinition { - + protected static final String NUMERIC_PRECISION_STEP = "numericPrecisionStep"; - + protected int _precisionStep = NumericUtils.PRECISION_STEP_DEFAULT; @Override @@ -59,6 +59,11 @@ public abstract class NumericFieldTypeDefinition extends FieldTypeDefinition { return true; } + @Override + public final boolean checkSupportForCustomQuery() { + return false; + } + public abstract Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/type/SpatialFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/SpatialFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/SpatialFieldTypeDefinition.java new file mode 100644 index 0000000..fa36cfe --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/SpatialFieldTypeDefinition.java @@ -0,0 +1,158 @@ +package org.apache.blur.analysis.type; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.blur.analysis.FieldTypeDefinition; +import org.apache.blur.analysis.type.spatial.SpatialArgsParser; +import org.apache.blur.thrift.generated.Column; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.search.Query; +import org.apache.lucene.spatial.SpatialStrategy; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; +import org.apache.lucene.spatial.query.SpatialArgs; + +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.io.ShapeReadWriter; +import com.spatial4j.core.shape.Shape; + +public class SpatialFieldTypeDefinition extends FieldTypeDefinition { + + private static final String MAX_LEVELS = "maxLevels"; + public static final String NAME = "spatial"; + + private SpatialStrategy _strategy; + private SpatialContext _ctx; + private ShapeReadWriter<SpatialContext> _shapeReadWriter; + private final Analyzer _queryAnalyzer = new KeywordAnalyzer(); + + @Override + public String getName() { + return NAME; + } + + @Override + public void configure(String fieldNameForThisInstance, Map<String, String> properties) { + _ctx = SpatialContext.GEO; + String maxLevelsStr = properties.get(MAX_LEVELS); + int maxLevels = 11; + if (maxLevelsStr != null) { + maxLevels = Integer.parseInt(maxLevelsStr.trim()); + } + SpatialPrefixTree grid = new GeohashPrefixTree(_ctx, maxLevels); + _strategy = new RecursivePrefixTreeStrategy(grid, fieldNameForThisInstance); + _shapeReadWriter = new ShapeReadWriter<SpatialContext>(_ctx); + } + + @Override + public Iterable<? extends Field> getFieldsForColumn(String family, Column column) { + String name = getName(family, column.getName()); + if (!_strategy.getFieldName().equals(name)) { + throw new RuntimeException("Strategy name and column name do not match."); + } + List<Field> fields = new ArrayList<Field>(); + Shape shape = getShape(column); + for (Field f : _strategy.createIndexableFields(shape)) { + fields.add(f); + } + fields.add(new StoredField(name, cleanup(shape))); + return fields; + } + + private String cleanup(Shape shape) { + return _shapeReadWriter.writeShape(shape); + } + + @Override + public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column, String subName) { + String name = getName(family, column.getName(), subName); + if (!_strategy.getFieldName().equals(name)) { + throw new RuntimeException("Strategy name and column name do not match."); + } + List<Field> fields = new ArrayList<Field>(); + Shape shape = getShape(column); + for (Field f : _strategy.createIndexableFields(shape)) { + fields.add(f); + } + return fields; + } + + private Shape getShape(Column column) { + return _shapeReadWriter.readShape(column.getValue()); + } + + @Override + public Query getCustomQuery(String text) { + SpatialArgs args = SpatialArgsParser.parse(text, _shapeReadWriter); + return _strategy.makeQuery(args); + } + + @Override + public FieldType getStoredFieldType() { + throw new RuntimeException("Not supported"); + } + + @Override + public FieldType getNotStoredFieldType() { + throw new RuntimeException("Not supported"); + } + + @Override + public Analyzer getAnalyzerForIndex() { + throw new RuntimeException("Not supported"); + } + + @Override + public Analyzer getAnalyzerForQuery() { + return _queryAnalyzer; + } + + @Override + public boolean checkSupportForFuzzyQuery() { + return false; + } + + @Override + public boolean checkSupportForWildcardQuery() { + return false; + } + + @Override + public boolean checkSupportForPrefixQuery() { + return false; + } + + @Override + public boolean isNumeric() { + return false; + } + + @Override + public boolean checkSupportForCustomQuery() { + return true; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/type/StoredFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/StoredFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/StoredFieldTypeDefinition.java index a77e4bd..3febf40 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/type/StoredFieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/StoredFieldTypeDefinition.java @@ -38,7 +38,7 @@ public class StoredFieldTypeDefinition extends FieldTypeDefinition { } @Override - public void configure(Map<String, String> properties) { + public void configure(String fieldNameForThisInstance, Map<String, String> properties) { } @@ -93,5 +93,10 @@ public class StoredFieldTypeDefinition extends FieldTypeDefinition { public boolean isNumeric() { return false; } + + @Override + public boolean checkSupportForCustomQuery() { + return false; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/type/StringFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/StringFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/StringFieldTypeDefinition.java index 9ac7c8f..221720b 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/type/StringFieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/StringFieldTypeDefinition.java @@ -36,7 +36,7 @@ public class StringFieldTypeDefinition extends FieldTypeDefinition { } @Override - public void configure(Map<String, String> properties) { + public void configure(String fieldNameForThisInstance, Map<String, String> properties) { } @@ -95,4 +95,8 @@ public class StringFieldTypeDefinition extends FieldTypeDefinition { return false; } + @Override + public boolean checkSupportForCustomQuery() { + return false; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java index 914ceb2..b64caed 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java @@ -48,7 +48,7 @@ public class TextFieldTypeDefinition extends FieldTypeDefinition { } @Override - public void configure(Map<String, String> properties) { + public void configure(String fieldNameForThisInstance, Map<String, String> properties) { } @@ -106,4 +106,8 @@ public class TextFieldTypeDefinition extends FieldTypeDefinition { return false; } + @Override + public boolean checkSupportForCustomQuery() { + return false; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialArgsParser.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialArgsParser.java b/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialArgsParser.java new file mode 100644 index 0000000..1afcaad --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialArgsParser.java @@ -0,0 +1,145 @@ +package org.apache.blur.analysis.type.spatial; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.exception.InvalidShapeException; +import com.spatial4j.core.io.ShapeReadWriter; +import com.spatial4j.core.shape.Shape; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.StringTokenizer; + +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialOperation; + +/** + * Parses a string that usually looks like "OPERATION(SHAPE)" into a + * {@link SpatialArgs} object. The set of operations supported are defined in + * {@link SpatialOperation}, such as "Intersects" being a common one. The shape + * portion is defined by {@link ShapeReadWriter#readShape(String)}. There are + * some optional name-value pair parameters that follow the closing parenthesis. + * Example: + * + * <pre> + * Intersects(-10,20,-8,22) distErrPct=0.025 + * </pre> + * <p/> + * In the future it would be good to support something at least + * semi-standardized like a variant of <a href= + * "http://docs.geoserver.org/latest/en/user/filter/ecql_reference.html#spatial-predicate"; + * > [E]CQL</a>. + * + * @lucene.experimental + */ +public class SpatialArgsParser { + + public static final String DIST_ERR_PCT = "distErrPct"; + public static final String DIST_ERR = "distErr"; + + /** Writes a close approximation to the parsed input format. */ + public static String writeSpatialArgs(SpatialArgs args, ShapeReadWriter<SpatialContext> shapeReadWriter) { + StringBuilder str = new StringBuilder(); + str.append(args.getOperation().getName()); + str.append('('); + str.append(shapeReadWriter.writeShape(args.getShape())); + if (args.getDistErrPct() != null) + str.append(" distErrPct=").append(String.format(Locale.ROOT, "%.2f%%", args.getDistErrPct() * 100d)); + if (args.getDistErr() != null) + str.append(" distErr=").append(args.getDistErr()); + str.append(')'); + return str.toString(); + } + + /** + * Parses a string such as "Intersects(-10,20,-8,22) distErrPct=0.025". + * + * @param v + * The string to parse. Mandatory. + * @param shapeReadWriter + * The spatial shapeReadWriter. Mandatory. + * @return Not null. + * @throws IllegalArgumentException + * If there is a problem parsing the string. + * @throws InvalidShapeException + * Thrown from {@link ShapeReadWriter#readShape(String)} + */ + public static SpatialArgs parse(String v, ShapeReadWriter<SpatialContext> shapeReadWriter) throws IllegalArgumentException, InvalidShapeException { + int idx = v.indexOf('('); + int edx = v.lastIndexOf(')'); + + if (idx < 0 || idx > edx) { + throw new IllegalArgumentException("missing parens: " + v, null); + } + + SpatialOperation op = SpatialOperation.get(v.substring(0, idx).trim()); + + String body = v.substring(idx + 1, edx).trim(); + if (body.length() < 1) { + throw new IllegalArgumentException("missing body : " + v, null); + } + + Shape shape = shapeReadWriter.readShape(body); + SpatialArgs args = new SpatialArgs(op, shape); + + if (v.length() > (edx + 1)) { + body = v.substring(edx + 1).trim(); + if (body.length() > 0) { + Map<String, String> aa = parseMap(body); + args.setDistErrPct(readDouble(aa.remove(DIST_ERR_PCT))); + args.setDistErr(readDouble(aa.remove(DIST_ERR))); + if (!aa.isEmpty()) { + throw new IllegalArgumentException("unused parameters: " + aa, null); + } + } + } + args.validate(); + return args; + } + + protected static Double readDouble(String v) { + return v == null ? null : Double.valueOf(v); + } + + protected static boolean readBool(String v, boolean defaultValue) { + return v == null ? defaultValue : Boolean.parseBoolean(v); + } + + /** + * Parses "a=b c=d f" (whitespace separated) into name-value pairs. If there + * is no '=' as in 'f' above then it's short for f=f. + */ + protected static Map<String, String> parseMap(String body) { + Map<String, String> map = new HashMap<String, String>(); + StringTokenizer st = new StringTokenizer(body, " \n\t"); + while (st.hasMoreTokens()) { + String a = st.nextToken(); + int idx = a.indexOf('='); + if (idx > 0) { + String k = a.substring(0, idx); + String v = a.substring(idx + 1); + map.put(k, v); + } else { + map.put(a, a); + } + } + return map; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java b/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java index 169ada4..9666cc9 100644 --- a/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java +++ b/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java @@ -47,13 +47,15 @@ public class BlurQueryParser extends QueryParser { protected Query newFuzzyQuery(Term term, float minimumSimilarity, int prefixLength) { String field = term.field(); try { - if (!_fieldManager.checkSupportForFuzzyQuery(field)) { + Boolean b = _fieldManager.checkSupportForFuzzyQuery(field); + if (!(b == null || b)) { throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field) + "] which does not support fuzzy queries."); } } catch (IOException e) { throw new RuntimeException(e); } + customQueryCheck(field); return addField(super.newFuzzyQuery(term, minimumSimilarity, prefixLength), term.field()); } @@ -69,7 +71,9 @@ public class BlurQueryParser extends QueryParser { public void add(Term[] terms, int position) { super.add(terms, position); for (Term term : terms) { - addField(this, term.field()); + String field = term.field(); + customQueryCheck(field); + addField(this, field); } } }; @@ -78,31 +82,35 @@ public class BlurQueryParser extends QueryParser { @Override protected PhraseQuery newPhraseQuery() { return new PhraseQuery() { - @Override public void add(Term term, int position) { super.add(term, position); - addField(this, term.field()); + String field = term.field(); + customQueryCheck(field); + addField(this, field); } }; } - + @Override protected Query newPrefixQuery(Term prefix) { String field = prefix.field(); try { - if (!_fieldManager.checkSupportForPrefixQuery(field)) { + Boolean b = _fieldManager.checkSupportForPrefixQuery(field); + if (!(b == null || b)) { throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field) + "] which does not support prefix queries."); } } catch (IOException e) { throw new RuntimeException(e); } + customQueryCheck(field); return addField(super.newPrefixQuery(prefix), field); } @Override protected Query newRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) { + customQueryCheck(field); Query q; try { q = _fieldManager.getNewRangeQuery(field, part1, part2, startInclusive, endInclusive); @@ -115,15 +123,36 @@ public class BlurQueryParser extends QueryParser { return addField(super.newRangeQuery(field, part1, part2, startInclusive, endInclusive), field); } + private void customQueryCheck(String field) { + try { + Boolean b = _fieldManager.checkSupportForCustomQuery(field); + if (b !=null && b) { + throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field) + + "] queries should exist with \" around them."); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override protected Query newTermQuery(Term term) { String field = term.field(); + try { + Boolean b = _fieldManager.checkSupportForCustomQuery(field); + if (b != null && b) { + return _fieldManager.getCustomQuery(field, term.text()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } Query q; try { q = _fieldManager.getTermQueryIfNumeric(field, term.text()); } catch (IOException e) { throw new RuntimeException(e); } + if (q != null) { return addField(q, field); } @@ -137,13 +166,15 @@ public class BlurQueryParser extends QueryParser { } String field = t.field(); try { - if (!_fieldManager.checkSupportForWildcardQuery(field)) { + Boolean b = _fieldManager.checkSupportForWildcardQuery(field); + if (!(b == null || b)) { throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field) + "] which does not support wildcard queries."); } } catch (IOException e) { throw new RuntimeException(e); } + customQueryCheck(field); return addField(super.newWildcardQuery(t), t.field()); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java index f2c8b94..165765c 100644 --- a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java +++ b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java @@ -2,6 +2,7 @@ package org.apache.blur.lucene.search; import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; @@ -10,6 +11,7 @@ import java.util.Map; import org.apache.blur.analysis.BaseFieldManager; import org.apache.blur.analysis.NoStopWordStandardAnalyzer; +import org.apache.blur.analysis.type.spatial.SpatialArgsParser; import org.apache.blur.thrift.generated.ScoreType; import org.apache.blur.utils.BlurConstants; import org.apache.lucene.analysis.Analyzer; @@ -23,10 +25,20 @@ import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.util.BytesRef; import org.junit.Before; import org.junit.Test; +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.distance.DistanceUtils; +import com.spatial4j.core.io.ShapeReadWriter; +import com.spatial4j.core.shape.Circle; + public class SuperParserTest { private SuperParser parser; @@ -34,21 +46,6 @@ public class SuperParserTest { @Before public void setup() throws IOException { - // AnalyzerDefinition ad = new AnalyzerDefinition(); - // ad.setDefaultDefinition(new - // ColumnDefinition(NoStopWordStandardAnalyzer.class.getName(), true, - // null)); - // ColumnFamilyDefinition cfDef = new ColumnFamilyDefinition(); - // cfDef.putToColumnDefinitions("id_l", new ColumnDefinition("long", false, - // null)); - // cfDef.putToColumnDefinitions("id_d", new ColumnDefinition("double", - // false, null)); - // cfDef.putToColumnDefinitions("id_f", new ColumnDefinition("float", false, - // null)); - // cfDef.putToColumnDefinitions("id_i", new ColumnDefinition("integer", - // false, null)); - // ad.putToColumnFamilyDefinitions("a", cfDef); - _fieldManager = getFieldManager(new NoStopWordStandardAnalyzer()); parser = new SuperParser(LUCENE_VERSION, _fieldManager, true, null, ScoreType.SUPER, new Term("_primedoc_")); } @@ -71,6 +68,7 @@ public class SuperParserTest { fieldManager.addColumnDefinitionDouble("a", "id_d"); fieldManager.addColumnDefinitionFloat("a", "id_f"); fieldManager.addColumnDefinitionLong("a", "id_l"); + fieldManager.addColumnDefinitionGis("a", "id_gis"); return fieldManager; } @@ -324,6 +322,26 @@ public class SuperParserTest { assertQuery(q1, q); } + @Test + public void test28() throws ParseException { + SpatialContext ctx = SpatialContext.GEO; + ShapeReadWriter<SpatialContext> shapeReadWriter = new ShapeReadWriter<SpatialContext>(ctx); + int maxLevels = 11; + SpatialPrefixTree grid = new GeohashPrefixTree(ctx, maxLevels); + RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy(grid, "a.id_gis"); + Circle circle = ctx.makeCircle(-80.0, 33.0, DistanceUtils.dist2Degrees(200, DistanceUtils.EARTH_MEAN_RADIUS_KM)); + SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, circle); + + String writeSpatialArgs = SpatialArgsParser.writeSpatialArgs(args, shapeReadWriter); + + // This has to be done because of rounding. + SpatialArgs spatialArgs = SpatialArgsParser.parse(writeSpatialArgs, shapeReadWriter); + Query q1 = sq(strategy.makeQuery(spatialArgs)); + Query q = parseSq("a.id_gis:\"" + writeSpatialArgs + "\""); + boolean equals = q1.equals(q); + assertTrue(equals); + } + public static BooleanClause bc_m(Query q) { return new BooleanClause(q, Occur.MUST); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/blur-store/pom.xml ---------------------------------------------------------------------- diff --git a/blur-store/pom.xml b/blur-store/pom.xml index d309498..1bc2be8 100644 --- a/blur-store/pom.xml +++ b/blur-store/pom.xml @@ -63,6 +63,11 @@ under the License. <version>${lucene.version}</version> </dependency> <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-spatial</artifactId> + <version>${lucene.version}</version> + </dependency> + <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> <version>${commons-cli.version}</version> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4252e8f6/distribution/src/assemble/bin.xml ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml index 288c114..b455807 100644 --- a/distribution/src/assemble/bin.xml +++ b/distribution/src/assemble/bin.xml @@ -43,6 +43,7 @@ under the License. <include>com.google.guava:guava</include> <include>org.apache.httpcomponents:*</include> <include>org.apache.lucene:*</include> + <include>com.spatial4j:spatial4j</include> <include>commons-cli:commons-cli</include> <include>com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru</include> <include>jline:jline</include> @@ -72,6 +73,7 @@ under the License. <exclude>com.google.guava:guava</exclude> <exclude>org.apache.httpcomponents:*</exclude> <exclude>org.apache.lucene:*</exclude> + <exclude>com.spatial4j:spatial4j</exclude> <exclude>commons-cli:commons-cli</exclude> <exclude>com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru</exclude> <exclude>jline:jline</exclude>
