Updated Branches: refs/heads/apache-blur-0.2 261613b63 -> ea6470880
Fixed BLUR-258 Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/ea647088 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/ea647088 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/ea647088 Branch: refs/heads/apache-blur-0.2 Commit: ea6470880b241c44dbefead6c579fbea9c74dd54 Parents: 261613b Author: Aaron McCurry <[email protected]> Authored: Tue Oct 15 09:12:21 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Oct 15 09:12:21 2013 -0400 ---------------------------------------------------------------------- .../type/CustomFieldTypeDefinition.java | 60 +++++++- .../apache/blur/analysis/type/ExampleType.java | 110 +++++++++++++++ ...lPointVectorStrategyFieldTypeDefinition.java | 10 -- docs/data-model.html | 138 +++++++++++++++++++ 4 files changed, 305 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea647088/blur-query/src/main/java/org/apache/blur/analysis/type/CustomFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/CustomFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/CustomFieldTypeDefinition.java index 936541c..d098b86 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/type/CustomFieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/CustomFieldTypeDefinition.java @@ -19,45 +19,99 @@ package org.apache.blur.analysis.type; import org.apache.blur.analysis.FieldTypeDefinition; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.WildcardQuery; public abstract class CustomFieldTypeDefinition extends FieldTypeDefinition { private static final String NOT_SUPPORTED = "Not supported"; private final Analyzer _queryAnalyzer = new KeywordAnalyzer(); + /** + * The {@link #getAnalyzerForIndex(String)} should never be called for a + * {@link CustomFieldTypeDefinition} because this type will create the + * {@link IndexableField} objects from the + * {@link #getFieldsForColumn(String, Column)} method or + * {@link #getFieldsForSubColumn(String, Column, String)} method. + * + * @throws @{@link RuntimeException}. + */ @Override - public Analyzer getAnalyzerForIndex(String fieldName) { + public final Analyzer getAnalyzerForIndex(String fieldName) { throw new RuntimeException(NOT_SUPPORTED); } + /** + * A {@link KeywordAnalyzer} is used to parse all the queries for + * {@link CustomFieldTypeDefinition} types. That means that your query string + * must contain all the parts of your query to be passed to the + * {@link #getCustomQuery(String)} method. + * + * @return {@link KeywordAnalyzer}. + */ @Override - public Analyzer getAnalyzerForQuery(String fieldName) { + public final Analyzer getAnalyzerForQuery(String fieldName) { return _queryAnalyzer; } + /** + * Custom query types do not support {@link FuzzyQuery}. + * + * @return false. + */ @Override public final boolean checkSupportForFuzzyQuery() { return false; } + /** + * Custom query types do not support {@link WildcardQuery}. + * + * @return false. + */ @Override public final boolean checkSupportForWildcardQuery() { return false; } + /** + * Custom query types do not support {@link PrefixQuery}. + * + * @return false. + */ @Override public final boolean checkSupportForPrefixQuery() { return false; } + /** + * Checks whether this type is numeric or not. If so you need this type to be + * numeric please extend {@link NumericFieldTypeDefinition} instead of this + * class. + * + * @return false. + */ @Override public final boolean isNumeric() { return false; } + /** + * By returning true this type will create {@link Query} objects from + * {@link #getCustomQuery(String)} method where the entire string from the + * query parser is passed to the method. {@link #getCustomQuery(String)} + * method (true). If you want to use the {@link #getAnalyzerForIndex(String)} + * method to create your query, please extend {@link TextFieldTypeDefinition} + * or {@link FieldTypeDefinition}. + * + * @return true. + */ @Override public final boolean checkSupportForCustomQuery() { return true; } - + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea647088/blur-query/src/main/java/org/apache/blur/analysis/type/ExampleType.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/ExampleType.java b/blur-query/src/main/java/org/apache/blur/analysis/type/ExampleType.java new file mode 100644 index 0000000..6decee8 --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/ExampleType.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.analysis.type; + +import java.util.Map; + +import org.apache.blur.thrift.generated.Blur; +import org.apache.blur.thrift.generated.Column; +import org.apache.hadoop.conf.Configuration; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; + +public class ExampleType extends CustomFieldTypeDefinition { + + private String _fieldNameForThisInstance; + + /** + * Get the name of the type. + * + * @return the name. + */ + @Override + public String getName() { + return "example"; + } + + /** + * Configures this instance for the type. + * + * @param fieldNameForThisInstance + * the field name for this instance. + * @param properties + * the properties passed into this type definition from the + * {@link Blur.Iface#addColumnDefinition(String, org.apache.blur.thrift.generated.ColumnDefinition)} + * method. + */ + @Override + public void configure(String fieldNameForThisInstance, Map<String, String> properties, Configuration configuration) { + _fieldNameForThisInstance = fieldNameForThisInstance; + } + + /** + * Create {@link Field}s for the index as well as for storing the original + * data for retrieval. + * + * @param family + * the family name. + * @param column + * the column that holds the name and value. + * + * @return the {@link Iterable} of {@link Field}s. + */ + @Override + public Iterable<? extends Field> getFieldsForColumn(String family, Column column) { + String name = family + "." + column.getName(); + String value = column.getValue(); + return makeIterable(new StringField(name, value, Store.YES)); + } + + /** + * Create {@link Field}s for the index do NOT store the data because the is a + * sub column. + * + * @param family + * the family name. + * @param column + * the column that holds the name and value. + * @param subName + * the sub column name. + * + * @return the {@link Iterable} of {@link Field}s. + */ + @Override + public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column, String subName) { + String name = family + "." + column.getName() + "." + subName; + String value = column.getValue(); + return makeIterable(new StringField(name, value, Store.NO)); + } + + /** + * Gets the query from the text provided by the query parser. + * + * @param text + * the text provided by the query parser. + * @return the {@link Query}. + */ + @Override + public Query getCustomQuery(String text) { + return new TermQuery(new Term(_fieldNameForThisInstance, text)); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea647088/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialPointVectorStrategyFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialPointVectorStrategyFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialPointVectorStrategyFieldTypeDefinition.java index fa3eec6..ace14ee 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialPointVectorStrategyFieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/spatial/SpatialPointVectorStrategyFieldTypeDefinition.java @@ -21,8 +21,6 @@ import java.util.Collection; import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.vector.PointVectorStrategy; @@ -55,12 +53,4 @@ public class SpatialPointVectorStrategyFieldTypeDefinition extends BaseSpatialFi public Collection<String> getAlternateFieldNames() { return _alternateFieldNames; } - - @Override - public Analyzer getAnalyzerForIndex(String fieldName) { - // This happens because the doubles are tokenized but the analyzer is - // fetched from the Field, but the offset is still fetched from the base - // analyzer. This seems like a bug. - return new KeywordAnalyzer(); - } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ea647088/docs/data-model.html ---------------------------------------------------------------------- diff --git a/docs/data-model.html b/docs/data-model.html index f6c397c..8cf3a35 100644 --- a/docs/data-model.html +++ b/docs/data-model.html @@ -92,6 +92,19 @@ </li> </ul> </li> + <li> + <a href="#custom_types">Custom Types</a> + <ul class="nav"> + <li><a href="#custom_types_creating">Creating</a></li> + <li><a href="#custom_types_distributing">Distributing</a></li> + <li><a href="#custom_types_using">Using</a> + <ul class="nav"> + <li><a href="#custom_types_using_cluster"> Cluster Wide</a></li> + <li><a href="#custom_types_using_table"> Single Table</a></li> + </ul> + </li> + </ul> + </li> </ul> </div> </div> @@ -470,6 +483,131 @@ To run a query to find all the rows that contain a location within 10 miles of g </p> </div> </section> + <section> + <div class="page-header"> + <h1 id="custom_types">Custom Types</h1> + </div> + <p class="lead">Custom types in Blur allow you to create your own types in Lucene + as well as plugging into the query parser so that you can use your custom type.</p> + <p>text here.</p> + <h3 id="custom_types_creating">Creating</h3> + <p> +You will need to extend the "org.apache.blur.analysis.FieldTypeDefinition" class found in the blur-query module. If you need to use a different Analyzer than the StandardAnalyzer used in the "text" type just extend the "org.apache.blur.analysis.type.TextFieldTypeDefinition" and make the appropriate changes. + </p> +<p> +For types that require custom query parsing or custom "org.apache.lucene.index.IndexableField" manipulation without the use of an Analyzer. Please extend "org.apache.blur.analysis.type.CustomFieldTypeDefinition". +</p> +<p> +<h4>Example</h4> +Below is a simple type that is basically the same as a "string" type, however it's implemented by extending "org.apache.blur.analysis.type.CustomFieldTypeDefinition".<br/><br/> +<pre><code class="java">public class ExampleType extends CustomFieldTypeDefinition { + + private String _fieldNameForThisInstance; + + /** + * Get the name of the type. + * + * @return the name. + */ + @Override + public String getName() { + return "example"; + } + + /** + * Configures this instance for the type. + * + * @param fieldNameForThisInstance + * the field name for this instance. + * @param properties + * the properties passed into this type definition from the + * {@link Blur.Iface#addColumnDefinition(String, ColumnDefinition)} + * method. + */ + @Override + public void configure(String fieldNameForThisInstance, Map<String, String> properties, + Configuration configuration) { + _fieldNameForThisInstance = fieldNameForThisInstance; + } + + /** + * Create {@link Field}s for the index as well as for storing the original + * data for retrieval. + * + * @param family + * the family name. + * @param column + * the column that holds the name and value. + * + * @return the {@link Iterable} of {@link Field}s. + */ + @Override + public Iterable<? extends Field> getFieldsForColumn(String family, Column column) { + String name = family + "." + column.getName(); + String value = column.getValue(); + return makeIterable(new StringField(name, value, Store.YES)); + } + + /** + * Create {@link Field}s for the index do NOT store the data because the is a + * sub column. + * + * @param family + * the family name. + * @param column + * the column that holds the name and value. + * @param subName + * the sub column name. + * + * @return the {@link Iterable} of {@link Field}s. + */ + @Override + public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column, + String subName) { + String name = family + "." + column.getName() + "." + subName; + String value = column.getValue(); + return makeIterable(new StringField(name, value, Store.NO)); + } + + /** + * Gets the query from the text provided by the query parser. + * + * @param text + * the text provided by the query parser. + * @return the {@link Query}. + */ + @Override + public Query getCustomQuery(String text) { + return new TermQuery(new Term(_fieldNameForThisInstance, text)); + } + +} +</code></pre> + +</p> + + <h3 id="custom_types_distributing">Distributing</h3> + <p>Once you have created and tested your custom type you will need to copy the jar file containing your custom type to all the servers in the cluster. The jar file will need to be located within the $BLUR_HOME/lib directory. Once there all the servers will need to be restarted to have the jar file be picked up in the classpath.<br/><br/>In a later version of Blur we hope to have this be a dynamic operation that can be performed without restarting the cluster.</p> + <h3 id="custom_types_using">Using</h3> + <p>You can either add your custom type to the entire cluster or per table.</p> + <h4 id="custom_types_using_cluster">Cluster Wide</h4> + <p>For cluster wide configuration you will need to add the new field types into the blur-site.properties file on each server. +<pre><code class="bash">blur.fieldtype.customfield1=org.apache.blur.analysis.type.ExampleType1 +blur.fieldtype.customfield2=org.apache.blur.analysis.type.ExampleType2 +...</code></pre> + +Please note that the prefix of "blur.fieldtype." is all that is used from the property name because the type gets it's name from the internal method of "getName". However the property names will need to be unique within the file. +</p> + <h4 id="custom_types_using_table">Single Table</h4> + <p>For a single table configuration you will need to add the new field types into the tableProperties map in the TableDescriptor as you define the table. +<pre><code class="java">tableDescriptor.putToTableProperties("blur.fieldtype.customfield1", + "org.apache.blur.analysis.type.ExampleType1"); +tableDescriptor.putToTableProperties("blur.fieldtype.customfield2", + "org.apache.blur.analysis.type.ExampleType2"); +...</code></pre> + +Please note that the prefix of "blur.fieldtype." is all that is used from the property name because the type gets it's name from the internal method of "getName". However the property names will need to be unique within the map.</p> + </section> </div> </div> </div>
