Another commit to the type system rewrite.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/a7a296b2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/a7a296b2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/a7a296b2 Branch: refs/heads/master Commit: a7a296b2f739fd0b1df15c153a975f5c4599c580 Parents: 8902b5f Author: Aaron McCurry <[email protected]> Authored: Sat Jun 29 13:26:34 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Sat Jun 29 13:26:34 2013 -0400 ---------------------------------------------------------------------- .../analysis/AnalyzerNotFoundException.java | 11 ++ .../blur/analysis/FieldTypeDefinition.java | 50 +++++++- .../apache/blur/analysis/LightBlurAnalyzer.java | 30 +++++ .../blur/analysis/MemoryFieldManager.java | 114 +++++++++++++++++++ .../analysis/type/TextFieldTypeDefinition.java | 73 ++++++++++++ 5 files changed, 277 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a7a296b2/blur-query/src/main/java/org/apache/blur/analysis/AnalyzerNotFoundException.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/AnalyzerNotFoundException.java b/blur-query/src/main/java/org/apache/blur/analysis/AnalyzerNotFoundException.java new file mode 100644 index 0000000..7d34981 --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/analysis/AnalyzerNotFoundException.java @@ -0,0 +1,11 @@ +package org.apache.blur.analysis; + +public class AnalyzerNotFoundException extends RuntimeException { + + private static final long serialVersionUID = -1669681684004161773L; + + public AnalyzerNotFoundException(String fieldName) { + super("FieldName [" + fieldName + "] not found in the analyzer"); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a7a296b2/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java index cc19cfe..7aecc95 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java @@ -16,6 +16,7 @@ package org.apache.blur.analysis; * See the License for the specific language governing permissions and * limitations under the License. */ +import java.util.Iterator; import java.util.Map; import org.apache.blur.thrift.generated.Column; @@ -47,7 +48,18 @@ public abstract class FieldTypeDefinition { * the {@link Column} * @return the {@link Iterable} of fields. */ - public abstract Iterable<? extends Field> getFields(Column column); + public abstract Iterable<? extends Field> getFieldsForColumn(String family, Column column); + + /** + * Gets the {@link Field}s for indexing from a single Column, but should not + * be stored because the original value should be stored in the main + * {@link Column}. + * + * @param column + * the {@link Column} + * @return the {@link Iterable} of fields. + */ + public abstract Iterable<? extends Field> getFieldsForSubColumn(String family, Column column, String subName); /** * Gets the {@link FieldType} for stored version. This is the normal Column @@ -81,4 +93,40 @@ public abstract class FieldTypeDefinition { */ public abstract Analyzer getAnalyzerForQuery(); + protected Iterable<? extends Field> makeIterable(final Field field) { + return new Iterable<Field>() { + @Override + public Iterator<Field> iterator() { + return new Iterator<Field>() { + + private boolean _hasNext = true; + + @Override + public void remove() { + + } + + @Override + public Field next() { + _hasNext = false; + return field; + } + + @Override + public boolean hasNext() { + return _hasNext; + } + }; + } + }; + } + + protected String getName(String family, String name) { + return family + "." + name; + } + + protected String getName(String family, String name, String subName) { + return family + "." + name + "." + subName; + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a7a296b2/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java b/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java new file mode 100644 index 0000000..a9c2c21 --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java @@ -0,0 +1,30 @@ +package org.apache.blur.analysis; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.AnalyzerWrapper; + +public class LightBlurAnalyzer extends AnalyzerWrapper { + + private final FieldManager _fieldManager; + private final boolean _usedForIndexing; + + public LightBlurAnalyzer(boolean usedForIndexing, FieldManager fieldManager) { + _usedForIndexing = usedForIndexing; + _fieldManager = fieldManager; + } + + @Override + protected Analyzer getWrappedAnalyzer(String fieldName) { + if (_usedForIndexing) { + return _fieldManager.getAnalyzerForIndex(fieldName); + } else { + return _fieldManager.getAnalyzerForQuery(fieldName); + } + } + + @Override + protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) { + return components; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a7a296b2/blur-query/src/main/java/org/apache/blur/analysis/MemoryFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/MemoryFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/MemoryFieldManager.java new file mode 100644 index 0000000..895b748 --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/analysis/MemoryFieldManager.java @@ -0,0 +1,114 @@ +package org.apache.blur.analysis; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; +import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.Record; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Field; + +public class MemoryFieldManager extends FieldManager { + + private static final Log LOG = LogFactory.getLog(MemoryFieldManager.class); + + private Map<String, FieldTypeDefinition> _fieldNameToDefMap = new ConcurrentHashMap<String, FieldTypeDefinition>(); + private Map<String, Class<? extends FieldTypeDefinition>> _typeMap = new ConcurrentHashMap<String, Class<? extends FieldTypeDefinition>>(); + + @Override + public Iterable<? extends Field> getFields(Record record) { + List<Field> fields = new ArrayList<Field>(); + String family = record.getFamily(); + List<Column> columns = record.getColumns(); + addDefaultFields(fields,record); + for (Column column : columns) { + getAndAddFields(fields, family, column, getFieldTypeDefinition(column)); + for (String subName : getSubColumns(column)) { + getAndAddFields(fields, family, column, getFieldTypeDefinition(column, subName)); + } + } + return fields; + } + + private void addDefaultFields(List<Field> fields, Record record) { + //add family + //record id fields + + } + + private void getAndAddFields(List<Field> fields, String family, Column column, FieldTypeDefinition fieldTypeDefinition) { + for (Field field : fieldTypeDefinition.getFieldsForColumn(family, column)) { + fields.add(field); + } + } + + private FieldTypeDefinition getFieldTypeDefinition(Column column, String subName) { + // TODO Auto-generated method stub + return null; + } + + private List<String> getSubColumns(Column column) { + // TODO Auto-generated method stub + return null; + } + + private FieldTypeDefinition getFieldTypeDefinition(Column column) { + // TODO Auto-generated method stub + return null; + } + + @Override + public void addColumnDefinition(String family, String columnName, String subColumnName, String fieldType, + Map<String, String> props) { + String baseFieldName = family + "." + columnName; + String fieldName; + if (subColumnName != null) { + if (!_fieldNameToDefMap.containsKey(baseFieldName)) { + throw new IllegalArgumentException("Base column of [" + baseFieldName + + "] not found, please add base before adding sub column."); + } + fieldName = baseFieldName + "." + subColumnName; + } else { + fieldName = baseFieldName; + } + Class<? extends FieldTypeDefinition> clazz = _typeMap.get(fieldType); + if (clazz == null) { + throw new IllegalArgumentException("FieldType of [" + fieldType + "] was not found."); + } + FieldTypeDefinition fieldTypeDefinition; + try { + fieldTypeDefinition = clazz.newInstance(); + } catch (InstantiationException e) { + LOG.error("Unknown error trying to create a type of [{0}] from class [{1}]", e, fieldType, clazz); + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + LOG.error("Unknown error trying to create a type of [{0}] from class [{1}]", e, fieldType, clazz); + throw new RuntimeException(e); + } + fieldTypeDefinition.configure(props); + _fieldNameToDefMap.put(fieldName, fieldTypeDefinition); + } + + @Override + public Analyzer getAnalyzerForIndex(String fieldName) { + FieldTypeDefinition fieldTypeDefinition = _fieldNameToDefMap.get(fieldName); + if (fieldTypeDefinition == null) { + throw new AnalyzerNotFoundException(fieldName); + } + return fieldTypeDefinition.getAnalyzerForIndex(); + } + + @Override + public Analyzer getAnalyzerForQuery(String fieldName) { + FieldTypeDefinition fieldTypeDefinition = _fieldNameToDefMap.get(fieldName); + if (fieldTypeDefinition == null) { + throw new AnalyzerNotFoundException(fieldName); + } + return fieldTypeDefinition.getAnalyzerForQuery(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a7a296b2/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java new file mode 100644 index 0000000..7e141fe --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/TextFieldTypeDefinition.java @@ -0,0 +1,73 @@ +package org.apache.blur.analysis.type; + +import java.util.Map; + +import org.apache.blur.analysis.FieldTypeDefinition; +import org.apache.blur.analysis.NoStopWordStandardAnalyzer; +import org.apache.blur.thrift.generated.Column; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.TextField; + +public class TextFieldTypeDefinition extends FieldTypeDefinition { + + private static final String TEXT = "text"; + private static final FieldType TYPE_NOT_STORED; + private static final FieldType TYPE_STORED; + + static { + TYPE_STORED = new FieldType(TextField.TYPE_STORED); + TYPE_STORED.setOmitNorms(true); + TYPE_STORED.freeze(); + + TYPE_NOT_STORED = new FieldType(TextField.TYPE_NOT_STORED); + TYPE_NOT_STORED.setOmitNorms(true); + TYPE_NOT_STORED.freeze(); + } + + @Override + public String getName() { + return TEXT; + } + + @Override + public void configure(Map<String, String> properties) { + + } + + @Override + public Iterable<? extends Field> getFieldsForColumn(String family, Column column) { + String name = getName(family, column.getName()); + Field field = new Field(name, column.getValue(), getStoredFieldType()); + return makeIterable(field); + } + + @Override + public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column, String subName) { + String name = getName(family, column.getName(), subName); + Field field = new Field(name, column.getValue(), getNotStoredFieldType()); + return makeIterable(field); + } + + @Override + public FieldType getStoredFieldType() { + return TYPE_STORED; + } + + @Override + public FieldType getNotStoredFieldType() { + return TYPE_NOT_STORED; + } + + @Override + public Analyzer getAnalyzerForIndex() { + return new NoStopWordStandardAnalyzer(); + } + + @Override + public Analyzer getAnalyzerForQuery() { + return new NoStopWordStandardAnalyzer(); + } + +}
