Created a HDFS field manager that stores the meta data about the table in ZK.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/ae516a44 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/ae516a44 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/ae516a44 Branch: refs/heads/0.2.0-newtypesystem Commit: ae516a442767b31d2c7e29b07a78aa08ec246dcf Parents: ccf87ce Author: Aaron McCurry <[email protected]> Authored: Tue Jul 16 14:09:10 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Jul 16 14:09:10 2013 -0400 ---------------------------------------------------------------------- .../manager/writer/TransactionRecorder.java | 2 +- .../apache/blur/analysis/BaseFieldManager.java | 113 +++++++---- .../org/apache/blur/analysis/FieldManager.java | 36 ++-- .../apache/blur/analysis/HdfsFieldManager.java | 65 ++++--- .../apache/blur/analysis/LightBlurAnalyzer.java | 45 ----- .../blur/lucene/search/BlurQueryParser.java | 45 +++-- .../blur/analysis/BaseFieldManagerTest.java | 193 +++++++++++++++++++ .../blur/analysis/BaseFiledManagerTest.java | 187 ------------------ .../blur/analysis/HdfsFieldManagerTest.java | 110 +++++++++++ .../blur/lucene/search/SuperParserTest.java | 12 +- 10 files changed, 491 insertions(+), 317 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java index ab9bfc6..994c687 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java @@ -356,7 +356,7 @@ public class TransactionRecorder extends TimerTask implements Closeable { } } - public static List<List<Field>> getDocs(Row row, FieldManager fieldManager) { + public static List<List<Field>> getDocs(Row row, FieldManager fieldManager) throws IOException { List<Record> records = row.records; int size = records.size(); final String rowId = row.id; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java index b2f005d..0404f2c 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java @@ -16,6 +16,7 @@ package org.apache.blur.analysis; * See the License for the specific language governing permissions and * limitations under the License. */ +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -24,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import org.apache.blur.analysis.type.DoubleFieldTypeDefinition; import org.apache.blur.analysis.type.FloatFieldTypeDefinition; @@ -50,8 +52,8 @@ public abstract class BaseFieldManager extends FieldManager { private static final Log LOG = LogFactory.getLog(BaseFieldManager.class); private static final Map<String, String> EMPTY_MAP = new HashMap<String, String>();; - private final Map<String, Set<String>> _columnToSubColumn = new ConcurrentHashMap<String, Set<String>>(); - private final Map<String, FieldTypeDefinition> _fieldNameToDefMap = new ConcurrentHashMap<String, FieldTypeDefinition>(); + private final ConcurrentMap<String, Set<String>> _columnToSubColumn = new ConcurrentHashMap<String, Set<String>>(); + private final ConcurrentMap<String, FieldTypeDefinition> _fieldNameToDefMap = new ConcurrentHashMap<String, FieldTypeDefinition>(); // This is loaded at object creation and never changed again. private final Map<String, Class<? extends FieldTypeDefinition>> _typeMap = new ConcurrentHashMap<String, Class<? extends FieldTypeDefinition>>(); @@ -87,7 +89,12 @@ public abstract class BaseFieldManager extends FieldManager { _baseAnalyzerForQuery = new AnalyzerWrapper() { @Override protected Analyzer getWrappedAnalyzer(String fieldName) { - FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName); + FieldTypeDefinition fieldTypeDefinition; + try { + fieldTypeDefinition = getFieldTypeDefinition(fieldName); + } catch (IOException e) { + throw new RuntimeException(e); + } if (fieldTypeDefinition == null) { return defaultAnalyzerForQuerying; } @@ -103,7 +110,12 @@ public abstract class BaseFieldManager extends FieldManager { _baseAnalyzerForIndex = new AnalyzerWrapper() { @Override protected Analyzer getWrappedAnalyzer(String fieldName) { - FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName); + FieldTypeDefinition fieldTypeDefinition; + try { + fieldTypeDefinition = getFieldTypeDefinition(fieldName); + } catch (IOException e) { + throw new RuntimeException(e); + } if (fieldTypeDefinition == null) { throw new RuntimeException("Field [" + fieldName + "] not found."); } @@ -118,7 +130,7 @@ public abstract class BaseFieldManager extends FieldManager { } @Override - public List<Field> getFields(String rowId, Record record) { + public List<Field> getFields(String rowId, Record record) throws IOException { List<Field> fields = new ArrayList<Field>(); String family = record.getFamily(); List<Column> columns = record.getColumns(); @@ -180,7 +192,7 @@ public abstract class BaseFieldManager extends FieldManager { fields.add(new Field(_fieldLessField, value, SUPER_FIELD_TYPE)); } - private FieldTypeDefinition getFieldTypeDefinition(String family, Column column, String subName) { + private FieldTypeDefinition getFieldTypeDefinition(String family, Column column, String subName) throws IOException { return getFieldTypeDefinition(getSubColumnName(family, column, subName)); } @@ -200,13 +212,13 @@ public abstract class BaseFieldManager extends FieldManager { return _columnToSubColumn.get(getColumnName(family, column)); } - private FieldTypeDefinition getFieldTypeDefinition(String family, Column column) { + private FieldTypeDefinition getFieldTypeDefinition(String family, Column column) throws IOException { return getFieldTypeDefinition(getColumnName(family, column)); } @Override public boolean addColumnDefinition(String family, String columnName, String subColumnName, boolean fieldLessIndexing, - String fieldType, Map<String, String> props) { + String fieldType, Map<String, String> props) throws IOException { String baseFieldName = family + "." + columnName; String fieldName; if (subColumnName != null) { @@ -230,10 +242,46 @@ public abstract class BaseFieldManager extends FieldManager { if (!tryToStore(fieldName, fieldLessIndexing, fieldType, props)) { return false; } + fieldTypeDefinition = newFieldTypeDefinition(fieldLessIndexing, fieldType, props); + registerFieldTypeDefinition(fieldName, fieldTypeDefinition); + return true; + } + + protected void registerFieldTypeDefinition(String fieldName, FieldTypeDefinition fieldTypeDefinition) { + _fieldNameToDefMap.put(fieldName, fieldTypeDefinition); + String baseFieldName = getBaseFieldName(fieldName); + String subColumnName = getSubColumnName(fieldName); + if (subColumnName != null) { + Set<String> subColumnNames = _columnToSubColumn.get(baseFieldName); + if (subColumnNames == null) { + subColumnNames = getConcurrentSet(); + _columnToSubColumn.put(baseFieldName, subColumnNames); + } + subColumnNames.add(subColumnName); + } + } + + private String getSubColumnName(String fieldName) { + int lastIndexOf = fieldName.lastIndexOf('.'); + int indexOf = fieldName.indexOf('.'); + if (indexOf == lastIndexOf) { + return null; + } + return fieldName.substring(lastIndexOf + 1); + } + + private String getBaseFieldName(String fieldName) { + int indexOf = fieldName.indexOf('.'); + return fieldName.substring(0, indexOf); + } + + protected FieldTypeDefinition newFieldTypeDefinition(boolean fieldLessIndexing, String fieldType, + Map<String, String> props) { Class<? extends FieldTypeDefinition> clazz = _typeMap.get(fieldType); if (clazz == null) { throw new IllegalArgumentException("FieldType of [" + fieldType + "] was not found."); } + FieldTypeDefinition fieldTypeDefinition; try { fieldTypeDefinition = clazz.newInstance(); } catch (InstantiationException e) { @@ -249,26 +297,20 @@ public abstract class BaseFieldManager extends FieldManager { fieldTypeDefinition.configure(props); } fieldTypeDefinition.setFieldLessIndexing(fieldLessIndexing); - _fieldNameToDefMap.put(fieldName, fieldTypeDefinition); - if (subColumnName != null) { - Set<String> subColumnNames = _columnToSubColumn.put(baseFieldName, getConcurrentSet()); - subColumnNames.add(subColumnName); - } - return true; + return fieldTypeDefinition; } protected abstract boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, - Map<String, String> props); + Map<String, String> props) throws IOException; - protected abstract void tryToLoad(String field, Map<String, FieldTypeDefinition> fieldNameToDefMap, - Map<String, Set<String>> columnToSubColumn); + protected abstract void tryToLoad(String fieldName) throws IOException; private Set<String> getConcurrentSet() { return Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); } @Override - public boolean isValidColumnDefinition(String family, String columnName) { + public boolean isValidColumnDefinition(String family, String columnName) throws IOException { String fieldName = getColumnName(family, columnName); FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName); if (fieldTypeDefinition == null) { @@ -278,7 +320,7 @@ public abstract class BaseFieldManager extends FieldManager { } @Override - public Analyzer getAnalyzerForIndex(String fieldName) { + public Analyzer getAnalyzerForIndex(String fieldName) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName); if (fieldTypeDefinition == null) { throw new AnalyzerNotFoundException(fieldName); @@ -287,7 +329,7 @@ public abstract class BaseFieldManager extends FieldManager { } @Override - public Analyzer getAnalyzerForQuery(String fieldName) { + public Analyzer getAnalyzerForQuery(String fieldName) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName); if (fieldTypeDefinition == null) { throw new AnalyzerNotFoundException(fieldName); @@ -295,31 +337,31 @@ public abstract class BaseFieldManager extends FieldManager { return fieldTypeDefinition.getAnalyzerForQuery(); } - public void addColumnDefinitionInt(String family, String columnName) { + public void addColumnDefinitionInt(String family, String columnName) throws IOException { addColumnDefinition(family, columnName, null, false, IntFieldTypeDefinition.NAME, null); } - public void addColumnDefinitionLong(String family, String columnName) { + public void addColumnDefinitionLong(String family, String columnName) throws IOException { addColumnDefinition(family, columnName, null, false, LongFieldTypeDefinition.NAME, null); } - public void addColumnDefinitionFloat(String family, String columnName) { + public void addColumnDefinitionFloat(String family, String columnName) throws IOException { addColumnDefinition(family, columnName, null, false, FloatFieldTypeDefinition.NAME, null); } - public void addColumnDefinitionDouble(String family, String columnName) { + public void addColumnDefinitionDouble(String family, String columnName) throws IOException { addColumnDefinition(family, columnName, null, false, DoubleFieldTypeDefinition.NAME, null); } - public void addColumnDefinitionString(String family, String columnName) { + public void addColumnDefinitionString(String family, String columnName) throws IOException { addColumnDefinition(family, columnName, null, false, StringFieldTypeDefinition.NAME, null); } - public void addColumnDefinitionText(String family, String columnName) { + public void addColumnDefinitionText(String family, String columnName) throws IOException { addColumnDefinition(family, columnName, null, false, TextFieldTypeDefinition.NAME, null); } - public void addColumnDefinitionTextFieldLess(String family, String columnName) { + public void addColumnDefinitionTextFieldLess(String family, String columnName) throws IOException { addColumnDefinition(family, columnName, null, true, TextFieldTypeDefinition.NAME, null); } @@ -334,31 +376,32 @@ public abstract class BaseFieldManager extends FieldManager { } @Override - public boolean isFieldLessIndexed(String field) { + public boolean isFieldLessIndexed(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); return fieldTypeDefinition.isFieldLessIndexed(); } @Override - public boolean checkSupportForFuzzyQuery(String field) { + public boolean checkSupportForFuzzyQuery(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); return fieldTypeDefinition.checkSupportForFuzzyQuery(); } @Override - public boolean checkSupportForPrefixQuery(String field) { + public boolean checkSupportForPrefixQuery(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); return fieldTypeDefinition.checkSupportForPrefixQuery(); } @Override - public boolean checkSupportForWildcardQuery(String field) { + public boolean checkSupportForWildcardQuery(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); return fieldTypeDefinition.checkSupportForWildcardQuery(); } @Override - public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) { + public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) + throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); if (fieldTypeDefinition != null && fieldTypeDefinition.isNumeric()) { NumericFieldTypeDefinition numericFieldTypeDefinition = (NumericFieldTypeDefinition) fieldTypeDefinition; @@ -368,7 +411,7 @@ public abstract class BaseFieldManager extends FieldManager { } @Override - public Query getTermQueryIfNumeric(String field, String text) { + public Query getTermQueryIfNumeric(String field, String text) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); if (fieldTypeDefinition != null && fieldTypeDefinition.isNumeric()) { NumericFieldTypeDefinition numericFieldTypeDefinition = (NumericFieldTypeDefinition) fieldTypeDefinition; @@ -378,10 +421,10 @@ public abstract class BaseFieldManager extends FieldManager { } @Override - public FieldTypeDefinition getFieldTypeDefinition(String field) { + public FieldTypeDefinition getFieldTypeDefinition(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = _fieldNameToDefMap.get(field); if (fieldTypeDefinition == null) { - tryToLoad(field, _fieldNameToDefMap, _columnToSubColumn); + tryToLoad(field); fieldTypeDefinition = _fieldNameToDefMap.get(field); } return fieldTypeDefinition; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java index 2cb7aea..2271726 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java @@ -16,6 +16,7 @@ package org.apache.blur.analysis; * See the License for the specific language governing permissions and * limitations under the License. */ +import java.io.IOException; import java.util.List; import java.util.Map; @@ -32,8 +33,9 @@ public abstract class FieldManager { * * @param record * @return + * @throws IOException */ - public abstract List<Field> getFields(String rowId, Record record); + public abstract List<Field> getFields(String rowId, Record record) throws IOException; /** * Adds a column definition. @@ -53,9 +55,10 @@ public abstract class FieldManager { * @param props * the configuration properties for this column and type. * @return + * @throws IOException */ public abstract boolean addColumnDefinition(String family, String columnName, String subColumnName, - boolean fieldLessIndexing, String fieldType, Map<String, String> props); + boolean fieldLessIndexing, String fieldType, Map<String, String> props) throws IOException; /** * Gets the analyzer for the indexing process. @@ -63,8 +66,9 @@ public abstract class FieldManager { * @param fieldName * the Lucene field name. * @return {@link Analyzer}. + * @throws IOException */ - public abstract Analyzer getAnalyzerForIndex(String fieldName); + public abstract Analyzer getAnalyzerForIndex(String fieldName) throws IOException; /** * Gets the analyzer for the querying. @@ -72,8 +76,9 @@ public abstract class FieldManager { * @param fieldName * the Lucene field name. * @return {@link Analyzer}. + * @throws IOException */ - public abstract Analyzer getAnalyzerForQuery(String fieldName); + public abstract Analyzer getAnalyzerForQuery(String fieldName) throws IOException; /** * Checks if there is a valid column definition for the given family and @@ -84,8 +89,9 @@ public abstract class FieldManager { * @param columnName * the column name. * @return boolean + * @throws IOException */ - public abstract boolean isValidColumnDefinition(String family, String columnName); + public abstract boolean isValidColumnDefinition(String family, String columnName) throws IOException; /** * Gets an {@link Analyzer} for quering. @@ -100,8 +106,9 @@ public abstract class FieldManager { * @param field * the field name. * @return boolean + * @throws IOException */ - public abstract boolean checkSupportForFuzzyQuery(String field); + public abstract boolean checkSupportForFuzzyQuery(String field) throws IOException; /** * Checks if this field supports prefix queries. @@ -109,8 +116,9 @@ public abstract class FieldManager { * @param field * the field name. * @return boolean + * @throws IOException */ - public abstract boolean checkSupportForPrefixQuery(String field); + public abstract boolean checkSupportForPrefixQuery(String field) throws IOException; /** * Checks if this field supports wildcard queries. @@ -118,8 +126,9 @@ public abstract class FieldManager { * @param field * the field name. * @return boolean + * @throws IOException */ - public abstract boolean checkSupportForWildcardQuery(String field); + public abstract boolean checkSupportForWildcardQuery(String field) throws IOException; /** * Gets a range query, if the method returns null the default term range query @@ -136,9 +145,10 @@ public abstract class FieldManager { * @param endInclusive * if the end is inclusive. * @return the new range query or null. + * @throws IOException */ public abstract Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, - boolean endInclusive); + boolean endInclusive) throws IOException; /** * This method should return a query (probably a range query) for numeric @@ -149,8 +159,9 @@ public abstract class FieldManager { * @param text * the text for the term. * @return the query or null. + * @throws IOException */ - public abstract Query getTermQueryIfNumeric(String field, String text); + public abstract Query getTermQueryIfNumeric(String field, String text) throws IOException; /** * Gets the {@link FieldTypeDefinition} for the given field. @@ -158,10 +169,11 @@ public abstract class FieldManager { * @param field * the field name. * @return the {@link FieldTypeDefinition} or null if missing. + * @throws IOException */ - public abstract FieldTypeDefinition getFieldTypeDefinition(String field); + public abstract FieldTypeDefinition getFieldTypeDefinition(String field) throws IOException; - public abstract boolean isFieldLessIndexed(String name); + public abstract boolean isFieldLessIndexed(String name) throws IOException; public abstract Analyzer getAnalyzerForIndex(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java index 3d5d5c2..9669e2e 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java @@ -17,14 +17,15 @@ package org.apache.blur.analysis; * limitations under the License. */ import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; -import java.util.Set; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -32,6 +33,9 @@ import org.apache.lucene.analysis.Analyzer; public class HdfsFieldManager extends BaseFieldManager { + private static final String FIELD_TYPE = "fieldType"; + private static final String FIELD_LESS_INDEXING = "fieldLessIndexing"; + private static final Log LOG = LogFactory.getLog(HdfsFieldManager.class); private final Configuration _configuration; @@ -47,43 +51,58 @@ public class HdfsFieldManager extends BaseFieldManager { } @Override - protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props) { + protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props) + throws IOException { + LOG.info("Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]", + fieldName, fieldLessIndexing, fieldType, props); Properties properties = new Properties(); - properties.put("fieldLessIndexing", fieldLessIndexing); - properties.put("fieldType", fieldType); + properties.setProperty(FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexing)); + properties.setProperty(FIELD_TYPE, fieldType); if (props != null) { for (Entry<String, String> e : props.entrySet()) { properties.put(e.getKey(), e.getValue()); } } - Path path = new Path(_storagePath, fieldName); - try { - if (_fileSystem.exists(path)) { - return false; - } - } catch (IOException e) { - LOG.error("Could not check filesystem for existence of path [{0}]", e, path); - return false; - } - try { - FSDataOutputStream outputStream = _fileSystem.create(path, false); - properties.store(outputStream, getComments()); - outputStream.close(); - } catch (IOException e) { - LOG.error("Could not create field [" + fieldName + "]", e); + Path path = getFieldPath(fieldName); + if (_fileSystem.exists(path)) { + LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props); return false; } + FSDataOutputStream outputStream = _fileSystem.create(path, false); + properties.store(outputStream, getComments()); + outputStream.close(); return true; } + private Path getFieldPath(String fieldName) { + return new Path(_storagePath, fieldName); + } + private String getComments() { - return null; + return "This file is generated from Apache Blur to store meta data about field types. DO NOT MODIFIY!"; } @Override - protected void tryToLoad(String field, Map<String, FieldTypeDefinition> fieldNameToDefMap, - Map<String, Set<String>> columnToSubColumn) { - + protected void tryToLoad(String fieldName) throws IOException { + Path path = getFieldPath(fieldName); + if (!_fileSystem.exists(path)) { + return; + } + FSDataInputStream inputStream = _fileSystem.open(path); + Properties props = new Properties(); + props.load(inputStream); + inputStream.close(); + boolean fieldLessIndexing = Boolean.parseBoolean(props.getProperty(FIELD_LESS_INDEXING)); + String fieldType = props.getProperty(FIELD_TYPE); + FieldTypeDefinition fieldTypeDefinition = newFieldTypeDefinition(fieldLessIndexing, fieldType, toMap(props)); + registerFieldTypeDefinition(fieldName, fieldTypeDefinition); } + private Map<String, String> toMap(Properties props) { + Map<String, String> result = new HashMap<String, String>(); + for (Entry<Object, Object> e : props.entrySet()) { + result.put(e.getKey().toString(), e.getValue().toString()); + } + return result; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java b/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java deleted file mode 100644 index 1939119..0000000 --- a/blur-query/src/main/java/org/apache/blur/analysis/LightBlurAnalyzer.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.apache.blur.analysis; -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.AnalyzerWrapper; - -public class LightBlurAnalyzer extends AnalyzerWrapper { - - private final FieldManager _fieldManager; - private final boolean _usedForIndexing; - - public LightBlurAnalyzer(boolean usedForIndexing, FieldManager fieldManager) { - _usedForIndexing = usedForIndexing; - _fieldManager = fieldManager; - } - - @Override - protected Analyzer getWrappedAnalyzer(String fieldName) { - if (_usedForIndexing) { - return _fieldManager.getAnalyzerForIndex(fieldName); - } else { - return _fieldManager.getAnalyzerForQuery(fieldName); - } - } - - @Override - protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) { - return components; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java b/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java index 26762a6..169ada4 100644 --- a/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java +++ b/blur-query/src/main/java/org/apache/blur/lucene/search/BlurQueryParser.java @@ -16,6 +16,7 @@ package org.apache.blur.lucene.search; * See the License for the specific language governing permissions and * limitations under the License. */ +import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.UUID; @@ -45,9 +46,13 @@ public class BlurQueryParser extends QueryParser { @Override protected Query newFuzzyQuery(Term term, float minimumSimilarity, int prefixLength) { String field = term.field(); - if (!_fieldManager.checkSupportForFuzzyQuery(field)) { - throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field) - + "] which does not support fuzzy queries."); + try { + if (!_fieldManager.checkSupportForFuzzyQuery(field)) { + throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field) + + "] which does not support fuzzy queries."); + } + } catch (IOException e) { + throw new RuntimeException(e); } return addField(super.newFuzzyQuery(term, minimumSimilarity, prefixLength), term.field()); } @@ -85,16 +90,25 @@ public class BlurQueryParser extends QueryParser { @Override protected Query newPrefixQuery(Term prefix) { String field = prefix.field(); - if (!_fieldManager.checkSupportForPrefixQuery(field)) { - throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field) - + "] which does not support prefix queries."); + try { + if (!_fieldManager.checkSupportForPrefixQuery(field)) { + throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field) + + "] which does not support prefix queries."); + } + } catch (IOException e) { + throw new RuntimeException(e); } return addField(super.newPrefixQuery(prefix), field); } @Override protected Query newRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) { - Query q = _fieldManager.getNewRangeQuery(field, part1, part2, startInclusive, endInclusive); + Query q; + try { + q = _fieldManager.getNewRangeQuery(field, part1, part2, startInclusive, endInclusive); + } catch (IOException e) { + throw new RuntimeException(e); + } if (q != null) { return addField(q, field); } @@ -104,7 +118,12 @@ public class BlurQueryParser extends QueryParser { @Override protected Query newTermQuery(Term term) { String field = term.field(); - Query q = _fieldManager.getTermQueryIfNumeric(field, term.text()); + Query q; + try { + q = _fieldManager.getTermQueryIfNumeric(field, term.text()); + } catch (IOException e) { + throw new RuntimeException(e); + } if (q != null) { return addField(q, field); } @@ -117,9 +136,13 @@ public class BlurQueryParser extends QueryParser { return new MatchAllDocsQuery(); } String field = t.field(); - if (!_fieldManager.checkSupportForWildcardQuery(field)) { - throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field) - + "] which does not support wildcard queries."); + try { + if (!_fieldManager.checkSupportForWildcardQuery(field)) { + throw new RuntimeException("Field [" + field + "] is type [" + _fieldManager.getFieldTypeDefinition(field) + + "] which does not support wildcard queries."); + } + } catch (IOException e) { + throw new RuntimeException(e); } return addField(super.newWildcardQuery(t), t.field()); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java new file mode 100644 index 0000000..cc44e84 --- /dev/null +++ b/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java @@ -0,0 +1,193 @@ +package org.apache.blur.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.blur.analysis.type.TextFieldTypeDefinition; +import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.Record; +import org.apache.blur.utils.BlurConstants; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Field; +import org.junit.Test; + +public class BaseFieldManagerTest { + + public static final String _fieldLessField = BlurConstants.SUPER; + + @Test + public void testFiledManager() throws IOException { + BaseFieldManager memoryFieldManager = newFieldManager(true); + memoryFieldManager.addColumnDefinition("fam1", "col1", null, true, "text", null); + + Record record = new Record(); + record.setFamily("fam1"); + record.setRecordId("1213"); + record.addToColumns(new Column("col1", "value1")); + + List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"), + newTextFieldNoStore(_fieldLessField, "value1")); + + int c = 0; + for (Field field : memoryFieldManager.getFields("1", record)) { + assertFieldEquals(fields.get(c++), field); + } + } + + @Test + public void testFiledManagerMultipleColumnsSameName() throws IOException { + BaseFieldManager memoryFieldManager = newFieldManager(true); + memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null); + + Record record = new Record(); + record.setFamily("fam1"); + record.setRecordId("1213"); + record.addToColumns(new Column("col1", "value1")); + record.addToColumns(new Column("col1", "value2")); + + List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"), + newTextField("fam1.col1", "value2")); + + int c = 0; + for (Field field : memoryFieldManager.getFields("1", record)) { + assertFieldEquals(fields.get(c++), field); + } + + } + + @Test + public void testFiledManagerMultipleColumnsDifferentNames() throws IOException { + BaseFieldManager memoryFieldManager = newFieldManager(true); + memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null); + memoryFieldManager.addColumnDefinition("fam1", "col2", null, true, "text", null); + + Record record = new Record(); + record.setFamily("fam1"); + record.setRecordId("1213"); + record.addToColumns(new Column("col1", "value1")); + record.addToColumns(new Column("col2", "value2")); + + List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"), + newTextField("fam1.col2", "value2"), newTextFieldNoStore(_fieldLessField, "value2")); + + int c = 0; + for (Field field : memoryFieldManager.getFields("1", record)) { + assertFieldEquals(fields.get(c++), field); + } + } + + @Test + public void testFiledManagerMultipleColumnsDifferentNamesDifferentFamilies() throws IOException { + BaseFieldManager memoryFieldManager = newFieldManager(true); + memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null); + memoryFieldManager.addColumnDefinition("fam2", "col2", null, false, "text", null); + + Record record1 = new Record(); + record1.setFamily("fam1"); + record1.setRecordId("1213"); + record1.addToColumns(new Column("col1", "value1")); + + List<Field> fields1 = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1")); + int c1 = 0; + for (Field field : memoryFieldManager.getFields("1", record1)) { + assertFieldEquals(fields1.get(c1++), field); + } + + Record record2 = new Record(); + record2.setFamily("fam2"); + record2.setRecordId("1213"); + record2.addToColumns(new Column("col2", "value1")); + + List<Field> fields2 = getFields("fam2", "1", "1213", newTextField("fam2.col2", "value1")); + int c2 = 0; + for (Field field : memoryFieldManager.getFields("1", record2)) { + assertFieldEquals(fields2.get(c2++), field); + } + } + + @Test + public void testFiledManagerSubNameWithMainColumnNameNoParent() throws IOException { + BaseFieldManager memoryFieldManager = newFieldManager(true); + try { + memoryFieldManager.addColumnDefinition("fam1", "col1", "sub1", false, "text", null); + fail("Should throw IllegalArgumentException"); + } catch (IllegalArgumentException e) { + } + } + + @Test + public void testFiledManagerSubNameWithMainColumnNameNoFieldLess() throws IOException { + BaseFieldManager memoryFieldManager = newFieldManager(true); + memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null); + try { + memoryFieldManager.addColumnDefinition("fam1", "col1", "sub1", true, "text", null); + fail("Should throw IllegalArgumentException"); + } catch (IllegalArgumentException e) { + } + } + + private List<Field> getFields(String family, String rowId, String recordId, Field... fields) { + List<Field> fieldLst = new ArrayList<Field>(); + fieldLst.add(new Field("family", family, BaseFieldManager.ID_TYPE)); + fieldLst.add(new Field("rowid", rowId, BaseFieldManager.ID_TYPE)); + fieldLst.add(new Field("recordid", recordId, BaseFieldManager.ID_TYPE)); + for (Field field : fields) { + fieldLst.add(field); + } + return fieldLst; + } + + private Field newTextField(String name, String value) { + return new Field(name, value, TextFieldTypeDefinition.TYPE_STORED); + } + + private Field newTextFieldNoStore(String name, String value) { + return new Field(name, value, TextFieldTypeDefinition.TYPE_NOT_STORED); + } + + private void assertFieldEquals(Field expected, Field actual) { + assertEquals("Names did not match Expected [" + expected + "] Actual [" + actual + "]", expected.name(), + actual.name()); + assertEquals("Values did not match Expected [" + expected + "] Actual [" + actual + "]", expected.stringValue(), + actual.stringValue()); + assertEquals("FileTypes did not match Expected [" + expected + "] Actual [" + actual + "]", expected.fieldType() + .toString(), actual.fieldType().toString()); + } + + protected BaseFieldManager newFieldManager(boolean create) throws IOException { + return new BaseFieldManager(_fieldLessField, new KeywordAnalyzer()) { + @Override + protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, + Map<String, String> props) { + return true; + } + + @Override + protected void tryToLoad(String field) { + + } + }; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/test/java/org/apache/blur/analysis/BaseFiledManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/BaseFiledManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/BaseFiledManagerTest.java deleted file mode 100644 index 19a7586..0000000 --- a/blur-query/src/test/java/org/apache/blur/analysis/BaseFiledManagerTest.java +++ /dev/null @@ -1,187 +0,0 @@ -package org.apache.blur.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.apache.blur.analysis.type.TextFieldTypeDefinition; -import org.apache.blur.thrift.generated.Column; -import org.apache.blur.thrift.generated.Record; -import org.apache.blur.utils.BlurConstants; -import org.apache.lucene.analysis.core.KeywordAnalyzer; -import org.apache.lucene.document.Field; -import org.junit.Test; - -public class BaseFiledManagerTest { - - private static final String _fieldLessField = BlurConstants.SUPER; - - @Test - public void testFiledManager() { - BaseFieldManager memoryFieldManager = newBaseFieldManager(); - memoryFieldManager.addColumnDefinition("fam1", "col1", null, true, "text", null); - - Record record = new Record(); - record.setFamily("fam1"); - record.setRecordId("1213"); - record.addToColumns(new Column("col1", "value1")); - - List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"), - newTextFieldNoStore(_fieldLessField, "value1")); - - int c = 0; - for (Field field : memoryFieldManager.getFields("1", record)) { - assertFieldEquals(fields.get(c++), field); - } - } - - @Test - public void testFiledManagerMultipleColumnsSameName() { - BaseFieldManager memoryFieldManager = newBaseFieldManager(); - memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null); - - Record record = new Record(); - record.setFamily("fam1"); - record.setRecordId("1213"); - record.addToColumns(new Column("col1", "value1")); - record.addToColumns(new Column("col1", "value2")); - - List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"), - newTextField("fam1.col1", "value2")); - - int c = 0; - for (Field field : memoryFieldManager.getFields("1", record)) { - assertFieldEquals(fields.get(c++), field); - } - - } - - @Test - public void testFiledManagerMultipleColumnsDifferentNames() { - BaseFieldManager memoryFieldManager = newBaseFieldManager(); - memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null); - memoryFieldManager.addColumnDefinition("fam1", "col2", null, true, "text", null); - - Record record = new Record(); - record.setFamily("fam1"); - record.setRecordId("1213"); - record.addToColumns(new Column("col1", "value1")); - record.addToColumns(new Column("col2", "value2")); - - List<Field> fields = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1"), - newTextField("fam1.col2", "value2"), newTextFieldNoStore(_fieldLessField, "value2")); - - int c = 0; - for (Field field : memoryFieldManager.getFields("1", record)) { - assertFieldEquals(fields.get(c++), field); - } - } - - @Test - public void testFiledManagerMultipleColumnsDifferentNamesDifferentFamilies() { - BaseFieldManager memoryFieldManager = newBaseFieldManager(); - memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null); - memoryFieldManager.addColumnDefinition("fam2", "col2", null, false, "text", null); - - Record record1 = new Record(); - record1.setFamily("fam1"); - record1.setRecordId("1213"); - record1.addToColumns(new Column("col1", "value1")); - - List<Field> fields1 = getFields("fam1", "1", "1213", newTextField("fam1.col1", "value1")); - int c1 = 0; - for (Field field : memoryFieldManager.getFields("1", record1)) { - assertFieldEquals(fields1.get(c1++), field); - } - - Record record2 = new Record(); - record2.setFamily("fam2"); - record2.setRecordId("1213"); - record2.addToColumns(new Column("col2", "value1")); - - List<Field> fields2 = getFields("fam2", "1", "1213", newTextField("fam2.col2", "value1")); - int c2 = 0; - for (Field field : memoryFieldManager.getFields("1", record2)) { - assertFieldEquals(fields2.get(c2++), field); - } - } - - @Test - public void testFiledManagerSubNameWithMainColumnNameNoParent() { - BaseFieldManager memoryFieldManager = newBaseFieldManager(); - try { - memoryFieldManager.addColumnDefinition("fam1", "col1", "sub1", false, "text", null); - fail("Should throw IllegalArgumentException"); - } catch (IllegalArgumentException e) { - } - } - - @Test - public void testFiledManagerSubNameWithMainColumnNameNoFieldLess() { - BaseFieldManager memoryFieldManager = newBaseFieldManager(); - memoryFieldManager.addColumnDefinition("fam1", "col1", null, false, "text", null); - try { - memoryFieldManager.addColumnDefinition("fam1", "col1", "sub1", true, "text", null); - fail("Should throw IllegalArgumentException"); - } catch (IllegalArgumentException e) { - } - } - - private List<Field> getFields(String family, String rowId, String recordId, Field... fields) { - List<Field> fieldLst = new ArrayList<Field>(); - fieldLst.add(new Field("family", family, BaseFieldManager.ID_TYPE)); - fieldLst.add(new Field("rowid", rowId, BaseFieldManager.ID_TYPE)); - fieldLst.add(new Field("recordid", recordId, BaseFieldManager.ID_TYPE)); - for (Field field : fields) { - fieldLst.add(field); - } - return fieldLst; - } - - private Field newTextField(String name, String value) { - return new Field(name, value, TextFieldTypeDefinition.TYPE_STORED); - } - - private Field newTextFieldNoStore(String name, String value) { - return new Field(name, value, TextFieldTypeDefinition.TYPE_NOT_STORED); - } - - private void assertFieldEquals(Field expected, Field actual) { - assertEquals("Names did not match Expected [" + expected + "] Actual [" + actual + "]", expected.name(), - actual.name()); - assertEquals("Values did not match Expected [" + expected + "] Actual [" + actual + "]", expected.stringValue(), - actual.stringValue()); - assertEquals("FileTypes did not match Expected [" + expected + "] Actual [" + actual + "]", expected.fieldType() - .toString(), actual.fieldType().toString()); - } - - private BaseFieldManager newBaseFieldManager() { - return new BaseFieldManager(_fieldLessField, new KeywordAnalyzer()) { - @Override - protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, - Map<String, String> props) { - return true; - } - }; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java new file mode 100644 index 0000000..ebf690a --- /dev/null +++ b/blur-query/src/test/java/org/apache/blur/analysis/HdfsFieldManagerTest.java @@ -0,0 +1,110 @@ +package org.apache.blur.analysis; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.junit.Test; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +public class HdfsFieldManagerTest extends BaseFieldManagerTest { + @Override + protected BaseFieldManager newFieldManager(boolean create) throws IOException { + Configuration config = new Configuration(); + Path path = new Path("./target/tmp/HdfsFieldManagerTest/meta"); + FileSystem fileSystem = path.getFileSystem(config); + if (create) { + fileSystem.delete(path, true); + } + return new HdfsFieldManager(_fieldLessField, new KeywordAnalyzer(), path, config); + } + + @SuppressWarnings("unchecked") + @Test + public void testStoreMetaData() throws IOException { + BaseFieldManager memoryFieldManager = newFieldManager(true); + memoryFieldManager.addColumnDefinition("fam1", "col1", null, true, "text", null); + memoryFieldManager.addColumnDefinition("fam2", "col2", null, false, "string", null); + memoryFieldManager.addColumnDefinition("fam2", "col2", "a", false, "text", null); + memoryFieldManager.addColumnDefinition("fam2", "col2", "b", false, "text", newMap(e("a", "b"))); + memoryFieldManager.addColumnDefinition("fam2", "col3", null, false, "int", null); + + BaseFieldManager memoryFieldManager2 = newFieldManager(false); + FieldTypeDefinition fieldTypeDefinition1 = memoryFieldManager2.getFieldTypeDefinition("fam1.col1"); + assertNotNull(fieldTypeDefinition1); + assertTrue(fieldTypeDefinition1.isFieldLessIndexed()); + assertFalse(fieldTypeDefinition1.isNumeric()); + + FieldTypeDefinition fieldTypeDefinition2 = memoryFieldManager2.getFieldTypeDefinition("fam2.col2"); + assertNotNull(fieldTypeDefinition2); + assertFalse(fieldTypeDefinition2.isFieldLessIndexed()); + assertFalse(fieldTypeDefinition2.isNumeric()); + + FieldTypeDefinition fieldTypeDefinition3 = memoryFieldManager2.getFieldTypeDefinition("fam2.col2.a"); + assertNotNull(fieldTypeDefinition3); + assertFalse(fieldTypeDefinition3.isFieldLessIndexed()); + assertFalse(fieldTypeDefinition3.isNumeric()); + + FieldTypeDefinition fieldTypeDefinition4 = memoryFieldManager2.getFieldTypeDefinition("fam2.col2.b"); + assertNotNull(fieldTypeDefinition4); + assertFalse(fieldTypeDefinition4.isFieldLessIndexed()); + assertFalse(fieldTypeDefinition4.isNumeric()); + + FieldTypeDefinition fieldTypeDefinition5 = memoryFieldManager2.getFieldTypeDefinition("fam2.col3"); + assertNotNull(fieldTypeDefinition5); + assertFalse(fieldTypeDefinition5.isFieldLessIndexed()); + assertTrue(fieldTypeDefinition5.isNumeric()); + + assertNull(memoryFieldManager2.getFieldTypeDefinition("fam2.col2.c")); + } + + private static <K, V> Map<K, V> newMap(Entry<K, V>... es) { + Map<K, V> map = new HashMap<K, V>(); + for (Entry<K, V> e : es) { + map.put(e.getKey(), e.getValue()); + } + return map; + } + + private static <K, V> Entry<K, V> e(final K key, final V value) { + return new Entry<K, V>() { + + @Override + public V setValue(V value) { + return null; + } + + @Override + public V getValue() { + return value; + } + + @Override + public K getKey() { + return key; + } + }; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ae516a44/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java index 89224f0..d183231 100644 --- a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java +++ b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java @@ -4,6 +4,7 @@ import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import java.io.IOException; import java.util.List; import java.util.Map; @@ -32,7 +33,7 @@ public class SuperParserTest { private BaseFieldManager _fieldManager; @Before - public void setup() { + public void setup() throws IOException { // AnalyzerDefinition ad = new AnalyzerDefinition(); // ad.setDefaultDefinition(new // ColumnDefinition(NoStopWordStandardAnalyzer.class.getName(), true, @@ -52,13 +53,18 @@ public class SuperParserTest { parser = new SuperParser(LUCENE_VERSION, _fieldManager, true, null, ScoreType.SUPER, new Term("_primedoc_")); } - private BaseFieldManager getFieldManager(Analyzer a) { + private BaseFieldManager getFieldManager(Analyzer a) throws IOException { BaseFieldManager fieldManager = new BaseFieldManager(BlurConstants.SUPER, a) { @Override protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props) { return true; } + + @Override + protected void tryToLoad(String fieldName) { + + } }; fieldManager.addColumnDefinitionInt("a", "id_i"); @@ -130,7 +136,7 @@ public class SuperParserTest { } @Test - public void test5() throws ParseException { + public void test5() throws ParseException, IOException { parser = new SuperParser(LUCENE_VERSION, getFieldManager(new WhitespaceAnalyzer(LUCENE_VERSION)), true, null, ScoreType.SUPER, new Term("_primedoc_")); Query query = parser.parse("super:<a.a:a a.d:{e TO f} a.b:b a.test:hello\\<> - super:<g.c:c g.d:d>");
