Updated Branches: refs/heads/master dccd15276 -> c7385fb29
Fixed some schema issues. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/510de991 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/510de991 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/510de991 Branch: refs/heads/master Commit: 510de9917c1db85a2c573c15de0c2612691f7f22 Parents: dccd152 Author: Aaron McCurry <[email protected]> Authored: Tue Aug 20 10:18:52 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Aug 20 10:18:52 2013 -0400 ---------------------------------------------------------------------- .../org/apache/blur/manager/IndexManager.java | 87 +++++++++----------- .../org/apache/blur/server/TableContext.java | 4 +- .../apache/blur/analysis/BaseFieldManager.java | 15 ++++ .../org/apache/blur/analysis/FieldManager.java | 7 ++ .../apache/blur/analysis/HdfsFieldManager.java | 16 ++++ .../blur/analysis/BaseFieldManagerTest.java | 5 ++ .../blur/lucene/search/SuperParserTest.java | 6 ++ .../blur/shell/AddColumnDefinitionCommand.java | 2 +- .../apache/blur/shell/SchemaTableCommand.java | 31 ++++++- 9 files changed, 118 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index eba9b29..f4ace8e 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -96,7 +96,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.StoredFieldVisitor; @@ -279,12 +278,11 @@ public class IndexManager { } TableContext context = getTableContext(table); - Filter preFilter = QueryParserUtil.parseFilter(table, query.recordFilter, false, fieldManager, - _filterCache, context); - Filter postFilter = QueryParserUtil.parseFilter(table, query.rowFilter, true, fieldManager, - _filterCache, context); - return QueryParserUtil.parseQuery(query.query, query.rowQuery, fieldManager, postFilter, - preFilter, getScoreType(query.scoreType), context); + Filter preFilter = QueryParserUtil.parseFilter(table, query.recordFilter, false, fieldManager, _filterCache, + context); + Filter postFilter = QueryParserUtil.parseFilter(table, query.rowFilter, true, fieldManager, _filterCache, context); + return QueryParserUtil.parseQuery(query.query, query.rowQuery, fieldManager, postFilter, preFilter, + getScoreType(query.scoreType), context); } private void populateSelector(String table, Selector selector) throws IOException, BlurException { @@ -392,10 +390,10 @@ public class IndexManager { org.apache.blur.thrift.generated.Query simpleQuery = blurQuery.query; Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.recordFilter, false, fieldManager, _filterCache, context); - Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.rowFilter, true, fieldManager, - _filterCache, context); - Query userQuery = QueryParserUtil.parseQuery(simpleQuery.query, simpleQuery.rowQuery, fieldManager, - postFilter, preFilter, getScoreType(simpleQuery.scoreType), context); + Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.rowFilter, true, fieldManager, _filterCache, + context); + Query userQuery = QueryParserUtil.parseQuery(simpleQuery.query, simpleQuery.rowQuery, fieldManager, postFilter, + preFilter, getScoreType(simpleQuery.scoreType), context); Query facetedQuery = getFacetedQuery(blurQuery, userQuery, facetedCounts, fieldManager, context, postFilter, preFilter); call = new SimpleQueryParallelCall(running, table, status, _indexServer, facetedQuery, blurQuery.selector, @@ -430,15 +428,16 @@ public class IndexManager { } } - public String parseQuery(String table, org.apache.blur.thrift.generated.Query simpleQuery) throws ParseException, BlurException { + public String parseQuery(String table, org.apache.blur.thrift.generated.Query simpleQuery) throws ParseException, + BlurException { TableContext context = getTableContext(table); FieldManager fieldManager = context.getFieldManager(); - Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.recordFilter, false, fieldManager, - _filterCache, context); - Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.rowFilter, true, fieldManager, - _filterCache, context); - Query userQuery = QueryParserUtil.parseQuery(simpleQuery.query, simpleQuery.rowQuery, fieldManager, - postFilter, preFilter, getScoreType(simpleQuery.scoreType), context); + Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.recordFilter, false, fieldManager, _filterCache, + context); + Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.rowFilter, true, fieldManager, _filterCache, + context); + Query userQuery = QueryParserUtil.parseQuery(simpleQuery.query, simpleQuery.rowQuery, fieldManager, postFilter, + preFilter, getScoreType(simpleQuery.scoreType), context); return userQuery.toString(); } @@ -459,8 +458,8 @@ public class IndexManager { int size = blurQuery.facets.size(); Query[] queries = new Query[size]; for (int i = 0; i < size; i++) { - queries[i] = QueryParserUtil.parseQuery(blurQuery.facets.get(i).queryStr, blurQuery.query.rowQuery, - fieldManager, postFilter, preFilter, ScoreType.CONSTANT, context); + queries[i] = QueryParserUtil.parseQuery(blurQuery.facets.get(i).queryStr, blurQuery.query.rowQuery, fieldManager, + postFilter, preFilter, ScoreType.CONSTANT, context); } return queries; } @@ -744,36 +743,26 @@ public class IndexManager { public Schema schema(String table) throws IOException { TableContext tableContext = getTableContext(table); FieldManager fieldManager = tableContext.getFieldManager(); - Schema schema = new Schema().setTable(table); - schema.setFamilies(new HashMap<String, Map<String,ColumnDefinition>>()); - Map<String, BlurIndex> blurIndexes = _indexServer.getIndexes(table); - for (BlurIndex blurIndex : blurIndexes.values()) { - IndexSearcherClosable searcher = blurIndex.getIndexReader(); - try { - FieldInfos mergedFieldInfos = MultiFields.getMergedFieldInfos(searcher.getIndexReader()); - INNER: - for (FieldInfo fieldInfo : mergedFieldInfos) { - String fieldName = fieldInfo.name; - FieldTypeDefinition fieldTypeDefinition = fieldManager.getFieldTypeDefinition(fieldName); - if (fieldTypeDefinition == null) { - continue INNER; - } - int index = fieldName.indexOf('.'); - if (index > 0) { - String columnFamily = fieldName.substring(0, index); - String column = fieldName.substring(index + 1); - Map<String, ColumnDefinition> map = schema.getFamilies().get(columnFamily); - if (map == null) { - map = new HashMap<String, ColumnDefinition>(); - schema.putToFamilies(columnFamily, map); - } - map.put(column, getColumnDefinition(fieldTypeDefinition)); - } - } - } finally { - // this will allow for closing of index - searcher.close(); + schema.setFamilies(new HashMap<String, Map<String, ColumnDefinition>>()); + Set<String> fieldNames = fieldManager.getFieldNames(); + INNER: for (String fieldName : fieldNames) { + FieldTypeDefinition fieldTypeDefinition = fieldManager.getFieldTypeDefinition(fieldName); + if (fieldTypeDefinition == null) { + continue INNER; + } + String columnName = fieldTypeDefinition.getColumnName(); + String columnFamily = fieldTypeDefinition.getFamily(); + String subColumnName = fieldTypeDefinition.getSubColumnName(); + Map<String, ColumnDefinition> map = schema.getFamilies().get(columnFamily); + if (map == null) { + map = new HashMap<String, ColumnDefinition>(); + schema.putToFamilies(columnFamily, map); + } + if (subColumnName == null) { + map.put(columnName, getColumnDefinition(fieldTypeDefinition)); + } else { + map.put(columnName + "." + subColumnName, getColumnDefinition(fieldTypeDefinition)); } } return schema; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-core/src/main/java/org/apache/blur/server/TableContext.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java index cc2d541..24fe308 100644 --- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java +++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java @@ -117,8 +117,10 @@ public class TableContext { Path storagePath = new Path(tableContext.tablePath, TYPES); try { - tableContext.fieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(), storagePath, + HdfsFieldManager hdfsFieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(), storagePath, configuration, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing, defaultMissingFieldProps); + hdfsFieldManager.load(); + tableContext.fieldManager = hdfsFieldManager; } catch (IOException e) { throw new RuntimeException(e); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java index 7d2165d..92779ac 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -155,6 +156,18 @@ public abstract class BaseFieldManager extends FieldManager { } @Override + public Set<String> getFieldNames() throws IOException { + return new TreeSet<String>(_fieldNameToDefMap.keySet()); + } + + public void load() throws IOException { + List<String> fieldNamesToLoad = getFieldNamesToLoad(); + for (String fieldName : fieldNamesToLoad) { + tryToLoad(fieldName); + } + } + + @Override public void registerType(Class<? extends FieldTypeDefinition> c) { try { FieldTypeDefinition fieldTypeDefinition = c.newInstance(); @@ -387,6 +400,8 @@ public abstract class BaseFieldManager extends FieldManager { protected abstract void tryToLoad(String fieldName) throws IOException; + protected abstract List<String> getFieldNamesToLoad() throws IOException; + private Set<String> getConcurrentSet() { return Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java index bd2033c..b8cc592 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java @@ -19,6 +19,7 @@ package org.apache.blur.analysis; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.blur.thrift.generated.Record; import org.apache.lucene.analysis.Analyzer; @@ -250,6 +251,12 @@ public abstract class FieldManager { */ public abstract Query getCustomQuery(String field, String text) throws IOException; + /** + * Register a {@link FieldTypeDefinition} into this field manager. + * @param c the class. + */ public abstract void registerType(Class<? extends FieldTypeDefinition> c); + + public abstract Set<String> getFieldNames() throws IOException; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java index 71f1434..b56faa1 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java @@ -17,7 +17,9 @@ package org.apache.blur.analysis; * limitations under the License. */ import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; @@ -29,6 +31,7 @@ import org.apache.blur.log.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.lucene.analysis.Analyzer; @@ -83,6 +86,18 @@ public class HdfsFieldManager extends BaseFieldManager { _configuration = configuration; _fileSystem = _storagePath.getFileSystem(_configuration); } + + @Override + protected List<String> getFieldNamesToLoad() throws IOException { + FileStatus[] listStatus = _fileSystem.listStatus(_storagePath); + List<String> fieldNames = new ArrayList<String>(); + for (FileStatus fileStatus : listStatus) { + if (!fileStatus.isDir()) { + fieldNames.add(fileStatus.getPath().getName()); + } + } + return fieldNames; + } @Override protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) throws IOException { @@ -192,4 +207,5 @@ public class HdfsFieldManager extends BaseFieldManager { public static void setLock(Lock lock) { _lock = lock; } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java index 917338c..bee4b88 100644 --- a/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java +++ b/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java @@ -185,6 +185,11 @@ public class BaseFieldManagerTest { protected void tryToLoad(String field) { } + + @Override + protected List<String> getFieldNamesToLoad() throws IOException { + return new ArrayList<String>(); + } }; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java index 12f5029..9ec2784 100644 --- a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java +++ b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java @@ -6,6 +6,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import org.apache.blur.analysis.BaseFieldManager; @@ -61,6 +62,11 @@ public class SuperParserTest { protected void tryToLoad(String fieldName) { } + + @Override + protected List<String> getFieldNamesToLoad() throws IOException { + return new ArrayList<String>(); + } }; fieldManager.addColumnDefinitionInt("a", "id_i"); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java index 8d7ef16..98b5a2c 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java @@ -48,7 +48,7 @@ public class AddColumnDefinitionCommand extends Command implements TableFirstArg columnDefinition.setColumnName(args[3]); columnDefinition.setFieldType(args[4]); if (cmd.hasOption("s")) { - columnDefinition.setColumnName(cmd.getOptionValue("s")); + columnDefinition.setSubColumnName(cmd.getOptionValue("s")); } if (cmd.hasOption("F")) { columnDefinition.setFieldLessIndexed(true); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java index 67606c3..b33655d 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java @@ -20,6 +20,10 @@ package org.apache.blur.shell; import java.io.PrintWriter; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.Blur; @@ -37,15 +41,34 @@ public class SchemaTableCommand extends Command implements TableFirstArgCommand String tablename = args[1]; Schema schema = client.schema(tablename); + out.println(schema); out.println(schema.getTable()); Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies(); - for (String cf : families.keySet()) { + Set<String> familyNames = new TreeSet<String>(families.keySet()); + for (String cf : familyNames) { out.println("family : " + cf); Map<String, ColumnDefinition> columns = families.get(cf); - for (String c : columns.keySet()) { + Set<String> columnNames = new TreeSet<String>(columns.keySet()); + for (String c : columnNames) { ColumnDefinition columnDefinition = columns.get(c); -// out.println("\tcolumn : " + c); - out.println("\tcolumn : " + columnDefinition); + out.println("\tcolumn : " + columnDefinition.getColumnName()); + String fieldType = columnDefinition.getFieldType(); + Map<String, String> properties = columnDefinition.getProperties(); + String subColumnName = columnDefinition.getSubColumnName(); + if (subColumnName != null) { + out.println( "\t\t\tsubName : " + subColumnName); + out.println( "\t\t\tfieldType : " + fieldType); + Map<String, String> props = new TreeMap<String, String>(properties); + for (Entry<String, String> e : props.entrySet()) { + out.println("\t\t\tprop : " + e); + } + } else { + out.println( "\t\tfieldType : " + fieldType); + Map<String, String> props = new TreeMap<String, String>(properties); + for (Entry<String, String> e : props.entrySet()) { + out.println("\t\tprop : " + e); + } + } } } }
