Fixed BLUR-205
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/72a0be2d Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/72a0be2d Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/72a0be2d Branch: refs/heads/master Commit: 72a0be2d117080bda984f804d846f814ed45cbdf Parents: ee933e4 Author: Aaron McCurry <[email protected]> Authored: Fri Aug 16 16:08:14 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Fri Aug 16 16:08:14 2013 -0400 ---------------------------------------------------------------------- .../org/apache/blur/manager/IndexManager.java | 34 +- .../blur/thrift/BlurControllerServer.java | 11 +- .../java/org/apache/blur/thrift/TableAdmin.java | 4 +- .../apache/blur/manager/IndexManagerTest.java | 9 +- .../apache/blur/jdbc/BlurDatabaseMetaData.java | 6 +- .../apache/blur/jdbc/BlurResultSetMetaData.java | 13 +- .../apache/blur/jdbc/BlurResultSetRecords.java | 6 +- .../org/apache/blur/jdbc/BlurResultSetRows.java | 2 +- .../apache/blur/analysis/BaseFieldManager.java | 32 +- .../org/apache/blur/analysis/FieldManager.java | 4 +- .../blur/analysis/FieldTypeDefinition.java | 56 +- .../apache/blur/analysis/HdfsFieldManager.java | 58 +- .../blur/analysis/ZooKeeperFieldManager.java | 157 ----- .../blur/analysis/BaseFieldManagerTest.java | 4 +- .../blur/lucene/search/SuperParserTest.java | 7 +- .../blur/shell/AddColumnDefinitionCommand.java | 2 +- .../apache/blur/shell/SchemaTableCommand.java | 13 +- .../org/apache/blur/thrift/generated/Blur.java | 512 +++++++------- .../blur/thrift/generated/ColumnDefinition.java | 148 ++--- .../apache/blur/thrift/generated/Metric.java | 132 ++-- .../apache/blur/thrift/generated/Schema.java | 241 +++---- .../blur/thrift/generated/TableDescriptor.java | 152 ++--- .../blur/thrift/util/RandomSearchTable.java | 6 +- .../util/RandomSearchTableContinuously.java | 4 +- .../src/main/scripts/interface/Blur.thrift | 88 +-- .../main/scripts/interface/gen-html/Blur.html | 56 +- .../org/apache/blur/thrift/generated/Blur.java | 512 +++++++------- .../blur/thrift/generated/ColumnDefinition.java | 148 ++--- .../apache/blur/thrift/generated/Metric.java | 132 ++-- .../apache/blur/thrift/generated/Schema.java | 241 +++---- .../blur/thrift/generated/TableDescriptor.java | 152 ++--- .../src/main/scripts/interface/gen-js/Blur.js | 456 ++++++------- .../main/scripts/interface/gen-js/Blur_types.js | 665 ++++++++++--------- .../scripts/interface/gen-perl/Blur/Blur.pm | 296 ++++----- .../scripts/interface/gen-perl/Blur/Types.pm | 563 ++++++++-------- .../main/scripts/interface/gen-rb/blur_types.rb | 96 +-- 36 files changed, 2510 insertions(+), 2508 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index dd2ffb9..521339a 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -35,7 +35,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -46,6 +45,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLongArray; import org.apache.blur.analysis.FieldManager; +import org.apache.blur.analysis.FieldTypeDefinition; import org.apache.blur.concurrent.Executors; import org.apache.blur.index.ExitableReader; import org.apache.blur.index.ExitableReader.ExitingReaderException; @@ -69,6 +69,7 @@ import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurQueryStatus; import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.ErrorType; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.FetchRowResult; @@ -742,25 +743,33 @@ public class IndexManager { } public Schema schema(String table) throws IOException { + TableContext tableContext = getTableContext(table); + FieldManager fieldManager = tableContext.getFieldManager(); + Schema schema = new Schema().setTable(table); - schema.columnFamilies = new TreeMap<String, Set<String>>(); + schema.setFamilies(new HashMap<String, Map<String,ColumnDefinition>>()); Map<String, BlurIndex> blurIndexes = _indexServer.getIndexes(table); for (BlurIndex blurIndex : blurIndexes.values()) { IndexSearcherClosable searcher = blurIndex.getIndexReader(); try { FieldInfos mergedFieldInfos = MultiFields.getMergedFieldInfos(searcher.getIndexReader()); + INNER: for (FieldInfo fieldInfo : mergedFieldInfos) { String fieldName = fieldInfo.name; + FieldTypeDefinition fieldTypeDefinition = fieldManager.getFieldTypeDefinition(fieldName); + if (fieldTypeDefinition == null) { + continue INNER; + } int index = fieldName.indexOf('.'); if (index > 0) { String columnFamily = fieldName.substring(0, index); String column = fieldName.substring(index + 1); - Set<String> set = schema.columnFamilies.get(columnFamily); - if (set == null) { - set = new TreeSet<String>(); - schema.columnFamilies.put(columnFamily, set); + Map<String, ColumnDefinition> map = schema.getFamilies().get(columnFamily); + if (map == null) { + map = new HashMap<String, ColumnDefinition>(); + schema.putToFamilies(columnFamily, map); } - set.add(column); + map.put(column, getColumnDefinition(fieldTypeDefinition)); } } } finally { @@ -771,6 +780,17 @@ public class IndexManager { return schema; } + private static ColumnDefinition getColumnDefinition(FieldTypeDefinition fieldTypeDefinition) { + ColumnDefinition columnDefinition = new ColumnDefinition(); + columnDefinition.setFamily(fieldTypeDefinition.getFamily()); + columnDefinition.setColumnName(fieldTypeDefinition.getColumnName()); + columnDefinition.setSubColumnName(fieldTypeDefinition.getSubColumnName()); + columnDefinition.setFieldLessIndexed(fieldTypeDefinition.isFieldLessIndexed()); + columnDefinition.setFieldType(fieldTypeDefinition.getFieldType()); + columnDefinition.setProperties(fieldTypeDefinition.getProperties()); + return columnDefinition; + } + public void setStatusCleanupTimerDelay(long delay) { _statusManager.setStatusCleanupTimerDelay(delay); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index 484c46f..cdac2f1 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -61,6 +61,7 @@ import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurQueryStatus; import org.apache.blur.thrift.generated.BlurResult; import org.apache.blur.thrift.generated.BlurResults; +import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.HighlightOptions; import org.apache.blur.thrift.generated.RowMutation; @@ -716,15 +717,15 @@ public class BlurControllerServer extends TableAdmin implements Iface { } public static Schema merge(Schema result, Schema schema) { - Map<String, Set<String>> destColumnFamilies = result.columnFamilies; - Map<String, Set<String>> srcColumnFamilies = schema.columnFamilies; + Map<String, Map<String, ColumnDefinition>> destColumnFamilies = result.getFamilies(); + Map<String, Map<String, ColumnDefinition>> srcColumnFamilies = schema.getFamilies(); for (String srcColumnFamily : srcColumnFamilies.keySet()) { - Set<String> destColumnNames = destColumnFamilies.get(srcColumnFamily); - Set<String> srcColumnNames = srcColumnFamilies.get(srcColumnFamily); + Map<String, ColumnDefinition> destColumnNames = destColumnFamilies.get(srcColumnFamily); + Map<String, ColumnDefinition> srcColumnNames = srcColumnFamilies.get(srcColumnFamily); if (destColumnNames == null) { destColumnFamilies.put(srcColumnFamily, srcColumnNames); } else { - destColumnNames.addAll(srcColumnNames); + destColumnNames.putAll(srcColumnNames); } } return result; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java index ea404d9..271ad62 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java @@ -405,14 +405,14 @@ public abstract class TableAdmin implements Iface { throw new BException("ColumnName in ColumnDefinition [{0}] cannot be null.", columnDefinition); } String subColumnName = columnDefinition.getSubColumnName(); - boolean fieldLessIndexing = columnDefinition.isFieldLessIndexing(); + boolean fieldLessIndexed = columnDefinition.isFieldLessIndexed(); String fieldType = columnDefinition.getFieldType(); if (fieldType == null) { throw new BException("FieldType in ColumnDefinition [{0}] cannot be null.", columnDefinition); } Map<String, String> props = columnDefinition.getProperties(); try { - return fieldManager.addColumnDefinition(family, columnName, subColumnName, fieldLessIndexing, fieldType, props); + return fieldManager.addColumnDefinition(family, columnName, subColumnName, fieldLessIndexed, fieldType, props); } catch (IOException e) { throw new BException( "Unknown error while trying to addColumnDefinition on table [{0}] with columnDefinition [{1}]", e, table, http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java index cfc3066..50b8532 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java @@ -53,6 +53,7 @@ import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurResult; import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.Facet; import org.apache.blur.thrift.generated.FetchRecordResult; import org.apache.blur.thrift.generated.FetchResult; @@ -610,11 +611,11 @@ public class IndexManagerTest { public void testSchema() throws Exception { Schema schema = indexManager.schema(TABLE); assertEquals(TABLE, schema.table); - Map<String, Set<String>> columnFamilies = schema.columnFamilies; - assertEquals(new TreeSet<String>(Arrays.asList(FAMILY, FAMILY2)), new TreeSet<String>(columnFamilies.keySet())); + Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies(); + assertEquals(new TreeSet<String>(Arrays.asList(FAMILY, FAMILY2)), new TreeSet<String>(families.keySet())); assertEquals(new TreeSet<String>(Arrays.asList("testcol1", "testcol2", "testcol3", "testcol12", "testcol13")), - new TreeSet<String>(columnFamilies.get(FAMILY))); - assertEquals(new TreeSet<String>(Arrays.asList("testcol18")), new TreeSet<String>(columnFamilies.get(FAMILY2))); + new TreeSet<String>(families.get(FAMILY).keySet())); + assertEquals(new TreeSet<String>(Arrays.asList("testcol18")), new TreeSet<String>(families.get(FAMILY2).keySet())); } @Test http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurDatabaseMetaData.java ---------------------------------------------------------------------- diff --git a/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurDatabaseMetaData.java b/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurDatabaseMetaData.java index 18a97ec..10bbd5e 100644 --- a/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurDatabaseMetaData.java +++ b/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurDatabaseMetaData.java @@ -25,7 +25,6 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.TreeMap; import org.apache.blur.jdbc.abstractimpl.AbstractBlurDatabaseMetaData; @@ -36,6 +35,7 @@ import org.apache.blur.thrift.BlurClientManager; import org.apache.blur.thrift.commands.BlurCommand; import org.apache.blur.thrift.generated.Blur.Client; import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.Schema; import org.apache.blur.thrift.generated.TableDescriptor; @@ -107,9 +107,9 @@ public class BlurDatabaseMetaData extends AbstractBlurDatabaseMetaData { continue; } Schema schema = schemaMap.get(table); - Map<String, Set<String>> columnFamilies = schema.columnFamilies; + Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies(); addTableRow(data, table, table); - for (String columnFamily : columnFamilies.keySet()) { + for (String columnFamily : families.keySet()) { String tablePlusCf = table + "." + columnFamily; if (tableNamePattern != null && !tablePlusCf.equals(tableNamePattern)) { continue; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetMetaData.java ---------------------------------------------------------------------- diff --git a/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetMetaData.java b/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetMetaData.java index f1b9c8d..f465d35 100644 --- a/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetMetaData.java +++ b/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetMetaData.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.TreeMap; import org.apache.blur.jdbc.abstractimpl.AbstractBlurResultSetMetaData; +import org.apache.blur.thrift.generated.ColumnDefinition; public class BlurResultSetMetaData extends AbstractBlurResultSetMetaData { @@ -45,7 +46,7 @@ public class BlurResultSetMetaData extends AbstractBlurResultSetMetaData { public static Collection<String> systemCols; private List<String> sqlColumnNames; private String tableName; - private Map<String, Set<String>> columnFamilies; + private Map<String, Map<String, ColumnDefinition>> families; static { systemCols = new HashSet<String>(); @@ -56,8 +57,8 @@ public class BlurResultSetMetaData extends AbstractBlurResultSetMetaData { systemCols.add(RECORDID); } - public BlurResultSetMetaData(List<String> queryColumnNames, Map<String, Set<String>> columnFamilies) { - this.columnFamilies = columnFamilies; + public BlurResultSetMetaData(List<String> queryColumnNames, Map<String, Map<String, ColumnDefinition>> families) { + this.families = families; this.sqlColumnNames = new ArrayList<String>(); this.sqlColumnNames.add(null); for (String queryColumnName : queryColumnNames) { @@ -90,7 +91,7 @@ public class BlurResultSetMetaData extends AbstractBlurResultSetMetaData { } private void addColumnFamily(String columnFamily) { - Set<String> columns = columnFamilies.get(columnFamily); + Set<String> columns = families.get(columnFamily).keySet(); if (columns == null) { throw new RuntimeException("Column family [" + columnFamily + "] not found."); } @@ -112,9 +113,9 @@ public class BlurResultSetMetaData extends AbstractBlurResultSetMetaData { sqlColumnNames.add(ROWID); sqlColumnNames.add(FAMILY); sqlColumnNames.add(RECORDID); - Map<String, Set<String>> columnFamilies = new TreeMap<String, Set<String>>(this.columnFamilies); + Map<String, Map<String, ColumnDefinition>> columnFamilies = new TreeMap<String, Map<String, ColumnDefinition>>(this.families); for (String family : columnFamilies.keySet()) { - Set<String> columnNames = columnFamilies.get(family); + Set<String> columnNames = columnFamilies.get(family).keySet(); for (String columnName : columnNames) { sqlColumnNames.add(family + "." + columnName); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetRecords.java ---------------------------------------------------------------------- diff --git a/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetRecords.java b/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetRecords.java index b4ffb24..82c8af4 100644 --- a/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetRecords.java +++ b/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetRecords.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Random; -import java.util.Set; import org.apache.blur.jdbc.abstractimpl.AbstractBlurResultSet; import org.apache.blur.jdbc.parser.Parser; @@ -40,6 +39,7 @@ import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurResult; import org.apache.blur.thrift.generated.BlurResults; import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.FetchRecordResult; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.Record; @@ -107,8 +107,8 @@ public class BlurResultSetRecords extends AbstractBlurResultSet { selector = new Selector(); setupSelector(selector, schema, columnNames); selector.recordOnly = !blurQuery.simpleQuery.superQueryOn; - Map<String, Set<String>> columnFamilies = schema.columnFamilies; - Set<String> cfSet = columnFamilies.get(columnFamily); + Map<String, Map<String, ColumnDefinition>> columnFamilies = schema.getFamilies(); + Map<String, ColumnDefinition> cfSet = columnFamilies.get(columnFamily); columnFamilies.clear(); columnFamilies.put(columnFamily, cfSet); blurResultSetMetaData = new BlurResultSetMetaData(columnNames, columnFamilies); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetRows.java ---------------------------------------------------------------------- diff --git a/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetRows.java b/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetRows.java index 2422609..a18c66e 100644 --- a/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetRows.java +++ b/blur-jdbc/src/main/java/org/apache/blur/jdbc/BlurResultSetRows.java @@ -102,7 +102,7 @@ public class BlurResultSetRows extends AbstractBlurResultSet { selector = new Selector(); setupSelector(selector, schema, columnNames); selector.recordOnly = !blurQuery.simpleQuery.superQueryOn; - blurResultSetMetaData = new BlurResultSetMetaData(columnNames, schema.columnFamilies); + blurResultSetMetaData = new BlurResultSetMetaData(columnNames, schema.getFamilies()); try { results = client.query(tableName, blurQuery); } catch (BlurException e) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java index 60b086d..7d2165d 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java @@ -269,7 +269,7 @@ public abstract class BaseFieldManager extends FieldManager { } @Override - public boolean addColumnDefinition(String family, String columnName, String subColumnName, boolean fieldLessIndexing, + public boolean addColumnDefinition(String family, String columnName, String subColumnName, boolean fieldLessIndexed, String fieldType, Map<String, String> props) throws IOException { String baseFieldName = family + "." + columnName; String fieldName; @@ -279,7 +279,7 @@ public abstract class BaseFieldManager extends FieldManager { throw new IllegalArgumentException("Base column of [" + baseFieldName + "] not found, please add base before adding sub column."); } - if (fieldLessIndexing) { + if (fieldLessIndexed) { throw new IllegalArgumentException("Subcolumn of [" + subColumnName + "] from base of [" + baseFieldName + "] cannot be added with fieldLessIndexing set to true."); } @@ -287,16 +287,16 @@ public abstract class BaseFieldManager extends FieldManager { } else { fieldName = baseFieldName; } - return addFieldTypeDefinition(fieldName, fieldLessIndexing, fieldType, props); + return addFieldTypeDefinition(family, columnName, subColumnName, fieldName, fieldLessIndexed, fieldType, props); } - private boolean addFieldTypeDefinition(String fieldName, boolean fieldLessIndexing, String fieldType, - Map<String, String> props) throws IOException { + private boolean addFieldTypeDefinition(String family, String columnName, String subColumnName, String fieldName, + boolean fieldLessIndexed, String fieldType, Map<String, String> props) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName); if (fieldTypeDefinition != null) { return false; } - fieldTypeDefinition = newFieldTypeDefinition(fieldName, fieldLessIndexing, fieldType, props); + fieldTypeDefinition = newFieldTypeDefinition(fieldName, fieldLessIndexed, fieldType, props); synchronized (_fieldNameToDefMap) { for (String alternateFieldName : fieldTypeDefinition.getAlternateFieldNames()) { if (_fieldNameToDefMap.containsKey(alternateFieldName)) { @@ -305,7 +305,8 @@ public abstract class BaseFieldManager extends FieldManager { + "], this field type definition cannot be added."); } } - if (!tryToStore(fieldName, fieldLessIndexing, fieldType, props)) { + setFields(fieldTypeDefinition, family, columnName, subColumnName, fieldLessIndexed, fieldType, props); + if (!tryToStore(fieldTypeDefinition, fieldName)) { return false; } registerFieldTypeDefinition(fieldName, fieldTypeDefinition); @@ -313,6 +314,16 @@ public abstract class BaseFieldManager extends FieldManager { return true; } + private void setFields(FieldTypeDefinition fieldTypeDefinition, String family, String columnName, + String subColumnName, boolean fieldLessIndexed, String fieldType, Map<String, String> props) { + fieldTypeDefinition.setFamily(family); + fieldTypeDefinition.setColumnName(columnName); + fieldTypeDefinition.setSubColumnName(subColumnName); + fieldTypeDefinition.setFieldLessIndexed(fieldLessIndexed); + fieldTypeDefinition.setFieldType(fieldType); + fieldTypeDefinition.setProperties(props); + } + protected void registerFieldTypeDefinition(String fieldName, FieldTypeDefinition fieldTypeDefinition) { _fieldNameToDefMap.put(fieldName, fieldTypeDefinition); for (String alternateFieldName : fieldTypeDefinition.getAlternateFieldNames()) { @@ -344,7 +355,7 @@ public abstract class BaseFieldManager extends FieldManager { return fieldName.substring(0, indexOf); } - protected FieldTypeDefinition newFieldTypeDefinition(String fieldName, boolean fieldLessIndexing, String fieldType, + protected FieldTypeDefinition newFieldTypeDefinition(String fieldName, boolean fieldLessIndexed, String fieldType, Map<String, String> props) { if (fieldType == null) { throw new IllegalArgumentException("Field type can not be null."); @@ -368,12 +379,11 @@ public abstract class BaseFieldManager extends FieldManager { } else { fieldTypeDefinition.configure(fieldName, props); } - fieldTypeDefinition.setFieldLessIndexing(fieldLessIndexing); + fieldTypeDefinition.setFieldLessIndexed(fieldLessIndexed); return fieldTypeDefinition; } - protected abstract boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, - Map<String, String> props) throws IOException; + protected abstract boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) throws IOException; protected abstract void tryToLoad(String fieldName) throws IOException; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java index 6b4fe23..bd2033c 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java @@ -47,7 +47,7 @@ public abstract class FieldManager { * @param subColumnName * the sub column name, optional can be null if it's not a sub * column. - * @param fieldLessIndexing + * @param fieldLessIndexed * indicates whether the field should be added to the default field * for the record for fieldless searching. * @param fieldType @@ -58,7 +58,7 @@ public abstract class FieldManager { * @throws IOException */ public abstract boolean addColumnDefinition(String family, String columnName, String subColumnName, - boolean fieldLessIndexing, String fieldType, Map<String, String> props) throws IOException; + boolean fieldLessIndexed, String fieldType, Map<String, String> props) throws IOException; /** * Gets the analyzer for the indexing process. http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java index 96c84d3..19a67a3 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java @@ -28,8 +28,13 @@ import org.apache.lucene.search.Query; public abstract class FieldTypeDefinition { - protected static final Collection<String> EMPTY_COLLECTION = Arrays.asList(new String[]{}); - protected boolean _fieldLessIndexing; + protected static final Collection<String> EMPTY_COLLECTION = Arrays.asList(new String[] {}); + private boolean _fieldLessIndexed; + private String _family; + private String _columnName; + private String _subColumnName; + private String _fieldType; + private Map<String, String> _properties; /** * Gets the name of the field type. @@ -146,11 +151,11 @@ public abstract class FieldTypeDefinition { } public boolean isFieldLessIndexed() { - return _fieldLessIndexing; + return _fieldLessIndexed; } - public void setFieldLessIndexing(boolean fieldLessIndexing) { - _fieldLessIndexing = fieldLessIndexing; + public void setFieldLessIndexed(boolean fieldLessIndexed) { + _fieldLessIndexed = fieldLessIndexed; } public abstract boolean checkSupportForFuzzyQuery(); @@ -167,4 +172,45 @@ public abstract class FieldTypeDefinition { throw new RuntimeException("Not supported."); } + public String getFamily() { + return _family; + } + + public void setFamily(String family) { + this._family = family; + } + + public String getColumnName() { + return _columnName; + } + + public void setColumnName(String columnName) { + this._columnName = columnName; + } + + public String getSubColumnName() { + return _subColumnName; + } + + public void setSubColumnName(String subColumnName) { + this._subColumnName = subColumnName; + } + + public String getFieldType() { + return _fieldType; + } + + public void setFieldType(String fieldType) { + this._fieldType = fieldType; + } + + public Map<String, String> getProperties() { + return _properties; + } + + public void setProperties(Map<String, String> properties) { + this._properties = properties; + } + + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java index 2cbd9cb..71f1434 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java @@ -44,8 +44,13 @@ public class HdfsFieldManager extends BaseFieldManager { } private static final Log LOG = LogFactory.getLog(HdfsFieldManager.class); - private static final String FIELD_TYPE = "fieldType"; - private static final String FIELD_LESS_INDEXING = "fieldLessIndexing"; + + private static final String FIELD_TYPE = "_fieldType_"; + private static final String FIELD_LESS_INDEXING = "_fieldLessIndexing_"; + private static final String FAMILY = "_family_"; + private static final String COLUMN_NAME = "_columnName_"; + private static final String SUB_COLUMN_NAME = "_subColumnName_"; + private static Lock _lock = new Lock() { private final java.util.concurrent.locks.Lock _javalock = new ReentrantReadWriteLock().writeLock(); @@ -80,16 +85,22 @@ public class HdfsFieldManager extends BaseFieldManager { } @Override - protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props) - throws IOException { + protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) throws IOException { // Might want to make this a ZK lock _lock.lock(); try { + String fieldType = fieldTypeDefinition.getFieldType(); + boolean fieldLessIndexed = fieldTypeDefinition.isFieldLessIndexed(); LOG.info("Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]", - fieldName, fieldLessIndexing, fieldType, props); + fieldName, fieldLessIndexed, fieldType, fieldTypeDefinition.getProperties()); Properties properties = new Properties(); - properties.setProperty(FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexing)); - properties.setProperty(FIELD_TYPE, fieldType); + setProperty(properties, FAMILY, fieldTypeDefinition.getFamily()); + setProperty(properties, FAMILY, fieldTypeDefinition.getFamily()); + setProperty(properties, COLUMN_NAME, fieldTypeDefinition.getColumnName()); + setProperty(properties, SUB_COLUMN_NAME, fieldTypeDefinition.getSubColumnName()); + setProperty(properties, FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexed)); + setProperty(properties, FIELD_TYPE, fieldType); + Map<String, String> props = fieldTypeDefinition.getProperties(); if (props != null) { for (Entry<String, String> e : props.entrySet()) { properties.setProperty(e.getKey(), e.getValue()); @@ -97,7 +108,7 @@ public class HdfsFieldManager extends BaseFieldManager { } Path path = getFieldPath(fieldName); if (_fileSystem.exists(path)) { - LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props); + LOG.info("Field [{0}] already exists.", fieldName); return false; } Path tmpPath = new Path(path.getParent(), UUID.randomUUID().toString() + ".tmp"); @@ -110,7 +121,7 @@ public class HdfsFieldManager extends BaseFieldManager { return true; } else { _fileSystem.delete(tmpPath, false); - LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props); + LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexed, fieldType, props); return false; } } finally { @@ -118,6 +129,13 @@ public class HdfsFieldManager extends BaseFieldManager { } } + private void setProperty(Properties properties, String key, String value) { + if (value == null) { + return; + } + properties.setProperty(key, value); + } + private Path getFieldPath(String fieldName) { return new Path(_storagePath, fieldName); } @@ -135,12 +153,19 @@ public class HdfsFieldManager extends BaseFieldManager { return; } FSDataInputStream inputStream = _fileSystem.open(path); - Properties props = new Properties(); - props.load(inputStream); + Properties properties = new Properties(); + properties.load(inputStream); inputStream.close(); - boolean fieldLessIndexing = Boolean.parseBoolean(props.getProperty(FIELD_LESS_INDEXING)); - String fieldType = props.getProperty(FIELD_TYPE); - FieldTypeDefinition fieldTypeDefinition = newFieldTypeDefinition(fieldName, fieldLessIndexing, fieldType, toMap(props)); + boolean fieldLessIndexing = Boolean.parseBoolean(properties.getProperty(FIELD_LESS_INDEXING)); + String fieldType = properties.getProperty(FIELD_TYPE); + Map<String, String> props = toMap(properties); + FieldTypeDefinition fieldTypeDefinition = newFieldTypeDefinition(fieldName, fieldLessIndexing, fieldType, props); + fieldTypeDefinition.setFamily(properties.getProperty(FAMILY)); + fieldTypeDefinition.setColumnName(properties.getProperty(COLUMN_NAME)); + fieldTypeDefinition.setSubColumnName(properties.getProperty(SUB_COLUMN_NAME)); + fieldTypeDefinition.setFieldLessIndexed(fieldLessIndexing); + fieldTypeDefinition.setFieldType(properties.getProperty(FIELD_TYPE)); + fieldTypeDefinition.setProperties(props); registerFieldTypeDefinition(fieldName, fieldTypeDefinition); } finally { _lock.unlock(); @@ -152,6 +177,11 @@ public class HdfsFieldManager extends BaseFieldManager { for (Entry<Object, Object> e : props.entrySet()) { result.put(e.getKey().toString(), e.getValue().toString()); } + result.remove(FAMILY); + result.remove(COLUMN_NAME); + result.remove(SUB_COLUMN_NAME); + result.remove(FIELD_TYPE); + result.remove(FIELD_LESS_INDEXING); return result; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java deleted file mode 100644 index 65b3030..0000000 --- a/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java +++ /dev/null @@ -1,157 +0,0 @@ -package org.apache.blur.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; - -import org.apache.blur.log.Log; -import org.apache.blur.log.LogFactory; -import org.apache.commons.io.output.ByteArrayOutputStream; -import org.apache.lucene.analysis.Analyzer; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; - -public class ZooKeeperFieldManager extends BaseFieldManager { - - private static final String FIELD_TYPE = "fieldType"; - private static final String FIELD_LESS_INDEXING = "fieldLessIndexing"; - private static final Object lock = new Object(); - - private static final Log LOG = LogFactory.getLog(ZooKeeperFieldManager.class); - private final ZooKeeper _zooKeeper; - private final String _basePath; - - public ZooKeeperFieldManager(String fieldLessField, Analyzer defaultAnalyzerForQuerying, ZooKeeper zooKeeper, String basePath) - throws IOException { - this(fieldLessField, defaultAnalyzerForQuerying, zooKeeper, basePath, true, null, false, null); - } - - public ZooKeeperFieldManager(String fieldLessField, Analyzer defaultAnalyzerForQuerying, ZooKeeper zooKeeper, - String basePath, boolean strict, String defaultMissingFieldType, boolean defaultMissingFieldLessIndexing, - Map<String, String> defaultMissingFieldProps) throws IOException { - super(fieldLessField, defaultAnalyzerForQuerying, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing, - defaultMissingFieldProps); - _zooKeeper = zooKeeper; - _basePath = basePath; - } - - @Override - protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props) - throws IOException { - // Might want to make this a ZK lock - synchronized (lock) { - LOG.info("Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]", - fieldName, fieldLessIndexing, fieldType, props); - Properties properties = new Properties(); - properties.setProperty(FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexing)); - properties.setProperty(FIELD_TYPE, fieldType); - if (props != null) { - for (Entry<String, String> e : props.entrySet()) { - properties.setProperty(e.getKey(), e.getValue()); - } - } - String path = getFieldPath(fieldName); - Stat stat; - try { - stat = _zooKeeper.exists(path, false); - } catch (KeeperException e) { - throw new IOException(e); - } catch (InterruptedException e) { - throw new IOException(e); - } - if (stat != null) { - LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props); - return false; - } - try { - _zooKeeper.create(path, toBytes(properties), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - return true; - } catch (KeeperException e) { - if (e.code() == KeeperException.Code.NODEEXISTS) { - LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props); - return false; - } - throw new IOException(e); - } catch (InterruptedException e) { - throw new IOException(e); - } - } - } - - private byte[] toBytes(Properties properties) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - properties.store(outputStream, getComments()); - outputStream.close(); - return outputStream.toByteArray(); - } - - private String getFieldPath(String fieldName) { - return _basePath + "/" + fieldName; - } - - private String getComments() { - return "This file is generated from Apache Blur to store meta data about field types. DO NOT MODIFIY!"; - } - - @Override - protected void tryToLoad(String fieldName) throws IOException { - String path = getFieldPath(fieldName); - Stat stat; - try { - stat = _zooKeeper.exists(path, false); - } catch (KeeperException e) { - throw new IOException(e); - } catch (InterruptedException e) { - throw new IOException(e); - } - if (stat != null) { - return; - } - byte[] data; - try { - data = _zooKeeper.getData(path, false, stat); - } catch (KeeperException e) { - throw new IOException(e); - } catch (InterruptedException e) { - throw new IOException(e); - } - Properties props = new Properties(); - ByteArrayInputStream inputStream = new ByteArrayInputStream(data); - props.load(inputStream); - inputStream.close(); - boolean fieldLessIndexing = Boolean.parseBoolean(props.getProperty(FIELD_LESS_INDEXING)); - String fieldType = props.getProperty(FIELD_TYPE); - FieldTypeDefinition fieldTypeDefinition = newFieldTypeDefinition(fieldName, fieldLessIndexing, fieldType, toMap(props)); - registerFieldTypeDefinition(fieldName, fieldTypeDefinition); - } - - private Map<String, String> toMap(Properties props) { - Map<String, String> result = new HashMap<String, String>(); - for (Entry<Object, Object> e : props.entrySet()) { - result.put(e.getKey().toString(), e.getValue().toString()); - } - return result; - } -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java index cc44e84..917338c 100644 --- a/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java +++ b/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java @@ -22,7 +22,6 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import org.apache.blur.analysis.type.TextFieldTypeDefinition; import org.apache.blur.thrift.generated.Column; @@ -178,8 +177,7 @@ public class BaseFieldManagerTest { protected BaseFieldManager newFieldManager(boolean create) throws IOException { return new BaseFieldManager(_fieldLessField, new KeywordAnalyzer()) { @Override - protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, - Map<String, String> props) { + protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) { return true; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java index 94726c2..52c1051 100644 --- a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java +++ b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java @@ -7,9 +7,9 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.util.List; -import java.util.Map; import org.apache.blur.analysis.BaseFieldManager; +import org.apache.blur.analysis.FieldTypeDefinition; import org.apache.blur.analysis.NoStopWordStandardAnalyzer; import org.apache.blur.analysis.type.spatial.SpatialArgsParser; import org.apache.blur.thrift.generated.ScoreType; @@ -53,8 +53,7 @@ public class SuperParserTest { private BaseFieldManager getFieldManager(Analyzer a) throws IOException { BaseFieldManager fieldManager = new BaseFieldManager(BlurConstants.SUPER, a) { @Override - protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, - Map<String, String> props) { + protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) { return true; } @@ -333,7 +332,7 @@ public class SuperParserTest { SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, circle); String writeSpatialArgs = SpatialArgsParser.writeSpatialArgs(args, shapeReadWriter); - + // This has to be done because of rounding. SpatialArgs spatialArgs = SpatialArgsParser.parse(writeSpatialArgs, shapeReadWriter); Query q1 = sq(strategy.makeQuery(spatialArgs)); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java index 757f1e7..611eea4 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java @@ -51,7 +51,7 @@ public class AddColumnDefinitionCommand extends Command { columnDefinition.setColumnName(cmd.getOptionValue("s")); } if (cmd.hasOption("F")) { - columnDefinition.setFieldLessIndexing(true); + columnDefinition.setFieldLessIndexed(true); } if (cmd.hasOption("p")) { Option[] options = cmd.getOptions(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/72a0be2d/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java index 3fa82eb..a165724 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java @@ -25,6 +25,7 @@ import java.util.Set; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.Blur; import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.Schema; public class SchemaTableCommand extends Command { @@ -38,12 +39,14 @@ public class SchemaTableCommand extends Command { Schema schema = client.schema(tablename); out.println(schema.getTable()); - Map<String, Set<String>> columnFamilies = schema.getColumnFamilies(); - for (String cf : columnFamilies.keySet()) { + Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies(); + for (String cf : families.keySet()) { out.println("family : " + cf); - Set<String> columns = columnFamilies.get(cf); - for (String c : columns) { - out.println("\tcolumn : " + c); + Map<String, ColumnDefinition> columns = families.get(cf); + for (String c : columns.keySet()) { + ColumnDefinition columnDefinition = columns.get(c); +// out.println("\tcolumn : " + c); + out.println("\tcolumn : " + columnDefinition); } } }
