Final changes to get the project running again with types. All unit tests pass but I still need to add external type additions and cluster lock on adding fields via ZooKeeper so that read write contention against HDFS is lowered during high field creation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/2378815f Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/2378815f Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/2378815f Branch: refs/heads/master Commit: 2378815f1a479589095edee4d6135e776ae9b907 Parents: b708464 Author: Aaron McCurry <[email protected]> Authored: Fri Aug 9 13:21:40 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Fri Aug 9 13:21:40 2013 -0400 ---------------------------------------------------------------------- .../manager/writer/TransactionRecorder.java | 96 ++-- .../org/apache/blur/server/TableContext.java | 40 +- .../org/apache/blur/mapreduce/BlurReducer.java | 56 +- .../blur/mapreduce/lib/BlurOutputFormat.java | 29 +- .../mapreduce/lib/BlurOutputFormatTest.java | 21 +- .../apache/blur/analysis/BaseFieldManager.java | 79 ++- .../org/apache/blur/analysis/FieldManager.java | 75 ++- .../apache/blur/analysis/HdfsFieldManager.java | 89 ++- .../blur/analysis/ZooKeeperFieldManager.java | 157 +++++ .../type/FieldLessFieldTypeDefinition.java | 104 ++++ .../org/apache/blur/thrift/generated/Blur.java | 548 +++++++++--------- .../apache/blur/thrift/generated/Metric.java | 132 ++--- .../blur/thrift/generated/TableDescriptor.java | 574 ++++++++++++++++++- .../src/main/scripts/interface/Blur.thrift | 21 +- .../main/scripts/interface/gen-html/Blur.html | 11 +- .../org/apache/blur/thrift/generated/Blur.java | 548 +++++++++--------- .../apache/blur/thrift/generated/Metric.java | 132 ++--- .../blur/thrift/generated/TableDescriptor.java | 574 ++++++++++++++++++- .../src/main/scripts/interface/gen-js/Blur.js | 486 ++++++++-------- .../main/scripts/interface/gen-js/Blur_types.js | 234 +++++--- .../scripts/interface/gen-perl/Blur/Blur.pm | 316 +++++----- .../scripts/interface/gen-perl/Blur/Types.pm | 175 ++++-- .../main/scripts/interface/gen-rb/blur_types.rb | 15 +- 23 files changed, 3125 insertions(+), 1387 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java index 994c687..207d21f 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java @@ -358,30 +358,41 @@ public class TransactionRecorder extends TimerTask implements Closeable { public static List<List<Field>> getDocs(Row row, FieldManager fieldManager) throws IOException { List<Record> records = row.records; + if (records == null) { + return null; + } int size = records.size(); + if (size == 0) { + return null; + } final String rowId = row.id; List<List<Field>> docs = new ArrayList<List<Field>>(size); for (int i = 0; i < size; i++) { Record record = records.get(i); - BlurUtil.validateRowIdAndRecord(rowId, record); - List<Field> fields = fieldManager.getFields(rowId, record); - if (i == 0) { - fields.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); - } + List<Field> fields = getDoc(fieldManager, rowId, record); docs.add(fields); } + List<Field> doc = docs.get(0); + doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); return docs; } -// public static Document convert(String rowId, Record record, Analyzer analyzer) { -// BlurUtil.validateRowIdAndRecord(rowId, record); -// Document document = new Document(); -// document.add(new Field(BlurConstants.ROW_ID, rowId, ID_TYPE)); -// document.add(new Field(BlurConstants.RECORD_ID, record.recordId, ID_TYPE)); -// document.add(new Field(BlurConstants.FAMILY, record.family, ID_TYPE)); -// addColumns(document, analyzer, record.family, record.columns); -// return document; -// } + public static List<Field> getDoc(FieldManager fieldManager, final String rowId, Record record) throws IOException { + BlurUtil.validateRowIdAndRecord(rowId, record); + List<Field> fields = fieldManager.getFields(rowId, record); + return fields; + } + + // public static Document convert(String rowId, Record record, Analyzer + // analyzer) { + // BlurUtil.validateRowIdAndRecord(rowId, record); + // Document document = new Document(); + // document.add(new Field(BlurConstants.ROW_ID, rowId, ID_TYPE)); + // document.add(new Field(BlurConstants.RECORD_ID, record.recordId, ID_TYPE)); + // document.add(new Field(BlurConstants.FAMILY, record.family, ID_TYPE)); + // addColumns(document, analyzer, record.family, record.columns); + // return document; + // } private Term createRowId(String id) { return new Term(BlurConstants.ROW_ID, id); @@ -409,34 +420,35 @@ public class TransactionRecorder extends TimerTask implements Closeable { } } -// public static boolean addColumns(Document document, Analyzer analyzer, String columnFamily, Iterable<Column> set) { -// if (set == null) { -// return false; -// } -// OUTER: for (Column column : set) { -// String name = column.getName(); -// String value = column.value; -// if (value == null || name == null) { -// continue OUTER; -// } -// String fieldName = getFieldName(columnFamily, name); -// FieldType fieldType = analyzer.getFieldType(fieldName); -// Field field = analyzer.getField(fieldName, value, fieldType); -// document.add(field); -// -// if (analyzer.isFullTextField(fieldName)) { -// document.add(new Field(SUPER, value, SUPER_FIELD_TYPE)); -// } -// Set<String> subFieldNames = analyzer.getSubIndexNames(fieldName); -// if (subFieldNames != null) { -// for (String subFieldName : subFieldNames) { -// FieldType subFieldType = analyzer.getFieldType(subFieldName); -// document.add(analyzer.getField(subFieldName, value, subFieldType)); -// } -// } -// } -// return true; -// } + // public static boolean addColumns(Document document, Analyzer analyzer, + // String columnFamily, Iterable<Column> set) { + // if (set == null) { + // return false; + // } + // OUTER: for (Column column : set) { + // String name = column.getName(); + // String value = column.value; + // if (value == null || name == null) { + // continue OUTER; + // } + // String fieldName = getFieldName(columnFamily, name); + // FieldType fieldType = analyzer.getFieldType(fieldName); + // Field field = analyzer.getField(fieldName, value, fieldType); + // document.add(field); + // + // if (analyzer.isFullTextField(fieldName)) { + // document.add(new Field(SUPER, value, SUPER_FIELD_TYPE)); + // } + // Set<String> subFieldNames = analyzer.getSubIndexNames(fieldName); + // if (subFieldNames != null) { + // for (String subFieldName : subFieldNames) { + // FieldType subFieldType = analyzer.getFieldType(subFieldName); + // document.add(analyzer.getField(subFieldName, value, subFieldType)); + // } + // } + // } + // return true; + // } public static String getFieldName(String columnFamily, String name) { return columnFamily + SEP + name; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-core/src/main/java/org/apache/blur/server/TableContext.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java index c3eec1d..cc2d541 100644 --- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java +++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java @@ -22,13 +22,15 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_COMMIT import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_REFRESHS; import static org.apache.blur.utils.BlurConstants.SUPER; +import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import org.apache.blur.analysis.BaseFieldManager; import org.apache.blur.analysis.FieldManager; +import org.apache.blur.analysis.HdfsFieldManager; import org.apache.blur.analysis.NoStopWordStandardAnalyzer; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; @@ -48,6 +50,7 @@ public class TableContext { private static final Log LOG = LogFactory.getLog(TableContext.class); private static final String LOGS = "logs"; + private static final String TYPES = "types"; private Path tablePath; private Path walTablePath; @@ -74,11 +77,18 @@ public class TableContext { } public static TableContext create(TableDescriptor tableDescriptor) { - TableContext tableContext = cache.get(tableDescriptor.getName()); + if (tableDescriptor == null) { + throw new NullPointerException("TableDescriptor can not be null."); + } + String name = tableDescriptor.getName(); + if (name == null) { + throw new NullPointerException("Table name in the TableDescriptor can not be null."); + } + TableContext tableContext = cache.get(name); if (tableContext != null) { return tableContext; } - LOG.info("Creating table context for table [{0}]", tableDescriptor.getName()); + LOG.info("Creating table context for table [{0}]", name); Configuration configuration = new Configuration(); Map<String, String> tableProperties = tableDescriptor.getTableProperties(); if (tableProperties != null) { @@ -93,14 +103,25 @@ public class TableContext { tableContext.walTablePath = new Path(tableContext.tablePath, LOGS); tableContext.defaultFieldName = SUPER; - tableContext.table = tableDescriptor.getName(); + tableContext.table = name; tableContext.descriptor = tableDescriptor; tableContext.timeBetweenCommits = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_COMMITS, 60000); tableContext.timeBetweenRefreshs = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_REFRESHS, 5000); tableContext.defaultPrimeDocTerm = new Term("_prime_", "true"); tableContext.defaultScoreType = ScoreType.SUPER; - tableContext.fieldManager = null; + boolean strict = tableDescriptor.isStrictTypes(); + String defaultMissingFieldType = tableDescriptor.getDefaultMissingFieldType(); + boolean defaultMissingFieldLessIndexing = tableDescriptor.isDefaultMissingFieldLessIndexing(); + Map<String, String> defaultMissingFieldProps = emptyIfNull(tableDescriptor.getDefaultMissingFieldProps()); + + Path storagePath = new Path(tableContext.tablePath, TYPES); + try { + tableContext.fieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(), storagePath, + configuration, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing, defaultMissingFieldProps); + } catch (IOException e) { + throw new RuntimeException(e); + } Class<?> c1 = configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE, KeepOnlyLastCommitDeletionPolicy.class); @@ -109,10 +130,17 @@ public class TableContext { Class<?> c2 = configuration.getClass(BLUR_SAHRD_INDEX_SIMILARITY, DefaultSimilarity.class); tableContext.similarity = (Similarity) configure(ReflectionUtils.newInstance(c2, configuration), tableContext); - cache.put(tableDescriptor.getName(), tableContext); + cache.put(name, tableContext); return tableContext; } + private static Map<String, String> emptyIfNull(Map<String, String> defaultMissingFieldProps) { + if (defaultMissingFieldProps == null) { + return new HashMap<String, String>(); + } + return defaultMissingFieldProps; + } + private static Object configure(Object o, TableContext tableContext) { if (o instanceof Configurable) { ((Configurable) o).setTableContext(tableContext); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java index bff087a..3669232 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java @@ -27,13 +27,12 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.blur.analysis.BlurAnalyzer; +import org.apache.blur.analysis.FieldManager; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.lucene.search.FairSimilarity; @@ -46,8 +45,10 @@ import org.apache.blur.mapreduce.lib.BlurOutputFormat; import org.apache.blur.mapreduce.lib.BlurRecord; import org.apache.blur.mapreduce.lib.DefaultBlurReducer; import org.apache.blur.mapreduce.lib.ProgressableDirectory; +import org.apache.blur.server.TableContext; import org.apache.blur.store.hdfs.HdfsDirectory; import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurConstants; @@ -61,6 +62,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.Reducer; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; @@ -115,7 +117,7 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { protected static final double MB = 1024 * 1024; protected IndexWriter _writer; protected Directory _directory; - protected BlurAnalyzer _analyzer; + protected Analyzer _analyzer; protected BlurTask _blurTask; protected Counter _recordCounter; @@ -133,6 +135,7 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { protected Map<String, Document> _newDocs = new HashMap<String, Document>(); protected Set<String> _recordIdsToDelete = new HashSet<String>(); protected Term _rowIdTerm = new Term(BlurConstants.ROW_ID); + private FieldManager _fieldManager; @Override protected void setup(Context context) throws IOException, InterruptedException { @@ -464,43 +467,34 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { } protected void setupAnalyzer(Context context) { - _analyzer = new BlurAnalyzer(_blurTask.getTableDescriptor().getAnalyzerDefinition()); + TableContext tableContext = TableContext.create(_blurTask.getTableDescriptor()); + _fieldManager = tableContext.getFieldManager(); + _analyzer = _fieldManager.getAnalyzerForIndex(); } - protected Document toDocument(BlurRecord record) { + protected Document toDocument(BlurRecord record) throws IOException { Document document = new Document(); document.add(new Field(BlurConstants.ROW_ID, record.getRowId(), TransactionRecorder.ID_TYPE)); document.add(new Field(BlurConstants.RECORD_ID, record.getRecordId(), TransactionRecorder.ID_TYPE)); - String columnFamily = record.getFamily(); - List<BlurColumn> columns = record.getColumns(); - final Iterator<BlurColumn> iterator = columns.iterator(); - TransactionRecorder.addColumns(document, _analyzer, columnFamily, new Iterable<Column>() { - @Override - public Iterator<Column> iterator() { - return new Iterator<Column>() { - - @Override - public Column next() { - BlurColumn bc = iterator.next(); - return new Column(bc.getName(), bc.getValue()); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public void remove() { - - } - }; - } - }); + List<Field> doc = TransactionRecorder.getDoc(_fieldManager, record.getRowId(), toRecord(record)); + for (Field field : doc) { + document.add(field); + } return document; } + private Record toRecord(BlurRecord record) { + Record r = new Record(); + r.setFamily(record.getFamily()); + r.setRecordId(record.getRecordId()); + List<BlurColumn> columns = record.getColumns(); + for (BlurColumn blurColumn : columns) { + r.addToColumns(new Column(blurColumn.getName(), blurColumn.getValue())); + } + return r; + } + protected static void report(Context context, long totalBytesCopied, long totalBytesToCopy, long startTime, String src) { long now = System.currentTimeMillis(); double seconds = (now - startTime) / 1000.0; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java index ef38f48..78b0d0f 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java @@ -26,11 +26,13 @@ import java.util.Map; import java.util.TreeMap; import java.util.UUID; +import org.apache.blur.analysis.FieldManager; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.lucene.LuceneVersionConstant; import org.apache.blur.manager.writer.TransactionRecorder; import org.apache.blur.mapreduce.lib.BlurMutate.MUTATE_TYPE; +import org.apache.blur.server.TableContext; import org.apache.blur.store.hdfs.HdfsDirectory; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TJSONProtocol; @@ -55,7 +57,9 @@ import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.util.Progressable; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.StringField; import org.apache.lucene.index.CorruptIndexException; @@ -343,9 +347,9 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { private static final Log LOG = LogFactory.getLog(BlurRecordWriter.class); private final Text _prevKey = new Text(); - private final Map<String, Document> _documents = new TreeMap<String, Document>(); + private final Map<String, List<Field>> _documents = new TreeMap<String, List<Field>>(); private final IndexWriter _writer; - private final BlurAnalyzer _analyzer; + private final FieldManager _fieldManager; private final Directory _finalDir; private final Directory _localDir; private final File _localPath; @@ -390,13 +394,16 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { _finalDir = new ProgressableDirectory(new HdfsDirectory(configuration, _newIndex), BlurOutputFormat.getProgressable()); _finalDir.setLockFactory(NoLockFactory.getNoLockFactory()); - - _analyzer = new BlurAnalyzer(tableDescriptor.getAnalyzerDefinition()); - _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, _analyzer); + + TableContext tableContext = TableContext.create(tableDescriptor); + _fieldManager = tableContext.getFieldManager(); + Analyzer analyzer = _fieldManager.getAnalyzerForIndex(); + + _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer); TieredMergePolicy mergePolicy = (TieredMergePolicy) _conf.getMergePolicy(); mergePolicy.setUseCompoundFile(false); - _overFlowConf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, _analyzer); + _overFlowConf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer); _overFlowConf.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES); if (_indexLocally) { @@ -452,12 +459,12 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { return; } _columnCount.increment(record.getColumns().size()); - Document document = TransactionRecorder.convert(blurRecord.getRowId(), record, _analyzer); - Document dup = _documents.put(recordId, document); + List<Field> document = TransactionRecorder.getDoc(_fieldManager,blurRecord.getRowId(), record); + List<Field> dup = _documents.put(recordId, document); if (dup != null) { _recordDuplicateCount.increment(1); } else { - _fieldCount.increment(document.getFields().size()); + _fieldCount.increment(document.size()); _recordCount.increment(1); } flushToTmpIndexIfNeeded(); @@ -481,7 +488,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { _localTmpWriter = new IndexWriter(_localTmpDir, _overFlowConf.clone()); //The local tmp writer has merging disabled so the first document in is going to be doc 0. //Therefore the first document added is the prime doc - List<Document> docs = new ArrayList<Document>(_documents.values()); + List<List<Field>> docs = new ArrayList<List<Field>>(_documents.values()); docs.get(0).add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); _localTmpWriter.addDocuments(docs); } else { @@ -527,7 +534,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { _rowDeleteCount.increment(1); } } else { - List<Document> docs = new ArrayList<Document>(_documents.values()); + List<List<Field>> docs = new ArrayList<List<Field>>(_documents.values()); docs.get(0).add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); _writer.addDocuments(docs); _recordRateCounter.mark(_documents.size()); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java index 67135cb..1416daf 100644 --- a/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java +++ b/blur-mapred/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java @@ -30,9 +30,9 @@ import java.io.PrintWriter; import java.util.Collection; import java.util.TreeSet; +import org.apache.blur.server.TableContext; import org.apache.blur.store.buffer.BufferStore; import org.apache.blur.store.hdfs.HdfsDirectory; -import org.apache.blur.thrift.generated.AnalyzerDefinition; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurUtil; import org.apache.hadoop.conf.Configuration; @@ -47,6 +47,7 @@ import org.apache.hadoop.mapreduce.TestMapReduceLocal.TrackingTextInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.lucene.index.DirectoryReader; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -61,7 +62,7 @@ public class BlurOutputFormatTest { private Path inDir = new Path(TEST_ROOT_DIR + "/in"); @BeforeClass - public static void setup() throws Exception { + public static void setupTest() throws Exception { System.setProperty("test.build.data", "./target/BlurOutputFormatTest/data"); TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "target/tmp/BlurOutputFormatTest_tmp")); System.setProperty("hadoop.log.dir", "./target/BlurOutputFormatTest/hadoop_log"); @@ -95,6 +96,11 @@ public class BlurOutputFormatTest { file.delete(); } + @Before + public void setup() { + TableContext.clear(); + } + @Test public void testBlurOutputFormat() throws IOException, InterruptedException, ClassNotFoundException { localFs.delete(inDir, true); @@ -113,8 +119,8 @@ public class BlurOutputFormatTest { TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setShardCount(1); - tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition()); tableDescriptor.setTableUri(tableUri); + tableDescriptor.setName("test"); createShardDirectories(outDir, 1); @@ -164,8 +170,8 @@ public class BlurOutputFormatTest { TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setShardCount(1); - tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition()); tableDescriptor.setTableUri(tableUri); + tableDescriptor.setName("test"); createShardDirectories(outDir, 1); @@ -206,8 +212,8 @@ public class BlurOutputFormatTest { TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setShardCount(2); - tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition()); tableDescriptor.setTableUri(tableUri); + tableDescriptor.setName("test"); createShardDirectories(outDir, 2); @@ -252,8 +258,8 @@ public class BlurOutputFormatTest { TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setShardCount(7); - tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition()); tableDescriptor.setTableUri(tableUri); + tableDescriptor.setName("test"); createShardDirectories(outDir, 7); @@ -299,8 +305,8 @@ public class BlurOutputFormatTest { TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setShardCount(1); - tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition()); tableDescriptor.setTableUri(tableUri); + tableDescriptor.setName("test"); createShardDirectories(outDir, 1); @@ -332,7 +338,6 @@ public class BlurOutputFormatTest { TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setShardCount(2); - tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition()); tableDescriptor.setTableUri(tableUri); createShardDirectories(outDir, 2); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java index 0404f2c..1b2ef0c 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java @@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.blur.analysis.type.DoubleFieldTypeDefinition; +import org.apache.blur.analysis.type.FieldLessFieldTypeDefinition; import org.apache.blur.analysis.type.FloatFieldTypeDefinition; import org.apache.blur.analysis.type.IntFieldTypeDefinition; import org.apache.blur.analysis.type.LongFieldTypeDefinition; @@ -60,6 +61,11 @@ public abstract class BaseFieldManager extends FieldManager { private final Analyzer _baseAnalyzerForQuery; private final Analyzer _baseAnalyzerForIndex; private final String _fieldLessField; + private final Map<String, String> _defaultMissingFieldProps; + private final String _defaultMissingFieldType; + private final boolean _defaultMissingFieldLessIndexing; + private final boolean _strict; + private final FieldTypeDefinition _fieldLessFieldTypeDefinition; public static FieldType ID_TYPE; static { @@ -76,7 +82,13 @@ public abstract class BaseFieldManager extends FieldManager { SUPER_FIELD_TYPE.setOmitNorms(true); } - public BaseFieldManager(String fieldLessField, final Analyzer defaultAnalyzerForQuerying) { + public BaseFieldManager(String fieldLessField, final Analyzer defaultAnalyzerForQuerying) throws IOException { + this(fieldLessField, defaultAnalyzerForQuerying, true, null, false, null); + } + + public BaseFieldManager(String fieldLessField, final Analyzer defaultAnalyzerForQuerying, boolean strict, + String defaultMissingFieldType, boolean defaultMissingFieldLessIndexing, + Map<String, String> defaultMissingFieldProps) throws IOException { _typeMap.put(TextFieldTypeDefinition.NAME, TextFieldTypeDefinition.class); _typeMap.put(StringFieldTypeDefinition.NAME, StringFieldTypeDefinition.class); _typeMap.put(StoredFieldTypeDefinition.NAME, StoredFieldTypeDefinition.class); @@ -85,6 +97,12 @@ public abstract class BaseFieldManager extends FieldManager { _typeMap.put(DoubleFieldTypeDefinition.NAME, DoubleFieldTypeDefinition.class); _typeMap.put(FloatFieldTypeDefinition.NAME, FloatFieldTypeDefinition.class); _fieldLessField = fieldLessField; + _strict = strict; + _defaultMissingFieldLessIndexing = defaultMissingFieldLessIndexing; + _defaultMissingFieldType = defaultMissingFieldType; + _defaultMissingFieldProps = defaultMissingFieldProps; + + _fieldLessFieldTypeDefinition = new FieldLessFieldTypeDefinition(); _baseAnalyzerForQuery = new AnalyzerWrapper() { @Override @@ -141,11 +159,22 @@ public abstract class BaseFieldManager extends FieldManager { if (value == null || name == null) { continue; } - getAndAddFields(fields, family, column, getFieldTypeDefinition(family, column)); + FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(family, column); + if (fieldTypeDefinition == null) { + if (isStrict()) { + LOG.error("Family [{0}] Column [{1}] not defined", family, column); + throw new IOException("Family [" + family + "] Column [" + column + "] not defined"); + } + addColumnDefinition(family, name, null, getDefaultMissingFieldLessIndexing(), getDefaultMissingFieldType(), + getDefaultMissingFieldProps()); + fieldTypeDefinition = getFieldTypeDefinition(family, column); + } + getAndAddFields(fields, family, column, fieldTypeDefinition); Collection<String> subColumns = getSubColumns(family, column); if (subColumns != null) { for (String subName : subColumns) { - getAndAddFields(fields, family, column, subName, getFieldTypeDefinition(family, column, subName)); + FieldTypeDefinition subFieldTypeDefinition = getFieldTypeDefinition(family, column, subName); + getAndAddFields(fields, family, column, subName, subFieldTypeDefinition); } } } @@ -235,6 +264,11 @@ public abstract class BaseFieldManager extends FieldManager { } else { fieldName = baseFieldName; } + return addFieldTypeDefinition(fieldName, fieldLessIndexing, fieldType, props); + } + + private boolean addFieldTypeDefinition(String fieldName, boolean fieldLessIndexing, String fieldType, + Map<String, String> props) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(fieldName); if (fieldTypeDefinition != null) { return false; @@ -277,6 +311,9 @@ public abstract class BaseFieldManager extends FieldManager { protected FieldTypeDefinition newFieldTypeDefinition(boolean fieldLessIndexing, String fieldType, Map<String, String> props) { + if (fieldType == null) { + throw new IllegalArgumentException("Field type can not be null."); + } Class<? extends FieldTypeDefinition> clazz = _typeMap.get(fieldType); if (clazz == null) { throw new IllegalArgumentException("FieldType of [" + fieldType + "] was not found."); @@ -378,24 +415,36 @@ public abstract class BaseFieldManager extends FieldManager { @Override public boolean isFieldLessIndexed(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); + if (fieldTypeDefinition == null) { + throw new IOException("FieldTypeDefinition for field [" + field + "] is missing."); + } return fieldTypeDefinition.isFieldLessIndexed(); } @Override public boolean checkSupportForFuzzyQuery(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); + if (fieldTypeDefinition == null) { + throw new IOException("FieldTypeDefinition for field [" + field + "] is missing."); + } return fieldTypeDefinition.checkSupportForFuzzyQuery(); } @Override public boolean checkSupportForPrefixQuery(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); + if (fieldTypeDefinition == null) { + throw new IOException("FieldTypeDefinition for field [" + field + "] is missing."); + } return fieldTypeDefinition.checkSupportForPrefixQuery(); } @Override public boolean checkSupportForWildcardQuery(String field) throws IOException { FieldTypeDefinition fieldTypeDefinition = getFieldTypeDefinition(field); + if (fieldTypeDefinition == null) { + throw new IOException("FieldTypeDefinition for field [" + field + "] is missing."); + } return fieldTypeDefinition.checkSupportForWildcardQuery(); } @@ -422,6 +471,9 @@ public abstract class BaseFieldManager extends FieldManager { @Override public FieldTypeDefinition getFieldTypeDefinition(String field) throws IOException { + if (field.equals(_fieldLessField)) { + return _fieldLessFieldTypeDefinition; + } FieldTypeDefinition fieldTypeDefinition = _fieldNameToDefMap.get(field); if (fieldTypeDefinition == null) { tryToLoad(field); @@ -430,8 +482,29 @@ public abstract class BaseFieldManager extends FieldManager { return fieldTypeDefinition; } + @Override public String getFieldLessFieldName() { return _fieldLessField; } + @Override + public Map<String, String> getDefaultMissingFieldProps() { + return _defaultMissingFieldProps; + } + + @Override + public String getDefaultMissingFieldType() { + return _defaultMissingFieldType; + } + + @Override + public boolean getDefaultMissingFieldLessIndexing() { + return _defaultMissingFieldLessIndexing; + } + + @Override + public boolean isStrict() { + return _strict; + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java index 2271726..519e397 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java @@ -33,7 +33,7 @@ public abstract class FieldManager { * * @param record * @return - * @throws IOException + * @throws IOException */ public abstract List<Field> getFields(String rowId, Record record) throws IOException; @@ -54,8 +54,8 @@ public abstract class FieldManager { * the field type name, required. * @param props * the configuration properties for this column and type. - * @return - * @throws IOException + * @return + * @throws IOException */ public abstract boolean addColumnDefinition(String family, String columnName, String subColumnName, boolean fieldLessIndexing, String fieldType, Map<String, String> props) throws IOException; @@ -66,7 +66,7 @@ public abstract class FieldManager { * @param fieldName * the Lucene field name. * @return {@link Analyzer}. - * @throws IOException + * @throws IOException */ public abstract Analyzer getAnalyzerForIndex(String fieldName) throws IOException; @@ -76,7 +76,7 @@ public abstract class FieldManager { * @param fieldName * the Lucene field name. * @return {@link Analyzer}. - * @throws IOException + * @throws IOException */ public abstract Analyzer getAnalyzerForQuery(String fieldName) throws IOException; @@ -89,7 +89,7 @@ public abstract class FieldManager { * @param columnName * the column name. * @return boolean - * @throws IOException + * @throws IOException */ public abstract boolean isValidColumnDefinition(String family, String columnName) throws IOException; @@ -106,7 +106,7 @@ public abstract class FieldManager { * @param field * the field name. * @return boolean - * @throws IOException + * @throws IOException */ public abstract boolean checkSupportForFuzzyQuery(String field) throws IOException; @@ -116,7 +116,7 @@ public abstract class FieldManager { * @param field * the field name. * @return boolean - * @throws IOException + * @throws IOException */ public abstract boolean checkSupportForPrefixQuery(String field) throws IOException; @@ -126,7 +126,7 @@ public abstract class FieldManager { * @param field * the field name. * @return boolean - * @throws IOException + * @throws IOException */ public abstract boolean checkSupportForWildcardQuery(String field) throws IOException; @@ -145,7 +145,7 @@ public abstract class FieldManager { * @param endInclusive * if the end is inclusive. * @return the new range query or null. - * @throws IOException + * @throws IOException */ public abstract Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) throws IOException; @@ -159,7 +159,7 @@ public abstract class FieldManager { * @param text * the text for the term. * @return the query or null. - * @throws IOException + * @throws IOException */ public abstract Query getTermQueryIfNumeric(String field, String text) throws IOException; @@ -169,14 +169,63 @@ public abstract class FieldManager { * @param field * the field name. * @return the {@link FieldTypeDefinition} or null if missing. - * @throws IOException + * @throws IOException */ public abstract FieldTypeDefinition getFieldTypeDefinition(String field) throws IOException; + /** + * Checks to see if the field should also be indexed in the field less field. + * + * @param name + * the field name. + * @return boolean + * @throws IOException + */ public abstract boolean isFieldLessIndexed(String name) throws IOException; + /** + * Gets the analyzer used for indexing. + * + * @return the {@link Analyzer}. + */ public abstract Analyzer getAnalyzerForIndex(); - + + /** + * Gets the Lucene field name of the field that is used for queries that do + * not specify a field. + * + * @return the field name. + */ public abstract String getFieldLessFieldName(); + /** + * The field properties used if the table is is not in strict mode. + * + * @return properties. + */ + public abstract Map<String, String> getDefaultMissingFieldProps(); + + /** + * The field type used if the table is is not in strict mode. + * + * @return field type. + */ + public abstract String getDefaultMissingFieldType(); + + /** + * Should the field be placed in the field less indexing if the table is is + * not in strict mode. + * + * @return boolean. + */ + public abstract boolean getDefaultMissingFieldLessIndexing(); + + /** + * Does the table have strict field names and types. If so then if a new + * Column is attempted to be used for indexing then an error will be raised. + * + * @return boolean + */ + public abstract boolean isStrict(); + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java index 9669e2e..9ada9f9 100644 --- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java +++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; @@ -33,10 +34,30 @@ import org.apache.lucene.analysis.Analyzer; public class HdfsFieldManager extends BaseFieldManager { + public static abstract class Lock { + + public abstract void lock(); + + public abstract void unlock(); + + } + + private static final Log LOG = LogFactory.getLog(HdfsFieldManager.class); private static final String FIELD_TYPE = "fieldType"; private static final String FIELD_LESS_INDEXING = "fieldLessIndexing"; + private static Lock _lock = new Lock() { + private final java.util.concurrent.locks.Lock _javalock = new ReentrantReadWriteLock().writeLock(); - private static final Log LOG = LogFactory.getLog(HdfsFieldManager.class); + @Override + public void lock() { + _javalock.lock(); + } + + @Override + public void unlock() { + _javalock.unlock(); + } + }; private final Configuration _configuration; private final Path _storagePath; @@ -44,7 +65,14 @@ public class HdfsFieldManager extends BaseFieldManager { public HdfsFieldManager(String fieldLessField, Analyzer defaultAnalyzerForQuerying, Path storagePath, Configuration configuration) throws IOException { - super(fieldLessField, defaultAnalyzerForQuerying); + this(fieldLessField, defaultAnalyzerForQuerying, storagePath, configuration, true, null, false, null); + } + + public HdfsFieldManager(String fieldLessField, Analyzer defaultAnalyzerForQuerying, Path storagePath, + Configuration configuration, boolean strict, String defaultMissingFieldType, + boolean defaultMissingFieldLessIndexing, Map<String, String> defaultMissingFieldProps) throws IOException { + super(fieldLessField, defaultAnalyzerForQuerying, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing, + defaultMissingFieldProps); _storagePath = storagePath; _configuration = configuration; _fileSystem = _storagePath.getFileSystem(_configuration); @@ -53,25 +81,40 @@ public class HdfsFieldManager extends BaseFieldManager { @Override protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props) throws IOException { - LOG.info("Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]", - fieldName, fieldLessIndexing, fieldType, props); - Properties properties = new Properties(); - properties.setProperty(FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexing)); - properties.setProperty(FIELD_TYPE, fieldType); - if (props != null) { - for (Entry<String, String> e : props.entrySet()) { - properties.put(e.getKey(), e.getValue()); + // Might want to make this a ZK lock + _lock.lock(); + try { + LOG.info("Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]", + fieldName, fieldLessIndexing, fieldType, props); + Properties properties = new Properties(); + properties.setProperty(FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexing)); + properties.setProperty(FIELD_TYPE, fieldType); + if (props != null) { + for (Entry<String, String> e : props.entrySet()) { + properties.setProperty(e.getKey(), e.getValue()); + } } + Path path = getFieldPath(fieldName); + if (_fileSystem.exists(path)) { + LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props); + return false; + } + try { + FSDataOutputStream outputStream = _fileSystem.create(path, false); + properties.store(outputStream, getComments()); + outputStream.close(); + } catch (IOException e) { + if (_fileSystem.exists(path)) { + LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props); + return false; + } else { + throw e; + } + } + return true; + } finally { + _lock.unlock(); } - Path path = getFieldPath(fieldName); - if (_fileSystem.exists(path)) { - LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props); - return false; - } - FSDataOutputStream outputStream = _fileSystem.create(path, false); - properties.store(outputStream, getComments()); - outputStream.close(); - return true; } private Path getFieldPath(String fieldName) { @@ -105,4 +148,12 @@ public class HdfsFieldManager extends BaseFieldManager { } return result; } + + public static Lock getLock() { + return _lock; + } + + public static void setLock(Lock lock) { + _lock = lock; + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java new file mode 100644 index 0000000..ccbe4f5 --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/analysis/ZooKeeperFieldManager.java @@ -0,0 +1,157 @@ +package org.apache.blur.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.lucene.analysis.Analyzer; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.Stat; + +public class ZooKeeperFieldManager extends BaseFieldManager { + + private static final String FIELD_TYPE = "fieldType"; + private static final String FIELD_LESS_INDEXING = "fieldLessIndexing"; + private static final Object lock = new Object(); + + private static final Log LOG = LogFactory.getLog(ZooKeeperFieldManager.class); + private final ZooKeeper _zooKeeper; + private final String _basePath; + + public ZooKeeperFieldManager(String fieldLessField, Analyzer defaultAnalyzerForQuerying, ZooKeeper zooKeeper, String basePath) + throws IOException { + this(fieldLessField, defaultAnalyzerForQuerying, zooKeeper, basePath, true, null, false, null); + } + + public ZooKeeperFieldManager(String fieldLessField, Analyzer defaultAnalyzerForQuerying, ZooKeeper zooKeeper, + String basePath, boolean strict, String defaultMissingFieldType, boolean defaultMissingFieldLessIndexing, + Map<String, String> defaultMissingFieldProps) throws IOException { + super(fieldLessField, defaultAnalyzerForQuerying, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing, + defaultMissingFieldProps); + _zooKeeper = zooKeeper; + _basePath = basePath; + } + + @Override + protected boolean tryToStore(String fieldName, boolean fieldLessIndexing, String fieldType, Map<String, String> props) + throws IOException { + // Might want to make this a ZK lock + synchronized (lock) { + LOG.info("Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]", + fieldName, fieldLessIndexing, fieldType, props); + Properties properties = new Properties(); + properties.setProperty(FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexing)); + properties.setProperty(FIELD_TYPE, fieldType); + if (props != null) { + for (Entry<String, String> e : props.entrySet()) { + properties.setProperty(e.getKey(), e.getValue()); + } + } + String path = getFieldPath(fieldName); + Stat stat; + try { + stat = _zooKeeper.exists(path, false); + } catch (KeeperException e) { + throw new IOException(e); + } catch (InterruptedException e) { + throw new IOException(e); + } + if (stat != null) { + LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props); + return false; + } + try { + _zooKeeper.create(path, toBytes(properties), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + return true; + } catch (KeeperException e) { + if (e.code() == KeeperException.Code.NODEEXISTS) { + LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexing, fieldType, props); + return false; + } + throw new IOException(e); + } catch (InterruptedException e) { + throw new IOException(e); + } + } + } + + private byte[] toBytes(Properties properties) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + properties.store(outputStream, getComments()); + outputStream.close(); + return outputStream.toByteArray(); + } + + private String getFieldPath(String fieldName) { + return _basePath + "/" + fieldName; + } + + private String getComments() { + return "This file is generated from Apache Blur to store meta data about field types. DO NOT MODIFIY!"; + } + + @Override + protected void tryToLoad(String fieldName) throws IOException { + String path = getFieldPath(fieldName); + Stat stat; + try { + stat = _zooKeeper.exists(path, false); + } catch (KeeperException e) { + throw new IOException(e); + } catch (InterruptedException e) { + throw new IOException(e); + } + if (stat != null) { + return; + } + byte[] data; + try { + data = _zooKeeper.getData(path, false, stat); + } catch (KeeperException e) { + throw new IOException(e); + } catch (InterruptedException e) { + throw new IOException(e); + } + Properties props = new Properties(); + ByteArrayInputStream inputStream = new ByteArrayInputStream(data); + props.load(inputStream); + inputStream.close(); + boolean fieldLessIndexing = Boolean.parseBoolean(props.getProperty(FIELD_LESS_INDEXING)); + String fieldType = props.getProperty(FIELD_TYPE); + FieldTypeDefinition fieldTypeDefinition = newFieldTypeDefinition(fieldLessIndexing, fieldType, toMap(props)); + registerFieldTypeDefinition(fieldName, fieldTypeDefinition); + } + + private Map<String, String> toMap(Properties props) { + Map<String, String> result = new HashMap<String, String>(); + for (Entry<Object, Object> e : props.entrySet()) { + result.put(e.getKey().toString(), e.getValue().toString()); + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2378815f/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java new file mode 100644 index 0000000..f6ca6f6 --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/analysis/type/FieldLessFieldTypeDefinition.java @@ -0,0 +1,104 @@ +package org.apache.blur.analysis.type; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.util.Map; + +import org.apache.blur.analysis.FieldTypeDefinition; +import org.apache.blur.analysis.NoStopWordStandardAnalyzer; +import org.apache.blur.thrift.generated.Column; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.TextField; + +public class FieldLessFieldTypeDefinition extends FieldTypeDefinition { + + public static final String NAME = "fieldless"; + public static final FieldType TYPE_NOT_STORED; + + static { + TYPE_NOT_STORED = new FieldType(TextField.TYPE_NOT_STORED); + TYPE_NOT_STORED.setOmitNorms(true); + TYPE_NOT_STORED.freeze(); + } + + @Override + public String getName() { + return NAME; + } + + @Override + public void configure(Map<String, String> properties) { + + } + + @Override + public Iterable<? extends Field> getFieldsForColumn(String family, Column column) { + String name = getName(family, column.getName()); + Field field = new Field(name, column.getValue(), getStoredFieldType()); + return makeIterable(field); + } + + @Override + public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column, String subName) { + String name = getName(family, column.getName(), subName); + Field field = new Field(name, column.getValue(), getNotStoredFieldType()); + return makeIterable(field); + } + + @Override + public FieldType getStoredFieldType() { + return TYPE_NOT_STORED; + } + + @Override + public FieldType getNotStoredFieldType() { + return TYPE_NOT_STORED; + } + + @Override + public Analyzer getAnalyzerForIndex() { + return new NoStopWordStandardAnalyzer(); + } + + @Override + public Analyzer getAnalyzerForQuery() { + return new NoStopWordStandardAnalyzer(); + } + + @Override + public boolean checkSupportForFuzzyQuery() { + return true; + } + + @Override + public boolean checkSupportForWildcardQuery() { + return true; + } + + @Override + public boolean checkSupportForPrefixQuery() { + return true; + } + + @Override + public boolean isNumeric() { + return false; + } + +}
