Updated Branches: refs/heads/master f04297024 -> afbe3ef35
Adding the super indexing back in, now to fix the highlighting. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/afbe3ef3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/afbe3ef3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/afbe3ef3 Branch: refs/heads/master Commit: afbe3ef3555a80674c08a5419ff5b05b0332475e Parents: f042970 Author: Aaron McCurry <[email protected]> Authored: Tue Jun 18 22:36:08 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Jun 18 22:36:08 2013 -0400 ---------------------------------------------------------------------- .../manager/writer/TransactionRecorder.java | 28 ++++++++------------ .../blur/manager/writer/IndexImporterTest.java | 4 +-- .../manager/writer/TransactionRecorderTest.java | 6 ++--- .../org/apache/blur/mapreduce/BlurReducer.java | 7 +++-- .../blur/mapreduce/lib/BlurOutputFormat.java | 3 +-- 5 files changed, 20 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/afbe3ef3/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java index b7453ba..8130fe7 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java @@ -17,6 +17,7 @@ package org.apache.blur.manager.writer; * limitations under the License. */ import static org.apache.blur.utils.BlurConstants.SEP; +import static org.apache.blur.utils.BlurConstants.SUPER; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -56,6 +57,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; import org.apache.lucene.index.BlurIndexWriter; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.Term; @@ -356,10 +358,9 @@ public class TransactionRecorder extends TimerTask implements Closeable { List<Record> records = row.records; int size = records.size(); final String rowId = row.id; - final StringBuilder builder = new StringBuilder(); List<Document> docs = new ArrayList<Document>(size); for (int i = 0; i < size; i++) { - Document document = convert(rowId, records.get(i), builder, analyzer); + Document document = convert(rowId, records.get(i), analyzer); if (i == 0) { document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); } @@ -368,13 +369,13 @@ public class TransactionRecorder extends TimerTask implements Closeable { return docs; } - public static Document convert(String rowId, Record record, StringBuilder builder, BlurAnalyzer analyzer) { + public static Document convert(String rowId, Record record, BlurAnalyzer analyzer) { BlurUtil.validateRowIdAndRecord(rowId, record); Document document = new Document(); document.add(new Field(BlurConstants.ROW_ID, rowId, ID_TYPE)); document.add(new Field(BlurConstants.RECORD_ID, record.recordId, ID_TYPE)); document.add(new Field(BlurConstants.FAMILY, record.family, ID_TYPE)); - addColumns(document, analyzer, builder, record.family, record.columns); + addColumns(document, analyzer, record.family, record.columns); return document; } @@ -403,13 +404,11 @@ public class TransactionRecorder extends TimerTask implements Closeable { } } } - - public static boolean addColumns(Document document, BlurAnalyzer analyzer, StringBuilder builder, - String columnFamily, Iterable<Column> set) { + + public static boolean addColumns(Document document, BlurAnalyzer analyzer, String columnFamily, Iterable<Column> set) { if (set == null) { return false; } - builder.setLength(0); OUTER: for (Column column : set) { String name = column.getName(); String value = column.value; @@ -420,11 +419,10 @@ public class TransactionRecorder extends TimerTask implements Closeable { FieldType fieldType = analyzer.getFieldType(fieldName); Field field = analyzer.getField(fieldName, value, fieldType); document.add(field); - - // @TODO remove full text stuff -// if (analyzer.isFullTextField(fieldName)) { -// builder.append(value).append(' '); -// } + + if (analyzer.isFullTextField(fieldName)) { + document.add(new Field(SUPER, value, TextField.TYPE_NOT_STORED)); + } Set<String> subFieldNames = analyzer.getSubIndexNames(fieldName); if (subFieldNames != null) { for (String subFieldName : subFieldNames) { @@ -433,10 +431,6 @@ public class TransactionRecorder extends TimerTask implements Closeable { } } } -// if (builder.length() != 0) { -// String superValue = builder.toString(); -// document.add(new Field(SUPER, superValue, Store.NO, Index.ANALYZED_NO_NORMS)); -// } return true; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/afbe3ef3/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java index 5d3466e..a3e1ad9 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java @@ -113,7 +113,7 @@ public class IndexImporterTest { @Test public void testIndexImporterWithCorrectRowIdShardCombination() throws IOException { - Document document = TransactionRecorder.convert("1", genRecord("1"), new StringBuilder(), new BlurAnalyzer()); + Document document = TransactionRecorder.convert("1", genRecord("1"), new BlurAnalyzer()); commitWriter.addDocument(document); commitWriter.commit(); commitWriter.close(); @@ -138,7 +138,7 @@ public class IndexImporterTest { @Test public void testIndexImporterWithWrongRowIdShardCombination() throws IOException { setupWriter(configuration); - Document document = TransactionRecorder.convert("2", genRecord("1"), new StringBuilder(), new BlurAnalyzer()); + Document document = TransactionRecorder.convert("2", genRecord("1"), new BlurAnalyzer()); commitWriter.addDocument(document); commitWriter.commit(); commitWriter.close(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/afbe3ef3/blur-core/src/test/java/org/apache/blur/manager/writer/TransactionRecorderTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/TransactionRecorderTest.java b/blur-core/src/test/java/org/apache/blur/manager/writer/TransactionRecorderTest.java index 17870d1..3c152da 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/writer/TransactionRecorderTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/writer/TransactionRecorderTest.java @@ -136,7 +136,7 @@ public class TransactionRecorderTest { column.setName("columnName_123-1"); record.setColumns(Arrays.asList(column)); - TransactionRecorder.convert(rowId, record, new StringBuilder(), new BlurAnalyzer()); + TransactionRecorder.convert(rowId, record, new BlurAnalyzer()); assert(true); } @@ -151,7 +151,7 @@ public class TransactionRecorderTest { column.setName("columnName_123-1"); record.setColumns(Arrays.asList(column)); - TransactionRecorder.convert(rowId, record, new StringBuilder(), new BlurAnalyzer()); + TransactionRecorder.convert(rowId, record, new BlurAnalyzer()); fail(); } @@ -166,7 +166,7 @@ public class TransactionRecorderTest { column.setName("columnName_123.1"); record.setColumns(Arrays.asList(column)); - TransactionRecorder.convert(rowId, record, new StringBuilder(), new BlurAnalyzer()); + TransactionRecorder.convert(rowId, record, new BlurAnalyzer()); fail(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/afbe3ef3/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java index f694c99..4edfb6b 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java @@ -124,7 +124,6 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { protected Counter _fieldCounter; protected Counter _rowBreak; protected Counter _rowFailures; - protected StringBuilder _builder = new StringBuilder(); protected byte[] _copyBuf; protected Configuration _configuration; protected long _start; @@ -184,7 +183,7 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { _recordIdsToDelete.add(record.getRecordId()); continue; } - Document document = toDocument(record, _builder); + Document document = toDocument(record); _newDocs.put(record.getRecordId(), document); context.progress(); @@ -469,13 +468,13 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { _analyzer = new BlurAnalyzer(_blurTask.getTableDescriptor().getAnalyzerDefinition()); } - protected Document toDocument(BlurRecord record, StringBuilder builder) { + protected Document toDocument(BlurRecord record) { Document document = new Document(); document.add(new Field(BlurConstants.ROW_ID, record.getRowId(), TransactionRecorder.ID_TYPE)); document.add(new Field(BlurConstants.RECORD_ID, record.getRecordId(), TransactionRecorder.ID_TYPE)); String columnFamily = record.getFamily(); - TransactionRecorder.addColumns(document, _analyzer, builder, columnFamily, new IterableConverter<BlurColumn, Column>( + TransactionRecorder.addColumns(document, _analyzer, columnFamily, new IterableConverter<BlurColumn, Column>( record.getColumns(), new Converter<BlurColumn, Column>() { @Override public Column convert(BlurColumn from) throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/afbe3ef3/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java index 174c477..f8190b2 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java @@ -344,7 +344,6 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { private final Map<String, Document> _documents = new TreeMap<String, Document>(); private final IndexWriter _writer; private final BlurAnalyzer _analyzer; - private final StringBuilder _builder = new StringBuilder(); private final Directory _finalDir; private final Directory _localDir; private final File _localPath; @@ -440,7 +439,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { _deletedRowId = blurRecord.getRowId(); return; } - Document document = TransactionRecorder.convert(blurRecord.getRowId(), record, _builder, _analyzer); + Document document = TransactionRecorder.convert(blurRecord.getRowId(), record, _analyzer); if (_documents.size() == 0) { document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); }
