Updated Branches: refs/heads/apache-blur-0.2 d01e009b3 -> 2f86c6288
Adding a new blur codec to allow for configuring the chunk and compression of the stored fields files. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/0e6d2d4c Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/0e6d2d4c Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/0e6d2d4c Branch: refs/heads/apache-blur-0.2 Commit: 0e6d2d4c821a29c82fee8b16d9cc5ffbfb062353 Parents: d01e009 Author: Aaron McCurry <[email protected]> Authored: Mon Dec 9 21:31:22 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Mon Dec 9 21:31:22 2013 -0500 ---------------------------------------------------------------------- .../blur/manager/writer/BlurIndexReader.java | 4 +- .../blur/manager/writer/BlurNRTIndex.java | 4 +- .../blur/utils/TableShardCountCollapser.java | 3 +- .../apache/blur/manager/IndexManagerTest.java | 6 +- .../org/apache/blur/mapreduce/BlurReducer.java | 6 +- .../blur/mapreduce/lib/BlurOutputFormat.java | 3 +- .../org/apache/blur/filter/FilterCacheTest.java | 8 +- .../apache/blur/lucene/codec/Blur021Codec.java | 3 +- .../lucene/codec/Blur021StoredFieldsFormat.java | 25 ---- .../apache/blur/lucene/codec/Blur022Codec.java | 130 +++++++++++++++++++ .../lucene/codec/Blur022SegmentInfoFormat.java | 50 +++++++ .../lucene/codec/Blur022SegmentInfoReader.java | 76 +++++++++++ .../lucene/codec/Blur022SegmentInfoWriter.java | 80 ++++++++++++ .../lucene/codec/Blur022StoredFieldsFormat.java | 107 +++++++++++++++ .../blur/lucene/codec/CachedDecompressor.java | 71 ++++++++++ .../lucene/codec/CachingCompressionMode.java | 95 -------------- .../org/apache/blur/lucene/codec/Testing.java | 37 ------ .../services/org.apache.lucene.codecs.Codec | 3 +- 18 files changed, 536 insertions(+), 175 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java index a4653a5..13effa3 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java @@ -26,7 +26,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.lucene.codec.Blur021Codec; +import org.apache.blur.lucene.codec.Blur022Codec; import org.apache.blur.lucene.warmup.TraceableDirectory; import org.apache.blur.server.IndexSearcherClosable; import org.apache.blur.server.ShardContext; @@ -67,7 +67,7 @@ public class BlurIndexReader extends BlurIndex { // if the directory is empty then create an empty index. IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer()); conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5)); - conf.setCodec(new Blur021Codec()); + conf.setCodec(new Blur022Codec()); new BlurIndexWriter(directory, conf).close(); } _indexReaderRef.set(DirectoryReader.open(directory)); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java index f246416..3909e5a 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java @@ -38,7 +38,7 @@ import org.apache.blur.analysis.FieldManager; import org.apache.blur.index.ExitableReader; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.lucene.codec.Blur021Codec; +import org.apache.blur.lucene.codec.Blur022Codec; import org.apache.blur.lucene.store.refcounter.DirectoryReferenceCounter; import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC; import org.apache.blur.lucene.warmup.TraceableDirectory; @@ -105,7 +105,7 @@ public class BlurNRTIndex extends BlurIndex { Analyzer analyzer = fieldManager.getAnalyzerForIndex(); IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, analyzer); conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5)); - conf.setCodec(new Blur021Codec()); + conf.setCodec(new Blur022Codec()); conf.setSimilarity(_tableContext.getSimilarity()); AtomicBoolean stop = new AtomicBoolean(); conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, stop, _isClosed)); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java b/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java index 90deabe..1e91baf 100644 --- a/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java +++ b/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java @@ -25,6 +25,7 @@ import java.util.SortedSet; import java.util.TreeSet; import org.apache.blur.lucene.codec.Blur021Codec; +import org.apache.blur.lucene.codec.Blur022Codec; import org.apache.blur.store.hdfs.HdfsDirectory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -130,7 +131,7 @@ public class TableShardCountCollapser extends Configured implements Tool { for (int i = 0; i < newShardCount; i++) { System.out.println("Base Index [" + paths[i] + "]"); IndexWriterConfig lconf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer()); - lconf.setCodec(new Blur021Codec()); + lconf.setCodec(new Blur022Codec()); HdfsDirectory dir = new HdfsDirectory(getConf(), paths[i]); IndexWriter indexWriter = new IndexWriter(dir, lconf); Directory[] dirs = new Directory[numberOfShardsToMergePerPass - 1]; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java index dbedd16..7d498dd 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java @@ -269,7 +269,7 @@ public class IndexManagerTest { indexManager.mutate(mutation7); } -// @Test + @Test public void testMutationReplaceLargeRow() throws Exception { final String rowId = "largerow"; indexManager.mutate(getLargeRow(rowId)); @@ -293,7 +293,7 @@ public class IndexManagerTest { Thread thread = new Thread(new Runnable() { @Override public void run() { - Trace.setupTrace(rowId); +// Trace.setupTrace(rowId); Selector selector = new Selector().setRowId(rowId); FetchResult fetchResult = new FetchResult(); long s = System.nanoTime(); @@ -304,7 +304,7 @@ public class IndexManagerTest { } long e = System.nanoTime(); assertNotNull(fetchResult.rowResult.row); - Trace.tearDownTrace(); +// Trace.tearDownTrace(); System.out.println((e - s) / 1000000.0); } }); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java index 93f14a3..793494f 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java @@ -35,7 +35,7 @@ import java.util.concurrent.TimeUnit; import org.apache.blur.analysis.FieldManager; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; -import org.apache.blur.lucene.codec.Blur021Codec; +import org.apache.blur.lucene.codec.Blur022Codec; import org.apache.blur.lucene.search.FairSimilarity; import org.apache.blur.manager.writer.TransactionRecorder; import org.apache.blur.mapreduce.BlurTask.INDEXING_TYPE; @@ -306,7 +306,7 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { if (optimize) { context.setStatus("Starting Copy-Optimize Phase"); IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _analyzer); - conf.setCodec(new Blur021Codec()); + conf.setCodec(new Blur022Codec()); TieredMergePolicy policy = (TieredMergePolicy) conf.getMergePolicy(); policy.setUseCompoundFile(false); long s = System.currentTimeMillis(); @@ -462,7 +462,7 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { nullCheck(_directory); nullCheck(_analyzer); IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, _analyzer); - config.setCodec(new Blur021Codec()); + config.setCodec(new Blur022Codec()); config.setSimilarity(new FairSimilarity()); config.setRAMBufferSizeMB(_blurTask.getRamBufferSizeMB()); TieredMergePolicy mergePolicy = (TieredMergePolicy) config.getMergePolicy(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java index 1731864..aa5ae1b 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java @@ -31,6 +31,7 @@ import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.lucene.LuceneVersionConstant; import org.apache.blur.lucene.codec.Blur021Codec; +import org.apache.blur.lucene.codec.Blur022Codec; import org.apache.blur.manager.writer.TransactionRecorder; import org.apache.blur.mapreduce.lib.BlurMutate.MUTATE_TYPE; import org.apache.blur.server.TableContext; @@ -399,7 +400,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { Analyzer analyzer = _fieldManager.getAnalyzerForIndex(); _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer); - _conf.setCodec(new Blur021Codec()); + _conf.setCodec(new Blur022Codec()); _conf.setSimilarity(tableContext.getSimilarity()); TieredMergePolicy mergePolicy = (TieredMergePolicy) _conf.getMergePolicy(); mergePolicy.setUseCompoundFile(false); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-query/src/test/java/org/apache/blur/filter/FilterCacheTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/filter/FilterCacheTest.java b/blur-query/src/test/java/org/apache/blur/filter/FilterCacheTest.java index ab484fb..7acadb8 100644 --- a/blur-query/src/test/java/org/apache/blur/filter/FilterCacheTest.java +++ b/blur-query/src/test/java/org/apache/blur/filter/FilterCacheTest.java @@ -16,17 +16,17 @@ */ package org.apache.blur.filter; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import java.io.IOException; import java.util.Arrays; import java.util.TreeSet; -import org.apache.blur.lucene.codec.Blur021Codec; +import org.apache.blur.lucene.codec.Blur022Codec; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -112,7 +112,7 @@ public class FilterCacheTest { private void writeDocs(FilterCache filterCache, RAMDirectory directory) throws IOException { IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer()); - conf.setCodec(new Blur021Codec()); + conf.setCodec(new Blur022Codec()); IndexWriter indexWriter = new IndexWriter(directory, conf); int count = 10000; addDocs(indexWriter, count); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java index 311f40b..b2f4caa 100644 --- a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java @@ -26,6 +26,7 @@ import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.TermVectorsFormat; import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat; +import org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat; import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosFormat; import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat; import org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat; @@ -33,7 +34,7 @@ import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; public class Blur021Codec extends Codec { - private final StoredFieldsFormat fieldsFormat = new Blur021StoredFieldsFormat(); + private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat(); private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat(); private final FieldInfosFormat fieldInfosFormat = new Lucene42FieldInfosFormat(); private final SegmentInfoFormat infosFormat = new Lucene40SegmentInfoFormat(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java deleted file mode 100644 index 86422a3..0000000 --- a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.blur.lucene.codec; - -import org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat; - -public final class Blur021StoredFieldsFormat extends CompressingStoredFieldsFormat { - public Blur021StoredFieldsFormat() { - super("Lucene41StoredFields", CachingCompressionMode.CACHING_FAST, 1 << 14); - } -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022Codec.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022Codec.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022Codec.java new file mode 100644 index 0000000..a4ebdcd --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022Codec.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.codec; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosFormat; +import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat; +import org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; + +public class Blur022Codec extends Codec { + private final StoredFieldsFormat fieldsFormat; + private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat(); + private final FieldInfosFormat fieldInfosFormat = new Lucene42FieldInfosFormat(); + private final SegmentInfoFormat infosFormat; + private final LiveDocsFormat liveDocsFormat = new Blur021LiveDocsFormat(); + + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return Blur022Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return Blur022Codec.this.getDocValuesFormatForField(field); + } + }; + + public Blur022Codec() { + this(1 << 14, CompressionMode.FAST); + } + + public Blur022Codec(int chunkSize, CompressionMode compressionMode) { + super("Blur022"); + infosFormat = new Blur022SegmentInfoFormat(chunkSize, compressionMode); + fieldsFormat = new Blur022StoredFieldsFormat(chunkSize, compressionMode); + } + + @Override + public final StoredFieldsFormat storedFieldsFormat() { + return fieldsFormat; + } + + @Override + public final TermVectorsFormat termVectorsFormat() { + return vectorsFormat; + } + + @Override + public final PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public final SegmentInfoFormat segmentInfoFormat() { + return infosFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + /** + * Returns the postings format that should be used for writing new segments of + * <code>field</code>. + * + * The default implementation always returns "Lucene41" + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + /** + * Returns the docvalues format that should be used for writing new segments + * of <code>field</code>. + * + * The default implementation always returns "Lucene42" + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41"); + private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene42"); + + private final NormsFormat normsFormat = new Lucene42NormsFormat(); + + @Override + public final NormsFormat normsFormat() { + return normsFormat; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoFormat.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoFormat.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoFormat.java new file mode 100644 index 0000000..b8b5e82 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoFormat.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.codec; + +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.SegmentInfoReader; +import org.apache.lucene.codecs.SegmentInfoWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.index.SegmentInfo; + +public class Blur022SegmentInfoFormat extends SegmentInfoFormat { + + /** File extension used to store {@link SegmentInfo}. */ + public final static String SI_EXTENSION = "si"; + static final String CODEC_NAME = "Blur022SegmentInfo"; + static final int VERSION_START = 0; + static final int VERSION_CURRENT = VERSION_START; + + private final SegmentInfoReader reader = new Blur022SegmentInfoReader(); + private final SegmentInfoWriter writer; + + public Blur022SegmentInfoFormat(int chunkSize, CompressionMode compressionMode) { + writer = new Blur022SegmentInfoWriter(chunkSize, compressionMode); + } + + @Override + public SegmentInfoReader getSegmentInfoReader() { + return reader; + } + + @Override + public SegmentInfoWriter getSegmentInfoWriter() { + return writer; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoReader.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoReader.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoReader.java new file mode 100644 index 0000000..2a0da16 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoReader.java @@ -0,0 +1,76 @@ +package org.apache.blur.lucene.codec; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.SegmentInfoReader; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.IOUtils; + +public class Blur022SegmentInfoReader extends SegmentInfoReader { + + @Override + public SegmentInfo read(Directory dir, String segment, IOContext context) throws IOException { + final String fileName = IndexFileNames.segmentFileName(segment, "", Blur022SegmentInfoFormat.SI_EXTENSION); + final IndexInput input = dir.openInput(fileName, context); + boolean success = false; + try { + CodecUtil.checkHeader(input, Blur022SegmentInfoFormat.CODEC_NAME, Blur022SegmentInfoFormat.VERSION_START, + Blur022SegmentInfoFormat.VERSION_CURRENT); + final String version = input.readString(); + final int docCount = input.readInt(); + if (docCount < 0) { + throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")"); + } + final boolean isCompoundFile = input.readByte() == SegmentInfo.YES; + final Map<String, String> diagnostics = input.readStringStringMap(); + final Map<String, String> attributes = input.readStringStringMap(); + final Set<String> files = input.readStringSet(); + + if (input.getFilePointer() != input.length()) { + throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")"); + } + + final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, + Collections.unmodifiableMap(attributes)); + si.setFiles(files); + + success = true; + + return si; + + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(input); + } else { + input.close(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoWriter.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoWriter.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoWriter.java new file mode 100644 index 0000000..1de6a4b --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022SegmentInfoWriter.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.codec; + +import java.io.IOException; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.SegmentInfoWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; + +public class Blur022SegmentInfoWriter extends SegmentInfoWriter { + + private final String _compressionMode; + private final int _compressionChunkSize; + + public Blur022SegmentInfoWriter(int compressionChunkSize, CompressionMode compressionMode) { + _compressionChunkSize = compressionChunkSize; + _compressionMode = compressionMode.toString(); + } + + @Override + public void write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException { + final String fileName = IndexFileNames.segmentFileName(si.name, "", Blur022SegmentInfoFormat.SI_EXTENSION); + si.addFile(fileName); + + final IndexOutput output = dir.createOutput(fileName, ioContext); + + boolean success = false; + try { + CodecUtil.writeHeader(output, Blur022SegmentInfoFormat.CODEC_NAME, Blur022SegmentInfoFormat.VERSION_CURRENT); + output.writeString(si.getVersion()); + output.writeInt(si.getDocCount()); + + output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO)); + output.writeStringStringMap(si.getDiagnostics()); + Map<String, String> attributes = si.attributes(); + TreeMap<String, String> newAttributes = new TreeMap<String, String>(); + if (attributes != null) { + newAttributes.putAll(attributes); + } + newAttributes.put(Blur022StoredFieldsFormat.STORED_FIELDS_FORMAT_CHUNK_SIZE, + Integer.toString(_compressionChunkSize)); + newAttributes.put(Blur022StoredFieldsFormat.STORED_FIELDS_FORMAT_COMPRESSION_MODE, _compressionMode); + output.writeStringStringMap(newAttributes); + output.writeStringSet(si.files()); + + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(output); + si.dir.deleteFile(fileName); + } else { + output.close(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java new file mode 100644 index 0000000..e84b210 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur022StoredFieldsFormat.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.codec; + +import java.io.IOException; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader; +import org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.compressing.Compressor; +import org.apache.lucene.codecs.compressing.Decompressor; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +public final class Blur022StoredFieldsFormat extends StoredFieldsFormat { + + static final String STORED_FIELDS_FORMAT_CHUNK_SIZE = "StoredFieldsFormat.chunkSize"; + static final String STORED_FIELDS_FORMAT_COMPRESSION_MODE = "StoredFieldsFormat.compressionMode"; + private static final String FAST_DECOMPRESSION = "FAST_DECOMPRESSION"; + private static final String FAST = "FAST"; + private static final String HIGH_COMPRESSION = "HIGH_COMPRESSION"; + private static final String FORMAT_NAME = "Blur022StoredFields"; + private static final String SEGMENT_SUFFIX = ""; + private final int _chunkSize; + private final CompressionMode _compressionMode; + + public Blur022StoredFieldsFormat(int chunkSize, CompressionMode compressionMode) { + _chunkSize = chunkSize; + _compressionMode = compressionMode; + } + + static class CachedCompressionMode extends CompressionMode { + + final CompressionMode _compressionMode; + + CachedCompressionMode(CompressionMode compressionMode) { + _compressionMode = compressionMode; + } + + @Override + public Compressor newCompressor() { + return _compressionMode.newCompressor(); + } + + @Override + public Decompressor newDecompressor() { + return new CachedDecompressor(_compressionMode.newDecompressor()); + } + + @Override + public String toString() { + return _compressionMode.toString(); + } + + } + + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) + throws IOException { + CompressionMode compressionMode = new CachedCompressionMode(getCompressionMode(si)); + return new CompressingStoredFieldsReader(directory, si, SEGMENT_SUFFIX, fn, context, FORMAT_NAME, compressionMode); + } + + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + return new CompressingStoredFieldsWriter(directory, si, SEGMENT_SUFFIX, context, FORMAT_NAME, _compressionMode, + _chunkSize); + } + + private CompressionMode getCompressionMode(SegmentInfo si) { + String attribute = si.getAttribute(STORED_FIELDS_FORMAT_COMPRESSION_MODE); + if (HIGH_COMPRESSION.equals(attribute)) { + return CompressionMode.HIGH_COMPRESSION; + } else if (FAST.equals(attribute)) { + return CompressionMode.FAST; + } else if (FAST_DECOMPRESSION.equals(attribute)) { + return CompressionMode.FAST_DECOMPRESSION; + } + // This happen during nrt udpates. + return _compressionMode; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java new file mode 100644 index 0000000..10b2882 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/CachedDecompressor.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.codec; + +import java.io.IOException; + +import org.apache.lucene.codecs.compressing.Decompressor; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BytesRef; + +public class CachedDecompressor extends Decompressor { + + static class Entry { + IndexInput _indexInput; + long _position = -1; + BytesRef _bytesRef = new BytesRef(); + } + + private final Decompressor _decompressor; + private final ThreadLocal<Entry> _cache = new ThreadLocal<Entry>() { + @Override + protected Entry initialValue() { + return new Entry(); + } + }; + + public CachedDecompressor(Decompressor decompressor) { + _decompressor = decompressor; + } + + @Override + public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { + if (in instanceof IndexInput) { + IndexInput indexInput = (IndexInput) in; + Entry entry = _cache.get(); + long filePointer = indexInput.getFilePointer(); + BytesRef cachedRef = entry._bytesRef; + if (entry._indexInput != indexInput || entry._position != filePointer) { + cachedRef.grow(originalLength); + _decompressor.decompress(in, originalLength, 0, originalLength, cachedRef); + entry._indexInput = indexInput; + entry._position = filePointer; + } + bytes.copyBytes(cachedRef); + bytes.offset = offset; + bytes.length = length; + } else { + _decompressor.decompress(in, originalLength, offset, length, bytes); + } + } + + @Override + public Decompressor clone() { + return this; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java deleted file mode 100644 index 94f1304..0000000 --- a/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.blur.lucene.codec; - -import java.io.IOException; - -import org.apache.lucene.codecs.compressing.CompressionMode; -import org.apache.lucene.codecs.compressing.Compressor; -import org.apache.lucene.codecs.compressing.Decompressor; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.BytesRef; - -public class CachingCompressionMode { - - public static final CompressionMode CACHING_FAST = new CompressionMode() { - - @Override - public Compressor newCompressor() { - return CompressionMode.FAST.newCompressor(); - } - - @Override - public Decompressor newDecompressor() { - return new CachedDecompressor(CompressionMode.FAST.newDecompressor()); - } - - @Override - public String toString() { - return "CACHING_FAST"; - } - - }; - - public static class CachedDecompressor extends Decompressor { - - static class Entry { - IndexInput _indexInput; - long _position = -1; - BytesRef _bytesRef = new BytesRef(); - } - - private final Decompressor _decompressor; - private final ThreadLocal<Entry> _cache = new ThreadLocal<Entry>() { - @Override - protected Entry initialValue() { - return new Entry(); - } - }; - - public CachedDecompressor(Decompressor decompressor) { - _decompressor = decompressor; - } - - @Override - public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { - if (in instanceof IndexInput) { - IndexInput indexInput = (IndexInput) in; - Entry entry = _cache.get(); - long filePointer = indexInput.getFilePointer(); - BytesRef cachedRef = entry._bytesRef; - if (entry._indexInput != indexInput || entry._position != filePointer) { - cachedRef.grow(originalLength); - _decompressor.decompress(in, originalLength, 0, originalLength, cachedRef); - entry._indexInput = indexInput; - entry._position = filePointer; - } - bytes.copyBytes(cachedRef); - bytes.offset = offset; - bytes.length = length; - } else { - _decompressor.decompress(in, originalLength, offset, length, bytes); - } - } - - @Override - public Decompressor clone() { - return this; - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-store/src/main/java/org/apache/blur/lucene/codec/Testing.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Testing.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Testing.java deleted file mode 100644 index 7bc1332..0000000 --- a/blur-store/src/main/java/org/apache/blur/lucene/codec/Testing.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.blur.lucene.codec; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class Testing { - - static final Pattern CODEC_FILE_PATTERN = Pattern.compile("_[a-z0-9]+(_.*)?\\..*"); - - public static void main(String[] args) { - String t = "_0_filter1.filter"; - Matcher matcher = CODEC_FILE_PATTERN.matcher(""); - matcher.reset(t); - - if (matcher.matches()) { - System.out.println("yay!"); - } - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6d2d4c/blur-store/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec ---------------------------------------------------------------------- diff --git a/blur-store/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/blur-store/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index fe91c2e..8fb3dea 100644 --- a/blur-store/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/blur-store/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.blur.lucene.codec.Blur021Codec \ No newline at end of file +org.apache.blur.lucene.codec.Blur021Codec +org.apache.blur.lucene.codec.Blur022Codec \ No newline at end of file
