Creating a caching decompressor, helps row fetching performance.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/985c72bf Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/985c72bf Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/985c72bf Branch: refs/heads/master Commit: 985c72bf79c3c906d468504f4796e3a9a584619b Parents: 346a201 Author: Aaron McCurry <[email protected]> Authored: Sun Dec 8 17:01:05 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Sun Dec 8 22:02:14 2013 -0500 ---------------------------------------------------------------------- .../apache/blur/manager/IndexManagerTest.java | 2 +- .../apache/blur/lucene/codec/Blur021Codec.java | 4 +- .../lucene/codec/Blur021StoredFieldsFormat.java | 25 ++++++ .../lucene/codec/CachingCompressionMode.java | 95 ++++++++++++++++++++ 4 files changed, 122 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/985c72bf/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java index 851df4f..5390a2c 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java @@ -313,7 +313,7 @@ public class IndexManagerTest { } Trace.setStorage(oldReporter); - + } private RowMutation getLargeRow(String rowId) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/985c72bf/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java index 2d47705..311f40b 100644 --- a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java @@ -26,7 +26,6 @@ import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.TermVectorsFormat; import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat; -import org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat; import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosFormat; import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat; import org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat; @@ -34,7 +33,7 @@ import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; public class Blur021Codec extends Codec { - private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat(); + private final StoredFieldsFormat fieldsFormat = new Blur021StoredFieldsFormat(); private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat(); private final FieldInfosFormat fieldInfosFormat = new Lucene42FieldInfosFormat(); private final SegmentInfoFormat infosFormat = new Lucene40SegmentInfoFormat(); @@ -47,7 +46,6 @@ public class Blur021Codec extends Codec { } }; - private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { @Override public DocValuesFormat getDocValuesFormatForField(String field) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/985c72bf/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java new file mode 100644 index 0000000..d2f678e --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.codec; + +import org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat; + +public final class Blur021StoredFieldsFormat extends CompressingStoredFieldsFormat { + public Blur021StoredFieldsFormat() { + super("Blur021StoredFields", CachingCompressionMode.CACHING_FAST, 1 << 14); + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/985c72bf/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java new file mode 100644 index 0000000..94f1304 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.codec; + +import java.io.IOException; + +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.compressing.Compressor; +import org.apache.lucene.codecs.compressing.Decompressor; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BytesRef; + +public class CachingCompressionMode { + + public static final CompressionMode CACHING_FAST = new CompressionMode() { + + @Override + public Compressor newCompressor() { + return CompressionMode.FAST.newCompressor(); + } + + @Override + public Decompressor newDecompressor() { + return new CachedDecompressor(CompressionMode.FAST.newDecompressor()); + } + + @Override + public String toString() { + return "CACHING_FAST"; + } + + }; + + public static class CachedDecompressor extends Decompressor { + + static class Entry { + IndexInput _indexInput; + long _position = -1; + BytesRef _bytesRef = new BytesRef(); + } + + private final Decompressor _decompressor; + private final ThreadLocal<Entry> _cache = new ThreadLocal<Entry>() { + @Override + protected Entry initialValue() { + return new Entry(); + } + }; + + public CachedDecompressor(Decompressor decompressor) { + _decompressor = decompressor; + } + + @Override + public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { + if (in instanceof IndexInput) { + IndexInput indexInput = (IndexInput) in; + Entry entry = _cache.get(); + long filePointer = indexInput.getFilePointer(); + BytesRef cachedRef = entry._bytesRef; + if (entry._indexInput != indexInput || entry._position != filePointer) { + cachedRef.grow(originalLength); + _decompressor.decompress(in, originalLength, 0, originalLength, cachedRef); + entry._indexInput = indexInput; + entry._position = filePointer; + } + bytes.copyBytes(cachedRef); + bytes.offset = offset; + bytes.length = length; + } else { + _decompressor.decompress(in, originalLength, offset, length, bytes); + } + } + + @Override + public Decompressor clone() { + return this; + } + } +}
