Updated Branches: refs/heads/apache-blur-0.2 9589b3d9c -> bce2c3120
Fixed BLUR-285 Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/6607cf27 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/6607cf27 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/6607cf27 Branch: refs/heads/apache-blur-0.2 Commit: 6607cf27d7c48a12938b4060b5127336e0c15c1b Parents: 9589b3d Author: Aaron McCurry <[email protected]> Authored: Wed Oct 23 10:17:34 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Wed Oct 23 10:17:34 2013 -0400 ---------------------------------------------------------------------- .../blur/manager/writer/BlurIndexReader.java | 2 + .../blur/manager/writer/BlurNRTIndex.java | 2 + .../org/apache/blur/mapreduce/BlurReducer.java | 3 + .../blur/mapreduce/lib/BlurOutputFormat.java | 2 + .../org/apache/blur/filter/FilterCache.java | 136 ++++++++++++++ .../org/apache/blur/filter/IndexFileBitSet.java | 106 +++++++++++ .../blur/filter/IndexFileBitSetIterator.java | 177 +++++++++++++++++++ .../apache/blur/filter/IndexInputLongArray.java | 40 +++++ .../org/apache/blur/filter/FilterCacheTest.java | 131 ++++++++++++++ .../apache/blur/filter/IndexFileBitSetTest.java | 78 ++++++++ .../apache/blur/lucene/codec/Blur021Codec.java | 125 +++++++++++++ .../lucene/codec/Blur021LiveDocsFormat.java | 48 +++++ .../org/apache/blur/lucene/codec/Testing.java | 37 ++++ .../services/org.apache.lucene.codecs.Codec | 16 ++ 14 files changed, 903 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java index dd87fdb..eb4b65e 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java @@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; +import org.apache.blur.lucene.codec.Blur021Codec; import org.apache.blur.lucene.warmup.TraceableDirectory; import org.apache.blur.server.IndexSearcherClosable; import org.apache.blur.server.ShardContext; @@ -66,6 +67,7 @@ public class BlurIndexReader extends BlurIndex { // if the directory is empty then create an empty index. IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer()); conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5)); + conf.setCodec(new Blur021Codec()); new BlurIndexWriter(directory, conf).close(); } _indexReaderRef.set(DirectoryReader.open(directory)); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java index a9428c6..5cf15e1 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java @@ -38,6 +38,7 @@ import org.apache.blur.analysis.FieldManager; import org.apache.blur.index.ExitableReader; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; +import org.apache.blur.lucene.codec.Blur021Codec; import org.apache.blur.lucene.store.refcounter.DirectoryReferenceCounter; import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC; import org.apache.blur.lucene.warmup.TraceableDirectory; @@ -104,6 +105,7 @@ public class BlurNRTIndex extends BlurIndex { Analyzer analyzer = fieldManager.getAnalyzerForIndex(); IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, analyzer); conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5)); + conf.setCodec(new Blur021Codec()); conf.setSimilarity(_tableContext.getSimilarity()); SnapshotDeletionPolicy sdp; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java index 3669232..26d1122 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java @@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit; import org.apache.blur.analysis.FieldManager; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; +import org.apache.blur.lucene.codec.Blur021Codec; import org.apache.blur.lucene.search.FairSimilarity; import org.apache.blur.manager.writer.TransactionRecorder; import org.apache.blur.mapreduce.BlurTask.INDEXING_TYPE; @@ -304,6 +305,7 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { if (optimize) { context.setStatus("Starting Copy-Optimize Phase"); IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _analyzer); + conf.setCodec(new Blur021Codec()); TieredMergePolicy policy = (TieredMergePolicy) conf.getMergePolicy(); policy.setUseCompoundFile(false); long s = System.currentTimeMillis(); @@ -459,6 +461,7 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { nullCheck(_directory); nullCheck(_analyzer); IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, _analyzer); + config.setCodec(new Blur021Codec()); config.setSimilarity(new FairSimilarity()); config.setRAMBufferSizeMB(_blurTask.getRamBufferSizeMB()); TieredMergePolicy mergePolicy = (TieredMergePolicy) config.getMergePolicy(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java index f2e689d..caa3504 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java @@ -30,6 +30,7 @@ import org.apache.blur.analysis.FieldManager; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.lucene.LuceneVersionConstant; +import org.apache.blur.lucene.codec.Blur021Codec; import org.apache.blur.manager.writer.TransactionRecorder; import org.apache.blur.mapreduce.lib.BlurMutate.MUTATE_TYPE; import org.apache.blur.server.TableContext; @@ -399,6 +400,7 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate> { Analyzer analyzer = _fieldManager.getAnalyzerForIndex(); _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer); + _conf.setCodec(new Blur021Codec()); TieredMergePolicy mergePolicy = (TieredMergePolicy) _conf.getMergePolicy(); mergePolicy.setUseCompoundFile(false); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-query/src/main/java/org/apache/blur/filter/FilterCache.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/filter/FilterCache.java b/blur-query/src/main/java/org/apache/blur/filter/FilterCache.java new file mode 100644 index 0000000..ab5ac68 --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/filter/FilterCache.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.filter; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.WeakHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.blur.log.Log; +import org.apache.blur.log.LogFactory; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.search.BitsFilteredDocIdSet; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Filter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; + +public class FilterCache extends Filter { + + private static final Log LOG = LogFactory.getLog(FilterCache.class); + + private final Map<Object, DocIdSet> _cache = Collections.synchronizedMap(new WeakHashMap<Object, DocIdSet>()); + private final Map<Object, Object> _lockMap = Collections.synchronizedMap(new WeakHashMap<Object, Object>()); + private final Filter _filter; + private final String _id; + private final AtomicLong _hits = new AtomicLong(); + private final AtomicLong _misses = new AtomicLong(); + + public FilterCache(String id, Filter filter) { + _id = id; + _filter = filter; + } + + @Override + public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException { + AtomicReader reader = context.reader(); + Object key = reader.getCoreCacheKey(); + DocIdSet docIdSet = _cache.get(key); + if (docIdSet != null) { + _hits.incrementAndGet(); + return BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs); + } + // This will only allow a single instance be created per reader per filter + Object lock = getLock(key); + synchronized (lock) { + SegmentReader segmentReader = getSegmentReader(reader); + if (segmentReader == null) { + LOG.warn("Could not find SegmentReader from [{0}]", reader); + return _filter.getDocIdSet(context, acceptDocs); + } + Directory directory = getDirectory(segmentReader); + if (directory == null) { + LOG.warn("Could not find Directory from [{0}]", segmentReader); + return _filter.getDocIdSet(context, acceptDocs); + } + _misses.incrementAndGet(); + String segmentName = segmentReader.getSegmentName(); + docIdSet = docIdSetToCache(_filter.getDocIdSet(context, null), reader, segmentName, directory); + _cache.put(key, docIdSet); + return BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs); + } + } + + private synchronized Object getLock(Object key) { + Object lock = _lockMap.get(key); + if (lock == null) { + lock = new Object(); + _lockMap.put(key, lock); + } + return lock; + } + + private DocIdSet docIdSetToCache(DocIdSet docIdSet, AtomicReader reader, String segmentName, Directory directory) + throws IOException { + if (docIdSet == null) { + // this is better than returning null, as the nonnull result can be cached + return DocIdSet.EMPTY_DOCIDSET; + } else if (docIdSet.isCacheable()) { + return docIdSet; + } else { + final DocIdSetIterator it = docIdSet.iterator(); + // null is allowed to be returned by iterator(), + // in this case we wrap with the empty set, + // which is cacheable. + if (it == null) { + return DocIdSet.EMPTY_DOCIDSET; + } else { + final IndexFileBitSet bits = new IndexFileBitSet(reader.maxDoc(), _id, segmentName, directory); + if (!bits.exists()) { + bits.create(it); + } + bits.load(); + return bits; + } + } + } + + private Directory getDirectory(SegmentReader reader) { + return reader.directory(); + } + + private SegmentReader getSegmentReader(AtomicReader reader) { + if (reader instanceof SegmentReader) { + return (SegmentReader) reader; + } + return null; + } + + public long getHits() { + return _hits.get(); + } + + public long getMisses() { + return _misses.get(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-query/src/main/java/org/apache/blur/filter/IndexFileBitSet.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/filter/IndexFileBitSet.java b/blur-query/src/main/java/org/apache/blur/filter/IndexFileBitSet.java new file mode 100644 index 0000000..4ef6031 --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/filter/IndexFileBitSet.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.filter; + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; + +public class IndexFileBitSet extends DocIdSet implements Closeable { + + public static final String EXTENSION = ".filter"; + + private final String _id; + private final String _segmentName; + private final Directory _directory; + private final int _numBits; + private IndexInput _indexInput; + + public IndexFileBitSet(int numBits, String id, String segmentName, Directory directory) { + _id = id; + _segmentName = segmentName; + _directory = directory; + _numBits = numBits; + } + + @Override + public DocIdSetIterator iterator() throws IOException { + return new IndexFileBitSetIterator(_indexInput.clone()); + } + + public boolean exists() throws IOException { + return _directory.fileExists(getFileName()); + } + + private String getFileName() { + return _segmentName + "_" + _id + EXTENSION; + } + + public void load() throws IOException { + String fileName = getFileName(); + _indexInput = _directory.openInput(fileName, IOContext.READ); + int words = (_numBits / 64) + 1; + int correctLength = words * 8; + long length = _indexInput.length(); + if (correctLength != length) { + throw new IOException("File [" + fileName + "] with length [" + length + "] does not match correct length of [" + + correctLength + "]"); + } + } + + public void create(DocIdSetIterator it) throws IOException { + String fileName = getFileName(); + IndexOutput output = _directory.createOutput(fileName, IOContext.READ); + int index; + int currentWordNum = 0; + long wordValue = 0; + while ((index = it.nextDoc()) < _numBits) { + int wordNum = index >> 6; // div 64 + if (currentWordNum > wordNum) { + throw new IOException("We got a problem here!"); + } + while (currentWordNum < wordNum) { + output.writeLong(wordValue); + currentWordNum++; + wordValue = 0; + } + int bit = index & 0x3f; // mod 64 + long bitmask = 1L << bit; + wordValue |= bitmask; + } + if (_numBits > 0) { + int totalWords = (_numBits / 64) + 1; + while (currentWordNum < totalWords) { + output.writeLong(wordValue); + currentWordNum++; + wordValue = 0; + } + } + output.close(); + } + + @Override + public void close() throws IOException { + _indexInput.close(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-query/src/main/java/org/apache/blur/filter/IndexFileBitSetIterator.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/filter/IndexFileBitSetIterator.java b/blur-query/src/main/java/org/apache/blur/filter/IndexFileBitSetIterator.java new file mode 100644 index 0000000..fe266a1 --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/filter/IndexFileBitSetIterator.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.blur.filter; + +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.IndexInput; + +/** + * An iterator to iterate over set bits in an OpenBitSet. This is faster than + * nextSetBit() for iterating over the complete set of bits, especially when the + * density of the bits set is high. + */ +public class IndexFileBitSetIterator extends DocIdSetIterator { + + // The General Idea: instead of having an array per byte that has + // the offsets of the next set bit, that array could be + // packed inside a 32 bit integer (8 4 bit numbers). That + // should be faster than accessing an array for each index, and + // the total array size is kept smaller (256*sizeof(int))=1K + protected final static int[] bitlist = { 0x0, 0x1, 0x2, 0x21, 0x3, 0x31, 0x32, 0x321, 0x4, 0x41, 0x42, 0x421, 0x43, + 0x431, 0x432, 0x4321, 0x5, 0x51, 0x52, 0x521, 0x53, 0x531, 0x532, 0x5321, 0x54, 0x541, 0x542, 0x5421, 0x543, + 0x5431, 0x5432, 0x54321, 0x6, 0x61, 0x62, 0x621, 0x63, 0x631, 0x632, 0x6321, 0x64, 0x641, 0x642, 0x6421, 0x643, + 0x6431, 0x6432, 0x64321, 0x65, 0x651, 0x652, 0x6521, 0x653, 0x6531, 0x6532, 0x65321, 0x654, 0x6541, 0x6542, + 0x65421, 0x6543, 0x65431, 0x65432, 0x654321, 0x7, 0x71, 0x72, 0x721, 0x73, 0x731, 0x732, 0x7321, 0x74, 0x741, + 0x742, 0x7421, 0x743, 0x7431, 0x7432, 0x74321, 0x75, 0x751, 0x752, 0x7521, 0x753, 0x7531, 0x7532, 0x75321, 0x754, + 0x7541, 0x7542, 0x75421, 0x7543, 0x75431, 0x75432, 0x754321, 0x76, 0x761, 0x762, 0x7621, 0x763, 0x7631, 0x7632, + 0x76321, 0x764, 0x7641, 0x7642, 0x76421, 0x7643, 0x76431, 0x76432, 0x764321, 0x765, 0x7651, 0x7652, 0x76521, + 0x7653, 0x76531, 0x76532, 0x765321, 0x7654, 0x76541, 0x76542, 0x765421, 0x76543, 0x765431, 0x765432, 0x7654321, + 0x8, 0x81, 0x82, 0x821, 0x83, 0x831, 0x832, 0x8321, 0x84, 0x841, 0x842, 0x8421, 0x843, 0x8431, 0x8432, 0x84321, + 0x85, 0x851, 0x852, 0x8521, 0x853, 0x8531, 0x8532, 0x85321, 0x854, 0x8541, 0x8542, 0x85421, 0x8543, 0x85431, + 0x85432, 0x854321, 0x86, 0x861, 0x862, 0x8621, 0x863, 0x8631, 0x8632, 0x86321, 0x864, 0x8641, 0x8642, 0x86421, + 0x8643, 0x86431, 0x86432, 0x864321, 0x865, 0x8651, 0x8652, 0x86521, 0x8653, 0x86531, 0x86532, 0x865321, 0x8654, + 0x86541, 0x86542, 0x865421, 0x86543, 0x865431, 0x865432, 0x8654321, 0x87, 0x871, 0x872, 0x8721, 0x873, 0x8731, + 0x8732, 0x87321, 0x874, 0x8741, 0x8742, 0x87421, 0x8743, 0x87431, 0x87432, 0x874321, 0x875, 0x8751, 0x8752, + 0x87521, 0x8753, 0x87531, 0x87532, 0x875321, 0x8754, 0x87541, 0x87542, 0x875421, 0x87543, 0x875431, 0x875432, + 0x8754321, 0x876, 0x8761, 0x8762, 0x87621, 0x8763, 0x87631, 0x87632, 0x876321, 0x8764, 0x87641, 0x87642, + 0x876421, 0x87643, 0x876431, 0x876432, 0x8764321, 0x8765, 0x87651, 0x87652, 0x876521, 0x87653, 0x876531, + 0x876532, 0x8765321, 0x87654, 0x876541, 0x876542, 0x8765421, 0x876543, 0x8765431, 0x8765432, 0x87654321 }; + /***** + * the python code that generated bitlist def bits2int(val): arr=0 for shift + * in range(8,0,-1): if val & 0x80: arr = (arr << 4) | shift val = val << 1 + * return arr + * + * def int_table(): tbl = [ hex(bits2int(val)).strip('L') for val in + * range(256) ] return ','.join(tbl) + ******/ + + // hmmm, what about an iterator that finds zeros though, + // or a reverse iterator... should they be separate classes + // for efficiency, or have a common root interface? (or + // maybe both? could ask for a SetBitsIterator, etc... + + final IndexInputLongArray arr; + final int words; + private int i = -1; + private long word; + private int wordShift; + private int indexArray; + private int curDocId = -1; + + public IndexFileBitSetIterator(IndexInput indexInput) { + arr = new IndexInputLongArray(indexInput); + words = (int) indexInput.length() / 8; + } + + // 64 bit shifts + private void shift() { + if ((int) word == 0) { + wordShift += 32; + word = word >>> 32; + } + if ((word & 0x0000FFFF) == 0) { + wordShift += 16; + word >>>= 16; + } + if ((word & 0x000000FF) == 0) { + wordShift += 8; + word >>>= 8; + } + indexArray = bitlist[(int) word & 0xff]; + } + + /***** + * alternate shift implementations // 32 bit shifts, but a long shift needed + * at the end private void shift2() { int y = (int)word; if (y==0) {wordShift + * +=32; y = (int)(word >>>32); } if ((y & 0x0000FFFF) == 0) { wordShift +=16; + * y>>>=16; } if ((y & 0x000000FF) == 0) { wordShift +=8; y>>>=8; } indexArray + * = bitlist[y & 0xff]; word >>>= (wordShift +1); } + * + * private void shift3() { int lower = (int)word; int lowByte = lower & 0xff; + * if (lowByte != 0) { indexArray=bitlist[lowByte]; return; } shift(); } + ******/ + + @Override + public int nextDoc() { + if (indexArray == 0) { + if (word != 0) { + word >>>= 8; + wordShift += 8; + } + + while (word == 0) { + if (++i >= words) { + return curDocId = NO_MORE_DOCS; + } + word = arr.get(i); + wordShift = -1; // loop invariant code motion should move this + } + + // after the first time, should I go with a linear search, or + // stick with the binary search in shift? + shift(); + } + + int bitIndex = (indexArray & 0x0f) + wordShift; + indexArray >>>= 4; + // should i<<6 be cached as a separate variable? + // it would only save one cycle in the best circumstances. + return curDocId = (i << 6) + bitIndex; + } + + @Override + public int advance(int target) { + indexArray = 0; + i = target >> 6; + if (i >= words) { + word = 0; // setup so next() will also return -1 + return curDocId = NO_MORE_DOCS; + } + wordShift = target & 0x3f; + word = arr.get(i) >>> wordShift; + if (word != 0) { + wordShift--; // compensate for 1 based arrIndex + } else { + while (word == 0) { + if (++i >= words) { + return curDocId = NO_MORE_DOCS; + } + word = arr.get(i); + } + wordShift = -1; + } + + shift(); + + int bitIndex = (indexArray & 0x0f) + wordShift; + indexArray >>>= 4; + // should i<<6 be cached as a separate variable? + // it would only save one cycle in the best circumstances. + return curDocId = (i << 6) + bitIndex; + } + + @Override + public int docID() { + return curDocId; + } + + @Override + public long cost() { + return words / 64; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-query/src/main/java/org/apache/blur/filter/IndexInputLongArray.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/filter/IndexInputLongArray.java b/blur-query/src/main/java/org/apache/blur/filter/IndexInputLongArray.java new file mode 100644 index 0000000..aeeeabe --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/filter/IndexInputLongArray.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.filter; + +import java.io.IOException; + +import org.apache.lucene.store.IndexInput; + +public class IndexInputLongArray { + + private IndexInput _indexInput; + + public IndexInputLongArray(IndexInput indexInput) { + _indexInput = indexInput; + } + + public long get(int index) { + try { + _indexInput.seek(index << 3); + return _indexInput.readLong(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-query/src/test/java/org/apache/blur/filter/FilterCacheTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/filter/FilterCacheTest.java b/blur-query/src/test/java/org/apache/blur/filter/FilterCacheTest.java new file mode 100644 index 0000000..ab484fb --- /dev/null +++ b/blur-query/src/test/java/org/apache/blur/filter/FilterCacheTest.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.filter; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.Arrays; +import java.util.TreeSet; + +import org.apache.blur.lucene.codec.Blur021Codec; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; +import org.junit.Test; + +public class FilterCacheTest { + + @Test + public void test1() throws IOException { + Filter filter = new QueryWrapperFilter(new TermQuery(new Term("f1", "t1"))); + FilterCache filterCache = new FilterCache("filter1", filter); + RAMDirectory directory = new RAMDirectory(); + writeDocs(filterCache, directory); + + DirectoryReader reader = DirectoryReader.open(directory); + + IndexSearcher searcher = new IndexSearcher(reader); + + Query query = new TermQuery(new Term("f2", "t2")); + TopDocs topDocs1 = searcher.search(query, filterCache, 10); + assertEquals(1, filterCache.getMisses()); + assertEquals(0, filterCache.getHits()); + assertEquals(1, topDocs1.totalHits); + + TopDocs topDocs2 = searcher.search(query, filterCache, 10); + assertEquals(1, filterCache.getMisses()); + assertEquals(1, filterCache.getHits()); + assertEquals(1, topDocs2.totalHits); + + TopDocs topDocs3 = searcher.search(query, filterCache, 10); + assertEquals(1, filterCache.getMisses()); + assertEquals(2, filterCache.getHits()); + assertEquals(1, topDocs3.totalHits); + } + + @Test + public void test2() throws IOException { + Filter filter = new QueryWrapperFilter(new TermQuery(new Term("f1", "t1"))); + FilterCache filterCache = new FilterCache("filter1", filter); + RAMDirectory directory = new RAMDirectory(); + writeDocs(filterCache, directory); + DirectoryReader reader = DirectoryReader.open(directory); + + IndexSearcher searcher = new IndexSearcher(reader); + + Query query = new TermQuery(new Term("f2", "t2")); + TopDocs topDocs1 = searcher.search(query, filterCache, 10); + assertEquals(1, filterCache.getMisses()); + assertEquals(0, filterCache.getHits()); + assertEquals(1, topDocs1.totalHits); + + TopDocs topDocs2 = searcher.search(query, filterCache, 10); + assertEquals(1, filterCache.getMisses()); + assertEquals(1, filterCache.getHits()); + assertEquals(1, topDocs2.totalHits); + + TopDocs topDocs3 = searcher.search(query, filterCache, 10); + assertEquals(1, filterCache.getMisses()); + assertEquals(2, filterCache.getHits()); + assertEquals(1, topDocs3.totalHits); + + System.out.println("==============="); + for (String s : new TreeSet<String>(Arrays.asList(directory.listAll()))) { + System.out.println(s); + } + + writeDocs(filterCache, directory); + + System.out.println("==============="); + for (String s : new TreeSet<String>(Arrays.asList(directory.listAll()))) { + System.out.println(s); + } + } + + private void writeDocs(FilterCache filterCache, RAMDirectory directory) throws IOException { + IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer()); + conf.setCodec(new Blur021Codec()); + IndexWriter indexWriter = new IndexWriter(directory, conf); + int count = 10000; + addDocs(indexWriter, count); + indexWriter.close(); + } + + private void addDocs(IndexWriter indexWriter, int count) throws IOException { + for (int i = 0; i < count; i++) { + Document document = new Document(); + document.add(new StringField("f1", "t" + i, Store.YES)); + document.add(new StringField("f2", "t2", Store.YES)); + indexWriter.addDocument(document); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-query/src/test/java/org/apache/blur/filter/IndexFileBitSetTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/filter/IndexFileBitSetTest.java b/blur-query/src/test/java/org/apache/blur/filter/IndexFileBitSetTest.java new file mode 100644 index 0000000..5135aad --- /dev/null +++ b/blur-query/src/test/java/org/apache/blur/filter/IndexFileBitSetTest.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.filter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.io.IOException; +import java.util.Random; + +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.FixedBitSet; +import org.junit.Before; +import org.junit.Test; + +public class IndexFileBitSetTest { + + private long _seed; + + @Before + public void setup() { + Random random = new Random(); + _seed = random.nextLong(); + } + + @Test + public void test() throws IOException { + Random random = new Random(_seed); + int numBits = random.nextInt(10000000); + FixedBitSet fixedBitSet = new FixedBitSet(numBits); + populate(random, numBits, fixedBitSet); + String id = "id"; + String segmentName = "seg1"; + RAMDirectory directory = new RAMDirectory(); + IndexFileBitSet indexFileBitSet = new IndexFileBitSet(numBits, id, segmentName, directory); + assertFalse(indexFileBitSet.exists()); + indexFileBitSet.create(fixedBitSet.iterator()); + indexFileBitSet.load(); + checkEquals(fixedBitSet.iterator(), indexFileBitSet.iterator(), numBits); + indexFileBitSet.close(); + + String[] listAll = directory.listAll(); + for (String s : listAll) { + System.out.println(s + " " + directory.fileLength(s)); + } + } + + private void populate(Random random, int numBits, FixedBitSet fixedBitSet) { + int population = random.nextInt(numBits); + for (int i = 0; i < population; i++) { + fixedBitSet.set(random.nextInt(numBits)); + } + } + + private void checkEquals(DocIdSetIterator expected, DocIdSetIterator actual, int numBits) throws IOException { + int expectedNextDoc; + while ((expectedNextDoc = expected.nextDoc()) < numBits) { + int actualNextDoc = actual.nextDoc(); + assertEquals(expectedNextDoc, actualNextDoc); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java new file mode 100644 index 0000000..2d47705 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.codec; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.TermVectorsFormat; +import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat; +import org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat; +import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosFormat; +import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat; +import org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; + +public class Blur021Codec extends Codec { + private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat(); + private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat(); + private final FieldInfosFormat fieldInfosFormat = new Lucene42FieldInfosFormat(); + private final SegmentInfoFormat infosFormat = new Lucene40SegmentInfoFormat(); + private final LiveDocsFormat liveDocsFormat = new Blur021LiveDocsFormat(); + + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return Blur021Codec.this.getPostingsFormatForField(field); + } + }; + + + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return Blur021Codec.this.getDocValuesFormatForField(field); + } + }; + + /** Sole constructor. */ + public Blur021Codec() { + super("Blur021"); + } + + @Override + public final StoredFieldsFormat storedFieldsFormat() { + return fieldsFormat; + } + + @Override + public final TermVectorsFormat termVectorsFormat() { + return vectorsFormat; + } + + @Override + public final PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public final SegmentInfoFormat segmentInfoFormat() { + return infosFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + /** Returns the postings format that should be used for writing + * new segments of <code>field</code>. + * + * The default implementation always returns "Lucene41" + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + /** Returns the docvalues format that should be used for writing + * new segments of <code>field</code>. + * + * The default implementation always returns "Lucene42" + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41"); + private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene42"); + + private final NormsFormat normsFormat = new Lucene42NormsFormat(); + + @Override + public final NormsFormat normsFormat() { + return normsFormat; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021LiveDocsFormat.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021LiveDocsFormat.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021LiveDocsFormat.java new file mode 100644 index 0000000..8c3d9a0 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021LiveDocsFormat.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.codec; + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat; +import org.apache.lucene.index.SegmentInfoPerCommit; +import org.apache.lucene.store.Directory; + +public class Blur021LiveDocsFormat extends Lucene40LiveDocsFormat { + + private static final String SEP = "_"; + private static final String FILTER = ".filter"; + private static final String SAMPLE = ".sample"; + + @Override + public void files(SegmentInfoPerCommit info, Collection<String> files) throws IOException { + super.files(info, files); + Directory dir = info.info.dir; + String[] listAll = dir.listAll(); + for (String file : listAll) { + if (file.endsWith(FILTER)) { + if (file.startsWith(info.info.name + SEP) && file.endsWith(FILTER)) { + files.add(file); + } + } else if (file.equals(info.info.name + SAMPLE)) { + files.add(file); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-store/src/main/java/org/apache/blur/lucene/codec/Testing.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Testing.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Testing.java new file mode 100644 index 0000000..7bc1332 --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Testing.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.codec; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class Testing { + + static final Pattern CODEC_FILE_PATTERN = Pattern.compile("_[a-z0-9]+(_.*)?\\..*"); + + public static void main(String[] args) { + String t = "_0_filter1.filter"; + Matcher matcher = CODEC_FILE_PATTERN.matcher(""); + matcher.reset(t); + + if (matcher.matches()) { + System.out.println("yay!"); + } + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6607cf27/blur-store/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec ---------------------------------------------------------------------- diff --git a/blur-store/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/blur-store/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec new file mode 100644 index 0000000..fe91c2e --- /dev/null +++ b/blur-store/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.blur.lucene.codec.Blur021Codec \ No newline at end of file
