diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java index 58f9185..964c6e1 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java @@ -48,14 +48,23 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.RAMFile; +import org.apache.lucene.store.RAMInputStream; +import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LongValues; import org.apache.lucene.util.packed.BlockPackedReader; import org.apache.lucene.util.packed.MonotonicBlockPackedReader; import org.apache.lucene.util.packed.PackedInts; +import org.apache.lucene.util.packed.PackedInts.Format; +import org.apache.lucene.util.packed.PackedInts.FormatAndBits; +import org.apache.lucene.util.packed.PackedInts.ReaderIterator; +import org.apache.lucene.util.packed.PackedInts.Writer; /** reader for {@link Lucene45DocValuesFormat} */ public class Lucene45DocValuesProducer extends DocValuesProducer implements Closeable { @@ -407,6 +416,34 @@ throw new RuntimeException(e); } } + + @Override + public BytesRefIterator bulkGet(final DocIdSetIterator docIds) + throws IOException { + return new BytesRefIterator() { + private BytesRef yield = new BytesRef(); + + @Override + public BytesRef next() throws IOException { + int id = docIds.nextDoc(); + if (id == DocIdSetIterator.NO_MORE_DOCS) { + return null; + } else { + long startAddress = bytes.offset + + (id == 0 ? 0 : addresses.get(id - 1)); + long endAddress = bytes.offset + addresses.get(id); + int length = (int) (endAddress - startAddress); + // + data.seek(startAddress); + yield.offset = 0; + yield.grow(length); + data.readBytes(yield.bytes, 0, length); + yield.length = length; + } + return yield; + } + }; + } }; } diff --git a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java index 67fe0b1..4138075 100644 --- a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java +++ b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java @@ -17,7 +17,12 @@ * limitations under the License. */ +import java.io.IOException; + +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; /** * A per-document byte[] @@ -40,4 +45,25 @@ result.length = 0; } }; + /** + * the purpose is an efficient sequentual memory access + * @throws IOException if anything sad happens + * */ + public BytesRefIterator bulkGet(final DocIdSetIterator docIds) throws IOException{ + return new BytesRefIterator (){ + private BytesRef ref = new BytesRef(); + + @Override + public BytesRef next() throws IOException { + int doc = docIds.nextDoc(); + if(doc==DocIdSetIterator.NO_MORE_DOCS){ + return null; + } else { + get(doc, ref); + } + return ref; + } + + }; + } } diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java b/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java index 6ad9c63..78a8fb6 100644 --- a/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java @@ -17,14 +17,21 @@ * limitations under the License. */ +import java.io.File; +import java.io.IOException; +import java.util.Random; + import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TimeUnits; import org.apache.lucene.util._TestUtil; @@ -33,9 +40,11 @@ import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; @TimeoutSuite(millis = 80 * TimeUnits.HOUR) -@Ignore("takes ~ 45 minutes") +//@Ignore("takes ~ 45 minutes") public class Test2BBinaryDocValues extends LuceneTestCase { + private static final int NUM_OF_DOCS = 200000000; + // indexes Integer.MAX_VALUE docs with a fixed binary field public void testFixedBinary() throws Exception { BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BFixedBinary")); @@ -57,7 +66,7 @@ BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data); doc.add(dvField); - for (int i = 0; i < Integer.MAX_VALUE; i++) { + for (int i = 0; i < NUM_OF_DOCS; i++) { bytes[0] = (byte)(i >> 24); bytes[1] = (byte)(i >> 16); bytes[2] = (byte)(i >> 8); @@ -98,13 +107,14 @@ // indexes Integer.MAX_VALUE docs with a variable binary field public void testVariableBinary() throws Exception { - BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BVariableBinary")); + /*BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BVariableBinary")); if (dir instanceof MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER); - } - - IndexWriter w = new IndexWriter(dir, - new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) + }*/ + MMapDirectory mmd= new MMapDirectory(new File("./2BVariableBinary")); + final Random random = random(); + IndexWriter w = new IndexWriter(mmd, + new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH) .setRAMBufferSizeMB(256.0) .setMergeScheduler(new ConcurrentMergeScheduler()) @@ -118,7 +128,8 @@ BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data); doc.add(dvField); - for (int i = 0; i < Integer.MAX_VALUE; i++) { + w.deleteAll(); + for (int i = 0; i < NUM_OF_DOCS; i++) { encoder.reset(bytes); encoder.writeVInt(i % 65535); // 1, 2, or 3 bytes data.length = encoder.getPosition(); @@ -135,23 +146,79 @@ System.out.println("verifying..."); System.out.flush(); - DirectoryReader r = DirectoryReader.open(dir); - int expectedValue = 0; + DirectoryReader r = DirectoryReader.open(mmd); ByteArrayDataInput input = new ByteArrayDataInput(); - for (AtomicReaderContext context : r.leaves()) { - AtomicReader reader = context.reader(); - BytesRef scratch = new BytesRef(bytes); - BinaryDocValues dv = reader.getBinaryDocValues("dv"); - for (int i = 0; i < reader.maxDoc(); i++) { - dv.get(i, scratch); - input.reset(scratch.bytes, scratch.offset, scratch.length); - assertEquals(expectedValue % 65535, input.readVInt()); - assertTrue(input.eof()); - expectedValue++; + + for (int test = 0; test < 2; test++) { + + long startMs = 0; + for (int pass = 0; pass < 100; pass++) { + if(pass==10){ // skip first loops + startMs = System.currentTimeMillis(); + } + int expectedValue = 0; + int verified = 0; + + for (AtomicReaderContext context : r.leaves()) { + AtomicReader reader = context.reader(); + BytesRef scratch = new BytesRef(bytes); + BinaryDocValues dv = reader.getBinaryDocValues("dv"); + + BytesRefIterator bytesIter = null; + MockDocIdSetIterator docIds = null; + + for (int i = 0; i < reader.maxDoc();) { + if(test==0){ + if(bytesIter==null){ + docIds = new MockDocIdSetIterator(); + bytesIter = dv.bulkGet(docIds ); + } + docIds.nextDocId = i; + scratch=bytesIter.next(); + }else{ + dv.get(i, scratch); + } + input.reset(scratch.bytes, scratch.offset, scratch.length); + assertEquals(expectedValue % 65535, input.readVInt()); + assertTrue(input.eof()); + verified++; + int step = random.nextInt(3) == 0 ? random.nextInt(100) : 1; + expectedValue += step; + i += step; + } + } + if(pass%10==0 && pass/10>0){ + System.out.println("verified "+verified+" of "+NUM_OF_DOCS); + } } + System.out.println((test==0?"bulkGet":"get")+ + " took:" + (System.currentTimeMillis() - startMs) + " ms"); + } + r.close(); + mmd.close(); + } + + static class MockDocIdSetIterator extends DocIdSetIterator { + public int nextDocId; + + @Override + public int nextDoc() throws IOException { + return nextDocId; } - r.close(); - dir.close(); + @Override + public int docID() { + return nextDocId; + } + + @Override + public long cost() { + return nextDocId; + } + + @Override + public int advance(int target) throws IOException { + return slowAdvance(target); + } } }