Lucene.Net.Core.Index: Deleted obsolete files that are no longer in use
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/228b970a Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/228b970a Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/228b970a Branch: refs/heads/api-work Commit: 228b970a54a9e2749387110273f092dd3ad82328 Parents: 5816429 Author: Shad Storhaug <[email protected]> Authored: Sat Apr 1 04:04:40 2017 +0700 Committer: Shad Storhaug <[email protected]> Committed: Sat Apr 1 07:52:01 2017 +0700 ---------------------------------------------------------------------- .../Index/AbstractAllTermDocs.cs | 118 --- src/Lucene.Net.Core/Index/AllTermDocs.cs | 45 - src/Lucene.Net.Core/Index/BufferedDeletes.cs | 196 ----- src/Lucene.Net.Core/Index/ByteBlockPool.cs | 172 ---- src/Lucene.Net.Core/Index/CharBlockPool.cs | 69 -- src/Lucene.Net.Core/Index/CompoundFileReader.cs | 317 ------- src/Lucene.Net.Core/Index/CompoundFileWriter.cs | 275 ------- .../Index/DefaultSkipListReader.cs | 128 --- .../Index/DefaultSkipListWriter.cs | 143 ---- .../Index/DocConsumerPerThread.cs | 37 - .../Index/DocFieldConsumerPerThread.cs | 30 - src/Lucene.Net.Core/Index/DocFieldConsumers.cs | 221 ----- .../Index/DocFieldConsumersPerField.cs | 56 -- .../Index/DocFieldConsumersPerThread.cs | 82 -- .../Index/DocFieldProcessorPerThread.cs | 478 ----------- .../Index/DocInverterPerThread.cs | 107 --- .../Index/DocumentsWriterThreadState.cs | 56 -- .../Index/FieldReaderException.cs | 96 --- .../Index/FieldSortedTermVectorMapper.cs | 78 -- src/Lucene.Net.Core/Index/FieldsReader.cs | 641 --------------- src/Lucene.Net.Core/Index/FieldsWriter.cs | 290 ------- src/Lucene.Net.Core/Index/FilterIndexReader.cs | 388 --------- .../Index/FormatPostingsDocsConsumer.cs | 36 - .../Index/FormatPostingsDocsWriter.cs | 134 --- .../Index/FormatPostingsFieldsConsumer.cs | 39 - .../Index/FormatPostingsFieldsWriter.cs | 71 -- .../Index/FormatPostingsPositionsConsumer.cs | 32 - .../Index/FormatPostingsPositionsWriter.cs | 101 --- .../Index/FormatPostingsTermsConsumer.cs | 52 -- .../Index/FormatPostingsTermsWriter.cs | 77 -- .../Index/FreqProxFieldMergeState.cs | 117 --- .../Index/FreqProxTermsWriterPerThread.cs | 52 -- .../Index/IndexFileNameFilter.cs | 107 --- src/Lucene.Net.Core/Index/IntBlockPool.cs | 79 -- .../Index/InvertedDocConsumerPerThread.cs | 30 - .../Index/InvertedDocEndConsumerPerThread.cs | 30 - src/Lucene.Net.Core/Index/MergeDocIDRemapper.cs | 127 --- .../Index/MultiLevelSkipListReader.cs | 341 -------- .../Index/MultiLevelSkipListWriter.cs | 171 ---- .../Index/MultipleTermPositions.cs | 256 ------ src/Lucene.Net.Core/Index/NormsWriter.cs | 206 ----- .../Index/NormsWriterPerField.cs | 90 -- .../Index/NormsWriterPerThread.cs | 55 -- src/Lucene.Net.Core/Index/ParallelReader.cs | 822 ------------------- src/Lucene.Net.Core/Index/Payload.cs | 217 ----- .../Index/PositionBasedTermVectorMapper.cs | 176 ---- src/Lucene.Net.Core/Index/RawPostingList.cs | 46 -- .../Index/ReadOnlyDirectoryReader.cs | 45 - .../Index/ReadOnlySegmentReader.cs | 42 - .../Index/ReusableStringReader.cs | 136 --- src/Lucene.Net.Core/Index/SegmentMergeInfo.cs | 108 --- src/Lucene.Net.Core/Index/SegmentMergeQueue.cs | 47 -- src/Lucene.Net.Core/Index/SegmentTermDocs.cs | 282 ------- src/Lucene.Net.Core/Index/SegmentTermEnum.cs | 247 ------ .../Index/SegmentTermPositionVector.cs | 73 -- .../Index/SegmentTermPositions.cs | 226 ----- src/Lucene.Net.Core/Index/SegmentTermVector.cs | 102 --- .../Index/SortedTermVectorMapper.cs | 133 --- .../Index/StaleReaderException.cs | 49 -- src/Lucene.Net.Core/Index/StoredFieldsWriter.cs | 266 ------ .../Index/StoredFieldsWriterPerThread.cs | 93 --- src/Lucene.Net.Core/Index/TermBuffer.cs | 166 ---- src/Lucene.Net.Core/Index/TermDocs.cs | 86 -- src/Lucene.Net.Core/Index/TermEnum.cs | 53 -- src/Lucene.Net.Core/Index/TermFreqVector.cs | 73 -- src/Lucene.Net.Core/Index/TermInfo.cs | 69 -- src/Lucene.Net.Core/Index/TermInfosReader.cs | 325 -------- src/Lucene.Net.Core/Index/TermInfosWriter.cs | 250 ------ src/Lucene.Net.Core/Index/TermPositionVector.cs | 50 -- src/Lucene.Net.Core/Index/TermPositions.cs | 79 -- src/Lucene.Net.Core/Index/TermVectorEntry.cs | 108 --- .../TermVectorEntryFreqSortedComparator.cs | 45 - src/Lucene.Net.Core/Index/TermVectorMapper.cs | 112 --- .../Index/TermVectorOffsetInfo.cs | 134 --- src/Lucene.Net.Core/Index/TermVectorsReader.cs | 731 ----------------- .../Index/TermVectorsTermsWriter.cs | 380 --------- .../Index/TermVectorsTermsWriterPerField.cs | 290 ------- .../Index/TermVectorsTermsWriterPerThread.cs | 106 --- src/Lucene.Net.Core/Index/TermVectorsWriter.cs | 246 ------ .../Index/TermsHashConsumerPerThread.cs | 30 - src/Lucene.Net.Core/Index/TermsHashPerThread.cs | 140 ---- src/Lucene.Net.Core/project.json | 118 --- 82 files changed, 12927 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/AbstractAllTermDocs.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/AbstractAllTermDocs.cs b/src/Lucene.Net.Core/Index/AbstractAllTermDocs.cs deleted file mode 100644 index 935b7fa..0000000 --- a/src/Lucene.Net.Core/Index/AbstractAllTermDocs.cs +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; - -namespace Lucene.Net.Index -{ - /// <summary> - /// Base class for enumerating all but deleted docs. - /// - /// <p/>NOTE: this class is meant only to be used internally - /// by Lucene; it's only public so it can be shared across - /// packages. This means the API is freely subject to - /// change, and, the class could be removed entirely, in any - /// Lucene release. Use directly at your own risk! */ - /// </summary> - public abstract class AbstractAllTermDocs : TermDocs - { - protected int maxDoc; - protected int internalDoc = -1; - - protected AbstractAllTermDocs(int maxDoc) - { - this.maxDoc = maxDoc; - } - - public void Seek(Term term) - { - if (term == null) - { - internalDoc = -1; - } - else - { - throw new NotSupportedException(); - } - } - - public void Seek(TermEnum termEnum) - { - throw new NotSupportedException(); - } - - public int Doc - { - get { return internalDoc; } - } - - public int Freq - { - get { return 1; } - } - - public bool Next() - { - return SkipTo(internalDoc + 1); - } - - public int Read(int[] docs, int[] freqs) - { - int length = docs.Length; - int i = 0; - while (i < length && internalDoc < maxDoc) - { - if (!IsDeleted(internalDoc)) - { - docs[i] = internalDoc; - freqs[i] = 1; - ++i; - } - internalDoc++; - } - return i; - } - - public bool SkipTo(int target) - { - internalDoc = target; - while (internalDoc < maxDoc) - { - if (!IsDeleted(internalDoc)) - { - return true; - } - internalDoc++; - } - return false; - } - - public void Close() - { - Dispose(); - } - - public void Dispose() - { - Dispose(true); - } - - protected abstract void Dispose(bool disposing); - - public abstract bool IsDeleted(int doc); - } -} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/AllTermDocs.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/AllTermDocs.cs b/src/Lucene.Net.Core/Index/AllTermDocs.cs deleted file mode 100644 index a3e51ab..0000000 --- a/src/Lucene.Net.Core/Index/AllTermDocs.cs +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using BitVector = Lucene.Net.Util.BitVector; - -namespace Lucene.Net.Index -{ - - class AllTermDocs : AbstractAllTermDocs - { - protected internal BitVector deletedDocs; - - protected internal AllTermDocs(SegmentReader parent) : base(parent.MaxDoc) - { - lock (parent) - { - this.deletedDocs = parent.deletedDocs; - } - } - - protected override void Dispose(bool disposing) - { - // Do nothing. - } - - public override bool IsDeleted(int doc) - { - return deletedDocs != null && deletedDocs.Get(doc); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/BufferedDeletes.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/BufferedDeletes.cs b/src/Lucene.Net.Core/Index/BufferedDeletes.cs deleted file mode 100644 index 5b10ce7..0000000 --- a/src/Lucene.Net.Core/Index/BufferedDeletes.cs +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System.Collections.Generic; -using Lucene.Net.Search; -using Lucene.Net.Support; - -namespace Lucene.Net.Index -{ - - /// <summary>Holds buffered deletes, by docID, term or query. We - /// hold two instances of this class: one for the deletes - /// prior to the last flush, the other for deletes after - /// the last flush. This is so if we need to abort - /// (discard all buffered docs) we can also discard the - /// buffered deletes yet keep the deletes done during - /// previously flushed segments. - /// </summary> - class BufferedDeletes - { - internal int numTerms; - internal IDictionary<Term,Num> terms = null; - internal IDictionary<Query, int> queries = new HashMap<Query, int>(); - internal List<int> docIDs = new List<int>(); - internal long bytesUsed; - internal bool doTermSort; - - public BufferedDeletes(bool doTermSort) - { - this.doTermSort = doTermSort; - if (doTermSort) - { - //TODO: Used in place of TreeMap - terms = new SortedDictionary<Term, Num>(); - } - else - { - terms = new HashMap<Term, Num>(); - } - } - - - // Number of documents a delete term applies to. - internal sealed class Num - { - internal int num; - - internal Num(int num) - { - this.num = num; - } - - internal int GetNum() - { - return num; - } - - internal void SetNum(int num) - { - // Only record the new number if it's greater than the - // current one. This is important because if multiple - // threads are replacing the same doc at nearly the - // same time, it's possible that one thread that got a - // higher docID is scheduled before the other - // threads. - if (num > this.num) - this.num = num; - } - } - - internal virtual int Size() - { - // We use numTerms not terms.size() intentionally, so - // that deletes by the same term multiple times "count", - // ie if you ask to flush every 1000 deletes then even - // dup'd terms are counted towards that 1000 - return numTerms + queries.Count + docIDs.Count; - } - - internal virtual void Update(BufferedDeletes @in) - { - numTerms += @in.numTerms; - bytesUsed += @in.bytesUsed; - foreach (KeyValuePair<Term, Num> term in @in.terms) - { - terms[term.Key] = term.Value; - } - foreach (KeyValuePair<Query, int> term in @in.queries) - { - queries[term.Key] = term.Value; - } - - docIDs.AddRange(@in.docIDs); - @in.Clear(); - } - - internal virtual void Clear() - { - terms.Clear(); - queries.Clear(); - docIDs.Clear(); - numTerms = 0; - bytesUsed = 0; - } - - internal virtual void AddBytesUsed(long b) - { - bytesUsed += b; - } - - internal virtual bool Any() - { - return terms.Count > 0 || docIDs.Count > 0 || queries.Count > 0; - } - - // Remaps all buffered deletes based on a completed - // merge - internal virtual void Remap(MergeDocIDRemapper mapper, SegmentInfos infos, int[][] docMaps, int[] delCounts, MergePolicy.OneMerge merge, int mergeDocCount) - { - lock (this) - { - IDictionary<Term, Num> newDeleteTerms; - - // Remap delete-by-term - if (terms.Count > 0) - { - if (doTermSort) - { - newDeleteTerms = new SortedDictionary<Term, Num>(); - } - else - { - newDeleteTerms = new HashMap<Term, Num>(); - } - foreach(var entry in terms) - { - Num num = entry.Value; - newDeleteTerms[entry.Key] = new Num(mapper.Remap(num.GetNum())); - } - } - else - newDeleteTerms = null; - - // Remap delete-by-docID - List<int> newDeleteDocIDs; - - if (docIDs.Count > 0) - { - newDeleteDocIDs = new List<int>(docIDs.Count); - foreach(int num in docIDs) - { - newDeleteDocIDs.Add(mapper.Remap(num)); - } - } - else - newDeleteDocIDs = null; - - // Remap delete-by-query - HashMap<Query, int> newDeleteQueries; - - if (queries.Count > 0) - { - newDeleteQueries = new HashMap<Query, int>(queries.Count); - foreach(var entry in queries) - { - int num = entry.Value; - newDeleteQueries[entry.Key] = mapper.Remap(num); - } - } - else - newDeleteQueries = null; - - if (newDeleteTerms != null) - terms = newDeleteTerms; - if (newDeleteDocIDs != null) - docIDs = newDeleteDocIDs; - if (newDeleteQueries != null) - queries = newDeleteQueries; - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/ByteBlockPool.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/ByteBlockPool.cs b/src/Lucene.Net.Core/Index/ByteBlockPool.cs deleted file mode 100644 index 50afde0..0000000 --- a/src/Lucene.Net.Core/Index/ByteBlockPool.cs +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* Class that Posting and PostingVector use to write byte -* streams into shared fixed-size byte[] arrays. The idea -* is to allocate slices of increasing lengths For -* example, the first slice is 5 bytes, the next slice is -* 14, etc. We start by writing our bytes into the first -* 5 bytes. When we hit the end of the slice, we allocate -* the next slice and then write the address of the new -* slice into the last 4 bytes of the previous slice (the -* "forwarding address"). -* -* Each slice is filled with 0's initially, and we mark -* the end with a non-zero byte. This way the methods -* that are writing into the slice don't need to record -* its length and instead allocate a new slice once they -* hit a non-zero byte. */ - -using System; -using System.Collections.Generic; -using Lucene.Net.Support; - -namespace Lucene.Net.Index -{ - - sealed public class ByteBlockPool - { - private void InitBlock() - { - byteUpto = DocumentsWriter.BYTE_BLOCK_SIZE; - } - - public /*internal*/ abstract class Allocator - { - public /*internal*/ abstract void RecycleByteBlocks(byte[][] blocks, int start, int end); - public /*internal*/ abstract void RecycleByteBlocks(IList<byte[]> blocks); - public /*internal*/ abstract byte[] GetByteBlock(bool trackAllocations); - } - - public byte[][] buffers = new byte[10][]; - - internal int bufferUpto = - 1; // Which buffer we are upto - public int byteUpto; // Where we are in head buffer - - public byte[] buffer; // Current head buffer - public int byteOffset = - DocumentsWriter.BYTE_BLOCK_SIZE; // Current head offset - - private readonly bool trackAllocations; - private readonly Allocator allocator; - - public ByteBlockPool(Allocator allocator, bool trackAllocations) - { - InitBlock(); - this.allocator = allocator; - this.trackAllocations = trackAllocations; - } - - public void Reset() - { - if (bufferUpto != - 1) - { - // We allocated at least one buffer - - for (int i = 0; i < bufferUpto; i++) - // Fully zero fill buffers that we fully used - System.Array.Clear(buffers[i], 0, buffers[i].Length); - - // Partial zero fill the final buffer - System.Array.Clear(buffers[bufferUpto], 0, byteUpto); - - if (bufferUpto > 0) - // Recycle all but the first buffer - allocator.RecycleByteBlocks(buffers, 1, 1 + bufferUpto); - - // Re-use the first buffer - bufferUpto = 0; - byteUpto = 0; - byteOffset = 0; - buffer = buffers[0]; - } - } - - public void NextBuffer() - { - if (1 + bufferUpto == buffers.Length) - { - var newBuffers = new byte[(int) (buffers.Length * 1.5)][]; - Array.Copy(buffers, 0, newBuffers, 0, buffers.Length); - buffers = newBuffers; - } - buffer = buffers[1 + bufferUpto] = allocator.GetByteBlock(trackAllocations); - bufferUpto++; - - byteUpto = 0; - byteOffset += DocumentsWriter.BYTE_BLOCK_SIZE; - } - - public int NewSlice(int size) - { - if (byteUpto > DocumentsWriter.BYTE_BLOCK_SIZE - size) - NextBuffer(); - int upto = byteUpto; - byteUpto += size; - buffer[byteUpto - 1] = 16; - return upto; - } - - // Size of each slice. These arrays should be at most 16 - // elements (index is encoded with 4 bits). First array - // is just a compact way to encode X+1 with a max. Second - // array is the length of each slice, ie first slice is 5 - // bytes, next slice is 14 bytes, etc. - internal static readonly int[] nextLevelArray = new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 9}; - internal static readonly int[] levelSizeArray = new int[]{5, 14, 20, 30, 40, 40, 80, 80, 120, 200}; - internal static readonly int FIRST_LEVEL_SIZE = levelSizeArray[0]; - public readonly static int FIRST_LEVEL_SIZE_For_NUnit_Test = levelSizeArray[0]; - - public int AllocSlice(byte[] slice, int upto) - { - - int level = slice[upto] & 15; - int newLevel = nextLevelArray[level]; - int newSize = levelSizeArray[newLevel]; - - // Maybe allocate another block - if (byteUpto > DocumentsWriter.BYTE_BLOCK_SIZE - newSize) - NextBuffer(); - - int newUpto = byteUpto; - int offset = newUpto + byteOffset; - byteUpto += newSize; - - // Copy forward the past 3 bytes (which we are about - // to overwrite with the forwarding address): - buffer[newUpto] = slice[upto - 3]; - buffer[newUpto + 1] = slice[upto - 2]; - buffer[newUpto + 2] = slice[upto - 1]; - - // Write forwarding address at end of last slice: - slice[upto - 3] = (byte) (Number.URShift(offset, 24)); - slice[upto - 2] = (byte) (Number.URShift(offset, 16)); - slice[upto - 1] = (byte) (Number.URShift(offset, 8)); - slice[upto] = (byte) offset; - - // Write new level: - buffer[byteUpto - 1] = (byte) (16 | newLevel); - - return newUpto + 3; - } - - public static int FIRST_LEVEL_SIZE_ForNUnit - { - get { return FIRST_LEVEL_SIZE; } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/CharBlockPool.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/CharBlockPool.cs b/src/Lucene.Net.Core/Index/CharBlockPool.cs deleted file mode 100644 index 9a35743..0000000 --- a/src/Lucene.Net.Core/Index/CharBlockPool.cs +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; - -namespace Lucene.Net.Index -{ - - sealed class CharBlockPool - { - private void InitBlock() - { - charUpto = DocumentsWriter.CHAR_BLOCK_SIZE; - } - - public char[][] buffers = new char[10][]; - internal int numBuffer; - - internal int bufferUpto = - 1; // Which buffer we are upto - public int charUpto; // Where we are in head buffer - - public char[] buffer; // Current head buffer - public int charOffset = - DocumentsWriter.CHAR_BLOCK_SIZE; // Current head offset - private readonly DocumentsWriter docWriter; - - public CharBlockPool(DocumentsWriter docWriter) - { - InitBlock(); - this.docWriter = docWriter; - } - - public void Reset() - { - docWriter.RecycleCharBlocks(buffers, 1 + bufferUpto); - bufferUpto = - 1; - charUpto = DocumentsWriter.CHAR_BLOCK_SIZE; - charOffset = - DocumentsWriter.CHAR_BLOCK_SIZE; - } - - public void NextBuffer() - { - if (1 + bufferUpto == buffers.Length) - { - var newBuffers = new char[(int) (buffers.Length * 1.5)][]; - Array.Copy(buffers, 0, newBuffers, 0, buffers.Length); - buffers = newBuffers; - } - buffer = buffers[1 + bufferUpto] = docWriter.GetCharBlock(); - bufferUpto++; - - charUpto = 0; - charOffset += DocumentsWriter.CHAR_BLOCK_SIZE; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/CompoundFileReader.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/CompoundFileReader.cs b/src/Lucene.Net.Core/Index/CompoundFileReader.cs deleted file mode 100644 index 3b7f4f3..0000000 --- a/src/Lucene.Net.Core/Index/CompoundFileReader.cs +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System.Linq; -using Lucene.Net.Support; -using BufferedIndexInput = Lucene.Net.Store.BufferedIndexInput; -using Directory = Lucene.Net.Store.Directory; -using IndexInput = Lucene.Net.Store.IndexInput; -using IndexOutput = Lucene.Net.Store.IndexOutput; -using Lock = Lucene.Net.Store.Lock; - -namespace Lucene.Net.Index -{ - - - /// <summary> Class for accessing a compound stream. - /// This class implements a directory, but is limited to only read operations. - /// Directory methods that would normally modify data throw an exception. - /// </summary> - public class CompoundFileReader : Directory - { - - private readonly int readBufferSize; - - private sealed class FileEntry - { - internal long offset; - internal long length; - } - - private bool isDisposed; - - // Base info - private readonly Directory directory; - private readonly System.String fileName; - - private IndexInput stream; - private HashMap<string, FileEntry> entries = new HashMap<string, FileEntry>(); - - - public CompoundFileReader(Directory dir, System.String name):this(dir, name, BufferedIndexInput.BUFFER_SIZE) - { - } - - public CompoundFileReader(Directory dir, System.String name, int readBufferSize) - { - directory = dir; - fileName = name; - this.readBufferSize = readBufferSize; - - bool success = false; - - try - { - stream = dir.OpenInput(name, readBufferSize); - - // read the directory and init files - int count = stream.ReadVInt(); - FileEntry entry = null; - for (int i = 0; i < count; i++) - { - long offset = stream.ReadLong(); - System.String id = stream.ReadString(); - - if (entry != null) - { - // set length of the previous entry - entry.length = offset - entry.offset; - } - - entry = new FileEntry {offset = offset}; - entries[id] = entry; - } - - // set the length of the final entry - if (entry != null) - { - entry.length = stream.Length() - entry.offset; - } - - success = true; - } - finally - { - if (!success && (stream != null)) - { - try - { - stream.Close(); - } - catch (System.IO.IOException) - { - } - } - } - } - - public virtual Directory Directory - { - get { return directory; } - } - - public virtual string Name - { - get { return fileName; } - } - - protected override void Dispose(bool disposing) - { - lock (this) - { - if (isDisposed) return; - if (disposing) - { - if (entries != null) - { - entries.Clear(); - } - if (stream != null) - { - stream.Close(); - } - } - - entries = null; - stream = null; - isDisposed = true; - } - } - - public override IndexInput OpenInput(System.String id) - { - lock (this) - { - // Default to readBufferSize passed in when we were opened - return OpenInput(id, readBufferSize); - } - } - - public override IndexInput OpenInput(System.String id, int readBufferSize) - { - lock (this) - { - if (stream == null) - throw new System.IO.IOException("Stream closed"); - - FileEntry entry = entries[id]; - if (entry == null) - throw new System.IO.IOException("No sub-file with id " + id + " found"); - - return new CSIndexInput(stream, entry.offset, entry.length, readBufferSize); - } - } - - /// <summary>Returns an array of strings, one for each file in the directory. </summary> - public override System.String[] ListAll() - { - return entries.Keys.ToArray(); - } - - /// <summary>Returns true iff a file with the given name exists. </summary> - public override bool FileExists(System.String name) - { - return entries.ContainsKey(name); - } - - /// <summary>Returns the time the compound file was last modified. </summary> - public override long FileModified(System.String name) - { - return directory.FileModified(fileName); - } - - /// <summary>Set the modified time of the compound file to now. </summary> - public override void TouchFile(System.String name) - { - directory.TouchFile(fileName); - } - - /// <summary>Not implemented</summary> - /// <throws> UnsupportedOperationException </throws> - public override void DeleteFile(System.String name) - { - throw new System.NotSupportedException(); - } - - /// <summary>Not implemented</summary> - /// <throws> UnsupportedOperationException </throws> - public void RenameFile(System.String from, System.String to) - { - throw new System.NotSupportedException(); - } - - /// <summary>Returns the length of a file in the directory.</summary> - /// <throws> IOException if the file does not exist </throws> - public override long FileLength(System.String name) - { - FileEntry e = entries[name]; - if (e == null) - throw new System.IO.IOException("File " + name + " does not exist"); - return e.length; - } - - /// <summary>Not implemented</summary> - /// <throws> UnsupportedOperationException </throws> - public override IndexOutput CreateOutput(System.String name) - { - throw new System.NotSupportedException(); - } - - /// <summary>Not implemented</summary> - /// <throws> UnsupportedOperationException </throws> - public override Lock MakeLock(System.String name) - { - throw new System.NotSupportedException(); - } - - /// <summary>Implementation of an IndexInput that reads from a portion of the - /// compound file. The visibility is left as "package" *only* because - /// this helps with testing since JUnit test cases in a different class - /// can then access package fields of this class. - /// </summary> - public /*internal*/ sealed class CSIndexInput : BufferedIndexInput - { - internal IndexInput base_Renamed; - internal long fileOffset; - internal long length; - - private bool isDisposed; - - internal CSIndexInput(IndexInput @base, long fileOffset, long length):this(@base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE) - { - } - - internal CSIndexInput(IndexInput @base, long fileOffset, long length, int readBufferSize):base(readBufferSize) - { - this.base_Renamed = (IndexInput) @base.Clone(); - this.fileOffset = fileOffset; - this.length = length; - } - - public override System.Object Clone() - { - var clone = (CSIndexInput) base.Clone(); - clone.base_Renamed = (IndexInput) base_Renamed.Clone(); - clone.fileOffset = fileOffset; - clone.length = length; - return clone; - } - - /// <summary>Expert: implements buffer refill. Reads bytes from the current - /// position in the input. - /// </summary> - /// <param name="b">the array to read bytes into - /// </param> - /// <param name="offset">the offset in the array to start storing bytes - /// </param> - /// <param name="len">the number of bytes to read - /// </param> - public override void ReadInternal(byte[] b, int offset, int len) - { - long start = FilePointer; - if (start + len > length) - throw new System.IO.IOException("read past EOF"); - base_Renamed.Seek(fileOffset + start); - base_Renamed.ReadBytes(b, offset, len, false); - } - - /// <summary>Expert: implements seek. Sets current position in this file, where - /// the next <see cref="ReadInternal(byte[],int,int)" /> will occur. - /// </summary> - /// <seealso cref="ReadInternal(byte[],int,int)"> - /// </seealso> - public override void SeekInternal(long pos) - { - } - - protected override void Dispose(bool disposing) - { - if (isDisposed) return; - - if (disposing) - { - if (base_Renamed != null) - { - base_Renamed.Close(); - } - } - - isDisposed = true; - } - - public override long Length() - { - return length; - } - - public IndexInput base_Renamed_ForNUnit - { - get { return base_Renamed; } - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/CompoundFileWriter.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/CompoundFileWriter.cs b/src/Lucene.Net.Core/Index/CompoundFileWriter.cs deleted file mode 100644 index ff25487..0000000 --- a/src/Lucene.Net.Core/Index/CompoundFileWriter.cs +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; -using System.Collections.Generic; -using Directory = Lucene.Net.Store.Directory; -using IndexInput = Lucene.Net.Store.IndexInput; -using IndexOutput = Lucene.Net.Store.IndexOutput; - -namespace Lucene.Net.Index -{ - - - /// <summary> Combines multiple files into a single compound file. - /// The file format:<br/> - /// <list type="bullet"> - /// <item>VInt fileCount</item> - /// <item>{Directory} - /// fileCount entries with the following structure:</item> - /// <list type="bullet"> - /// <item>long dataOffset</item> - /// <item>String fileName</item> - /// </list> - /// <item>{File Data} - /// fileCount entries with the raw data of the corresponding file</item> - /// </list> - /// - /// The fileCount integer indicates how many files are contained in this compound - /// file. The {directory} that follows has that many entries. Each directory entry - /// contains a long pointer to the start of this file's data section, and a String - /// with that file's name. - /// </summary> - public sealed class CompoundFileWriter : IDisposable - { - - private sealed class FileEntry - { - /// <summary>source file </summary> - internal System.String file; - - /// <summary>temporary holder for the start of directory entry for this file </summary> - internal long directoryOffset; - - /// <summary>temporary holder for the start of this file's data section </summary> - internal long dataOffset; - } - - - private readonly Directory directory; - private readonly String fileName; - private readonly HashSet<string> ids; - private readonly LinkedList<FileEntry> entries; - private bool merged = false; - private readonly SegmentMerger.CheckAbort checkAbort; - - /// <summary>Create the compound stream in the specified file. The file name is the - /// entire name (no extensions are added). - /// </summary> - /// <throws> NullPointerException if <c>dir</c> or <c>name</c> is null </throws> - public CompoundFileWriter(Directory dir, System.String name):this(dir, name, null) - { - } - - internal CompoundFileWriter(Directory dir, System.String name, SegmentMerger.CheckAbort checkAbort) - { - if (dir == null) - throw new ArgumentNullException("dir"); - if (name == null) - throw new ArgumentNullException("name"); - this.checkAbort = checkAbort; - directory = dir; - fileName = name; - ids = new HashSet<string>(); - entries = new LinkedList<FileEntry>(); - } - - /// <summary>Returns the directory of the compound file. </summary> - public Directory Directory - { - get { return directory; } - } - - /// <summary>Returns the name of the compound file. </summary> - public string Name - { - get { return fileName; } - } - - /// <summary>Add a source stream. <c>file</c> is the string by which the - /// sub-stream will be known in the compound stream. - /// - /// </summary> - /// <throws> IllegalStateException if this writer is closed </throws> - /// <throws> NullPointerException if <c>file</c> is null </throws> - /// <throws> IllegalArgumentException if a file with the same name </throws> - /// <summary> has been added already - /// </summary> - public void AddFile(String file) - { - if (merged) - throw new InvalidOperationException("Can't add extensions after merge has been called"); - - if (file == null) - throw new ArgumentNullException("file"); - - try - { - ids.Add(file); - } - catch (Exception) - { - throw new ArgumentException("File " + file + " already added"); - } - - var entry = new FileEntry {file = file}; - entries.AddLast(entry); - } - - [Obsolete("Use Dispose() instead")] - public void Close() - { - Dispose(); - } - - /// <summary>Merge files with the extensions added up to now. - /// All files with these extensions are combined sequentially into the - /// compound stream. After successful merge, the source files - /// are deleted. - /// </summary> - /// <throws> IllegalStateException if close() had been called before or </throws> - /// <summary> if no file has been added to this object - /// </summary> - public void Dispose() - { - // Extract into protected method if class ever becomes unsealed - - // TODO: Dispose shouldn't throw exceptions! - if (merged) - throw new SystemException("Merge already performed"); - - if ((entries.Count == 0)) - throw new SystemException("No entries to merge have been defined"); - - merged = true; - - // open the compound stream - IndexOutput os = null; - try - { - os = directory.CreateOutput(fileName); - - // Write the number of entries - os.WriteVInt(entries.Count); - - // Write the directory with all offsets at 0. - // Remember the positions of directory entries so that we can - // adjust the offsets later - long totalSize = 0; - foreach (FileEntry fe in entries) - { - fe.directoryOffset = os.FilePointer; - os.WriteLong(0); // for now - os.WriteString(fe.file); - totalSize += directory.FileLength(fe.file); - } - - // Pre-allocate size of file as optimization -- - // this can potentially help IO performance as - // we write the file and also later during - // searching. It also uncovers a disk-full - // situation earlier and hopefully without - // actually filling disk to 100%: - long finalLength = totalSize + os.FilePointer; - os.SetLength(finalLength); - - // Open the files and copy their data into the stream. - // Remember the locations of each file's data section. - var buffer = new byte[16384]; - foreach (FileEntry fe in entries) - { - fe.dataOffset = os.FilePointer; - CopyFile(fe, os, buffer); - } - - // Write the data offsets into the directory of the compound stream - foreach (FileEntry fe in entries) - { - os.Seek(fe.directoryOffset); - os.WriteLong(fe.dataOffset); - } - - System.Diagnostics.Debug.Assert(finalLength == os.Length); - - // Close the output stream. Set the os to null before trying to - // close so that if an exception occurs during the close, the - // finally clause below will not attempt to close the stream - // the second time. - IndexOutput tmp = os; - os = null; - tmp.Close(); - } - finally - { - if (os != null) - try - { - os.Close(); - } - catch (System.IO.IOException) - { - } - } - } - - - /// <summary>Copy the contents of the file with specified extension into the - /// provided output stream. Use the provided buffer for moving data - /// to reduce memory allocation. - /// </summary> - private void CopyFile(FileEntry source, IndexOutput os, byte[] buffer) - { - IndexInput isRenamed = null; - try - { - long startPtr = os.FilePointer; - - isRenamed = directory.OpenInput(source.file); - long length = isRenamed.Length(); - long remainder = length; - int chunk = buffer.Length; - - while (remainder > 0) - { - var len = (int) Math.Min(chunk, remainder); - isRenamed.ReadBytes(buffer, 0, len, false); - os.WriteBytes(buffer, len); - remainder -= len; - if (checkAbort != null) - // Roughly every 2 MB we will check if - // it's time to abort - checkAbort.Work(80); - } - - // Verify that remainder is 0 - if (remainder != 0) - throw new System.IO.IOException("Non-zero remainder length after copying: " + remainder + " (id: " + source.file + ", length: " + length + ", buffer size: " + chunk + ")"); - - // Verify that the output length diff is equal to original file - long endPtr = os.FilePointer; - long diff = endPtr - startPtr; - if (diff != length) - throw new System.IO.IOException("Difference in the output file offsets " + diff + " does not match the original file length " + length); - } - finally - { - if (isRenamed != null) - isRenamed.Close(); - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/DefaultSkipListReader.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/DefaultSkipListReader.cs b/src/Lucene.Net.Core/Index/DefaultSkipListReader.cs deleted file mode 100644 index 470df0d..0000000 --- a/src/Lucene.Net.Core/Index/DefaultSkipListReader.cs +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using Lucene.Net.Support; -using IndexInput = Lucene.Net.Store.IndexInput; - -namespace Lucene.Net.Index -{ - - /// <summary> Implements the skip list reader for the default posting list format - /// that stores positions and payloads. - /// - /// </summary> - class DefaultSkipListReader:MultiLevelSkipListReader - { - private bool currentFieldStoresPayloads; - private readonly long[] freqPointer; - private readonly long[] proxPointer; - private readonly int[] payloadLength; - - private long lastFreqPointer; - private long lastProxPointer; - private int lastPayloadLength; - - - internal DefaultSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval):base(skipStream, maxSkipLevels, skipInterval) - { - freqPointer = new long[maxSkipLevels]; - proxPointer = new long[maxSkipLevels]; - payloadLength = new int[maxSkipLevels]; - } - - internal virtual void Init(long skipPointer, long freqBasePointer, long proxBasePointer, int df, bool storesPayloads) - { - base.Init(skipPointer, df); - this.currentFieldStoresPayloads = storesPayloads; - lastFreqPointer = freqBasePointer; - lastProxPointer = proxBasePointer; - - for (int i = 0; i < freqPointer.Length; i++) freqPointer[i] = freqBasePointer; - for (int i = 0; i < proxPointer.Length; i++) proxPointer[i] = proxBasePointer; - for (int i = 0; i < payloadLength.Length; i++) payloadLength[i] = 0; - } - - /// <summary>Returns the freq pointer of the doc to which the last call of - /// <see cref="MultiLevelSkipListReader.SkipTo(int)" /> has skipped. - /// </summary> - internal virtual long GetFreqPointer() - { - return lastFreqPointer; - } - - /// <summary>Returns the prox pointer of the doc to which the last call of - /// <see cref="MultiLevelSkipListReader.SkipTo(int)" /> has skipped. - /// </summary> - internal virtual long GetProxPointer() - { - return lastProxPointer; - } - - /// <summary>Returns the payload length of the payload stored just before - /// the doc to which the last call of <see cref="MultiLevelSkipListReader.SkipTo(int)" /> - /// has skipped. - /// </summary> - internal virtual int GetPayloadLength() - { - return lastPayloadLength; - } - - protected internal override void SeekChild(int level) - { - base.SeekChild(level); - freqPointer[level] = lastFreqPointer; - proxPointer[level] = lastProxPointer; - payloadLength[level] = lastPayloadLength; - } - - protected internal override void SetLastSkipData(int level) - { - base.SetLastSkipData(level); - lastFreqPointer = freqPointer[level]; - lastProxPointer = proxPointer[level]; - lastPayloadLength = payloadLength[level]; - } - - - protected internal override int ReadSkipData(int level, IndexInput skipStream) - { - int delta; - if (currentFieldStoresPayloads) - { - // the current field stores payloads. - // if the doc delta is odd then we have - // to read the current payload length - // because it differs from the length of the - // previous payload - delta = skipStream.ReadVInt(); - if ((delta & 1) != 0) - { - payloadLength[level] = skipStream.ReadVInt(); - } - delta = Number.URShift(delta, 1); - } - else - { - delta = skipStream.ReadVInt(); - } - freqPointer[level] += skipStream.ReadVInt(); - proxPointer[level] += skipStream.ReadVInt(); - - return delta; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/DefaultSkipListWriter.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/DefaultSkipListWriter.cs b/src/Lucene.Net.Core/Index/DefaultSkipListWriter.cs deleted file mode 100644 index 1c2de1b..0000000 --- a/src/Lucene.Net.Core/Index/DefaultSkipListWriter.cs +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; - -using IndexOutput = Lucene.Net.Store.IndexOutput; - -namespace Lucene.Net.Index -{ - - - /// <summary> Implements the skip list writer for the default posting list format - /// that stores positions and payloads. - /// - /// </summary> - class DefaultSkipListWriter:MultiLevelSkipListWriter - { - private int[] lastSkipDoc; - private int[] lastSkipPayloadLength; - private long[] lastSkipFreqPointer; - private long[] lastSkipProxPointer; - - private IndexOutput freqOutput; - private IndexOutput proxOutput; - - private int curDoc; - private bool curStorePayloads; - private int curPayloadLength; - private long curFreqPointer; - private long curProxPointer; - - internal DefaultSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, IndexOutput freqOutput, IndexOutput proxOutput):base(skipInterval, numberOfSkipLevels, docCount) - { - this.freqOutput = freqOutput; - this.proxOutput = proxOutput; - - lastSkipDoc = new int[numberOfSkipLevels]; - lastSkipPayloadLength = new int[numberOfSkipLevels]; - lastSkipFreqPointer = new long[numberOfSkipLevels]; - lastSkipProxPointer = new long[numberOfSkipLevels]; - } - - internal virtual void SetFreqOutput(IndexOutput freqOutput) - { - this.freqOutput = freqOutput; - } - - internal virtual void SetProxOutput(IndexOutput proxOutput) - { - this.proxOutput = proxOutput; - } - - /// <summary> Sets the values for the current skip data. </summary> - internal virtual void SetSkipData(int doc, bool storePayloads, int payloadLength) - { - this.curDoc = doc; - this.curStorePayloads = storePayloads; - this.curPayloadLength = payloadLength; - this.curFreqPointer = freqOutput.FilePointer; - if (proxOutput != null) - this.curProxPointer = proxOutput.FilePointer; - } - - protected internal override void ResetSkip() - { - base.ResetSkip(); - for (int i = 0; i < lastSkipDoc.Length; i++) lastSkipDoc[i] = 0; - for (int i = 0; i < lastSkipPayloadLength.Length; i++) lastSkipPayloadLength[i] = -1; // we don't have to write the first length in the skip list - for (int i = 0; i < lastSkipFreqPointer.Length; i++) lastSkipFreqPointer[i] = freqOutput.FilePointer; - if (proxOutput != null) - for (int i = 0; i < lastSkipProxPointer.Length; i++) lastSkipProxPointer[i] = proxOutput.FilePointer; - } - - protected internal override void WriteSkipData(int level, IndexOutput skipBuffer) - { - // To efficiently store payloads in the posting lists we do not store the length of - // every payload. Instead we omit the length for a payload if the previous payload had - // the same length. - // However, in order to support skipping the payload length at every skip point must be known. - // So we use the same length encoding that we use for the posting lists for the skip data as well: - // Case 1: current field does not store payloads - // SkipDatum --> DocSkip, FreqSkip, ProxSkip - // DocSkip,FreqSkip,ProxSkip --> VInt - // DocSkip records the document number before every SkipInterval th document in TermFreqs. - // Document numbers are represented as differences from the previous value in the sequence. - // Case 2: current field stores payloads - // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip - // DocSkip,FreqSkip,ProxSkip --> VInt - // PayloadLength --> VInt - // In this case DocSkip/2 is the difference between - // the current and the previous value. If DocSkip - // is odd, then a PayloadLength encoded as VInt follows, - // if DocSkip is even, then it is assumed that the - // current payload length equals the length at the previous - // skip point - if (curStorePayloads) - { - int delta = curDoc - lastSkipDoc[level]; - if (curPayloadLength == lastSkipPayloadLength[level]) - { - // the current payload length equals the length at the previous skip point, - // so we don't store the length again - skipBuffer.WriteVInt(delta * 2); - } - else - { - // the payload length is different from the previous one. We shift the DocSkip, - // set the lowest bit and store the current payload length as VInt. - skipBuffer.WriteVInt(delta * 2 + 1); - skipBuffer.WriteVInt(curPayloadLength); - lastSkipPayloadLength[level] = curPayloadLength; - } - } - else - { - // current field does not store payloads - skipBuffer.WriteVInt(curDoc - lastSkipDoc[level]); - } - skipBuffer.WriteVInt((int) (curFreqPointer - lastSkipFreqPointer[level])); - skipBuffer.WriteVInt((int) (curProxPointer - lastSkipProxPointer[level])); - - lastSkipDoc[level] = curDoc; - //System.out.println("write doc at level " + level + ": " + curDoc); - - lastSkipFreqPointer[level] = curFreqPointer; - lastSkipProxPointer[level] = curProxPointer; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/DocConsumerPerThread.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/DocConsumerPerThread.cs b/src/Lucene.Net.Core/Index/DocConsumerPerThread.cs deleted file mode 100644 index d92457d..0000000 --- a/src/Lucene.Net.Core/Index/DocConsumerPerThread.cs +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; - -namespace Lucene.Net.Index -{ - - abstract class DocConsumerPerThread - { - - /// <summary>Process the document. If there is - /// something for this document to be done in docID order, - /// you should encapsulate that as a - /// DocumentsWriter.DocWriter and return it. - /// DocumentsWriter then calls finish() on this object - /// when it's its turn. - /// </summary> - public abstract DocumentsWriter.DocWriter ProcessDocument(); - - public abstract void Abort(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/DocFieldConsumerPerThread.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/DocFieldConsumerPerThread.cs b/src/Lucene.Net.Core/Index/DocFieldConsumerPerThread.cs deleted file mode 100644 index 1b8b58f..0000000 --- a/src/Lucene.Net.Core/Index/DocFieldConsumerPerThread.cs +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; - -namespace Lucene.Net.Index -{ - - abstract class DocFieldConsumerPerThread - { - public abstract void StartDocument(); - public abstract DocumentsWriter.DocWriter FinishDocument(); - public abstract DocFieldConsumerPerField AddField(FieldInfo fi); - public abstract void Abort(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/DocFieldConsumers.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/DocFieldConsumers.cs b/src/Lucene.Net.Core/Index/DocFieldConsumers.cs deleted file mode 100644 index 11db7ec..0000000 --- a/src/Lucene.Net.Core/Index/DocFieldConsumers.cs +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; -using System.Collections.Generic; -using Lucene.Net.Support; -using ArrayUtil = Lucene.Net.Util.ArrayUtil; - -namespace Lucene.Net.Index -{ - - /// <summary>This is just a "splitter" class: it lets you wrap two - /// DocFieldConsumer instances as a single consumer. - /// </summary> - - sealed class DocFieldConsumers : DocFieldConsumer - { - private void InitBlock() - { - docFreeList = new PerDoc[1]; - } - internal DocFieldConsumer one; - internal DocFieldConsumer two; - - public DocFieldConsumers(DocFieldConsumer one, DocFieldConsumer two) - { - InitBlock(); - this.one = one; - this.two = two; - } - - internal override void SetFieldInfos(FieldInfos fieldInfos) - { - base.SetFieldInfos(fieldInfos); - one.SetFieldInfos(fieldInfos); - two.SetFieldInfos(fieldInfos); - } - - public override void Flush(IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> threadsAndFields, SegmentWriteState state) - { - - var oneThreadsAndFields = new HashMap<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>(); - var twoThreadsAndFields = new HashMap<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>(); - - foreach(var entry in threadsAndFields) - { - DocFieldConsumersPerThread perThread = (DocFieldConsumersPerThread) entry.Key; - ICollection<DocFieldConsumerPerField> fields = entry.Value; - - IEnumerator<DocFieldConsumerPerField> fieldsIt = fields.GetEnumerator(); - ICollection<DocFieldConsumerPerField> oneFields = new HashSet<DocFieldConsumerPerField>(); - ICollection<DocFieldConsumerPerField> twoFields = new HashSet<DocFieldConsumerPerField>(); - while (fieldsIt.MoveNext()) - { - DocFieldConsumersPerField perField = (DocFieldConsumersPerField) fieldsIt.Current; - oneFields.Add(perField.one); - twoFields.Add(perField.two); - } - - oneThreadsAndFields[perThread.one] = oneFields; - twoThreadsAndFields[perThread.two] = twoFields; - } - - - one.Flush(oneThreadsAndFields, state); - two.Flush(twoThreadsAndFields, state); - } - - public override void CloseDocStore(SegmentWriteState state) - { - try - { - one.CloseDocStore(state); - } - finally - { - two.CloseDocStore(state); - } - } - - public override void Abort() - { - try - { - one.Abort(); - } - finally - { - two.Abort(); - } - } - - public override bool FreeRAM() - { - bool any = one.FreeRAM(); - any |= two.FreeRAM(); - return any; - } - - public override DocFieldConsumerPerThread AddThread(DocFieldProcessorPerThread docFieldProcessorPerThread) - { - return new DocFieldConsumersPerThread(docFieldProcessorPerThread, this, one.AddThread(docFieldProcessorPerThread), two.AddThread(docFieldProcessorPerThread)); - } - - internal PerDoc[] docFreeList; - internal int freeCount; - internal int allocCount; - - internal PerDoc GetPerDoc() - { - lock (this) - { - if (freeCount == 0) - { - allocCount++; - if (allocCount > docFreeList.Length) - { - // Grow our free list up front to make sure we have - // enough space to recycle all outstanding PerDoc - // instances - System.Diagnostics.Debug.Assert(allocCount == 1 + docFreeList.Length); - docFreeList = new PerDoc[ArrayUtil.GetNextSize(allocCount)]; - } - return new PerDoc(this); - } - else - return docFreeList[--freeCount]; - } - } - - internal void FreePerDoc(PerDoc perDoc) - { - lock (this) - { - System.Diagnostics.Debug.Assert(freeCount < docFreeList.Length); - docFreeList[freeCount++] = perDoc; - } - } - - internal class PerDoc:DocumentsWriter.DocWriter - { - public PerDoc(DocFieldConsumers enclosingInstance) - { - InitBlock(enclosingInstance); - } - private void InitBlock(DocFieldConsumers enclosingInstance) - { - this.enclosingInstance = enclosingInstance; - } - private DocFieldConsumers enclosingInstance; - public DocFieldConsumers Enclosing_Instance - { - get - { - return enclosingInstance; - } - - } - - internal DocumentsWriter.DocWriter one; - internal DocumentsWriter.DocWriter two; - - public override long SizeInBytes() - { - return one.SizeInBytes() + two.SizeInBytes(); - } - - public override void Finish() - { - try - { - try - { - one.Finish(); - } - finally - { - two.Finish(); - } - } - finally - { - Enclosing_Instance.FreePerDoc(this); - } - } - - public override void Abort() - { - try - { - try - { - one.Abort(); - } - finally - { - two.Abort(); - } - } - finally - { - Enclosing_Instance.FreePerDoc(this); - } - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/DocFieldConsumersPerField.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/DocFieldConsumersPerField.cs b/src/Lucene.Net.Core/Index/DocFieldConsumersPerField.cs deleted file mode 100644 index e8ae3ea..0000000 --- a/src/Lucene.Net.Core/Index/DocFieldConsumersPerField.cs +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; -using Lucene.Net.Documents; - -namespace Lucene.Net.Index -{ - - sealed class DocFieldConsumersPerField:DocFieldConsumerPerField - { - - internal DocFieldConsumerPerField one; - internal DocFieldConsumerPerField two; - internal DocFieldConsumersPerThread perThread; - - public DocFieldConsumersPerField(DocFieldConsumersPerThread perThread, DocFieldConsumerPerField one, DocFieldConsumerPerField two) - { - this.perThread = perThread; - this.one = one; - this.two = two; - } - - public override void ProcessFields(IFieldable[] fields, int count) - { - one.ProcessFields(fields, count); - two.ProcessFields(fields, count); - } - - public override void Abort() - { - try - { - one.Abort(); - } - finally - { - two.Abort(); - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/228b970a/src/Lucene.Net.Core/Index/DocFieldConsumersPerThread.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/DocFieldConsumersPerThread.cs b/src/Lucene.Net.Core/Index/DocFieldConsumersPerThread.cs deleted file mode 100644 index eea1378..0000000 --- a/src/Lucene.Net.Core/Index/DocFieldConsumersPerThread.cs +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; - -namespace Lucene.Net.Index -{ - - sealed class DocFieldConsumersPerThread:DocFieldConsumerPerThread - { - - internal DocFieldConsumerPerThread one; - internal DocFieldConsumerPerThread two; - internal DocFieldConsumers parent; - internal DocumentsWriter.DocState docState; - - public DocFieldConsumersPerThread(DocFieldProcessorPerThread docFieldProcessorPerThread, DocFieldConsumers parent, DocFieldConsumerPerThread one, DocFieldConsumerPerThread two) - { - this.parent = parent; - this.one = one; - this.two = two; - docState = docFieldProcessorPerThread.docState; - } - - public override void StartDocument() - { - one.StartDocument(); - two.StartDocument(); - } - - public override void Abort() - { - try - { - one.Abort(); - } - finally - { - two.Abort(); - } - } - - public override DocumentsWriter.DocWriter FinishDocument() - { - DocumentsWriter.DocWriter oneDoc = one.FinishDocument(); - DocumentsWriter.DocWriter twoDoc = two.FinishDocument(); - if (oneDoc == null) - return twoDoc; - else if (twoDoc == null) - return oneDoc; - else - { - DocFieldConsumers.PerDoc both = parent.GetPerDoc(); - both.docID = docState.docID; - System.Diagnostics.Debug.Assert(oneDoc.docID == docState.docID); - System.Diagnostics.Debug.Assert(twoDoc.docID == docState.docID); - both.one = oneDoc; - both.two = twoDoc; - return both; - } - } - - public override DocFieldConsumerPerField AddField(FieldInfo fi) - { - return new DocFieldConsumersPerField(this, one.AddField(fi), two.AddField(fi)); - } - } -} \ No newline at end of file
