Lucene.Net.Codecs.Sep: Did a fresh port of SepSkipListReader and SepSkipListWriter
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/ffaa021f Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/ffaa021f Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/ffaa021f Branch: refs/heads/master Commit: ffaa021f062c848b3faee69810c58514ac6059e9 Parents: 7ae81ef Author: Shad Storhaug <[email protected]> Authored: Fri Sep 22 00:03:58 2017 +0700 Committer: Shad Storhaug <[email protected]> Committed: Fri Sep 22 12:45:44 2017 +0700 ---------------------------------------------------------------------- src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs | 203 +++++++++++--------- src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs | 173 +++++++++-------- 2 files changed, 196 insertions(+), 180 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucenenet/blob/ffaa021f/src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs b/src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs index 5cc9df4..9cb4574 100644 --- a/src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs +++ b/src/Lucene.Net.Codecs/Sep/SepSkipListReader.cs @@ -32,171 +32,184 @@ namespace Lucene.Net.Codecs.Sep // TODO: rewrite this as recursive classes? internal class SepSkipListReader : MultiLevelSkipListReader { - private bool _currentFieldStoresPayloads; - private readonly Int32IndexInput.Index[] _freqIndex; - private readonly Int32IndexInput.Index[] _docIndex; - private readonly Int32IndexInput.Index[] _posIndex; - private readonly long[] _payloadPointer; - private readonly int[] _payloadLength; - - private readonly Int32IndexInput.Index _lastFreqIndex; - private readonly Int32IndexInput.Index _lastDocIndex; - private readonly Int32IndexInput.Index _lastPosIndex; - - private long _lastPayloadPointer; - private int _lastPayloadLength; - - internal SepSkipListReader(IndexInput skipStream, Int32IndexInput freqIn, Int32IndexInput docIn, Int32IndexInput posIn, - int maxSkipLevels, int skipInterval) + private bool currentFieldStoresPayloads; + private Int32IndexInput.Index[] freqIndex; + private Int32IndexInput.Index[] docIndex; + private Int32IndexInput.Index[] posIndex; + private long[] payloadPointer; + private int[] payloadLength; + + private readonly Int32IndexInput.Index lastFreqIndex; + private readonly Int32IndexInput.Index lastDocIndex; + // TODO: -- make private again + internal readonly Int32IndexInput.Index lastPosIndex; + + private long lastPayloadPointer; + private int lastPayloadLength; + + /// <exception cref="System.IO.IOException"/> + internal SepSkipListReader(IndexInput skipStream, + Int32IndexInput freqIn, + Int32IndexInput docIn, + Int32IndexInput posIn, + int maxSkipLevels, + int skipInterval) : base(skipStream, maxSkipLevels, skipInterval) { if (freqIn != null) { - _freqIndex = new Int32IndexInput.Index[maxSkipLevels]; + freqIndex = new Int32IndexInput.Index[maxSkipLevels]; } - _docIndex = new Int32IndexInput.Index[maxSkipLevels]; + docIndex = new Int32IndexInput.Index[maxSkipLevels]; if (posIn != null) { - _posIndex = new Int32IndexInput.Index[m_maxNumberOfSkipLevels]; + posIndex = new Int32IndexInput.Index[m_maxNumberOfSkipLevels]; } - - for (var i = 0; i < maxSkipLevels; i++) + for (int i = 0; i < maxSkipLevels; i++) { if (freqIn != null) { - _freqIndex[i] = freqIn.GetIndex(); + freqIndex[i] = freqIn.GetIndex(); } - - _docIndex[i] = docIn.GetIndex(); - + docIndex[i] = docIn.GetIndex(); if (posIn != null) { - _posIndex[i] = posIn.GetIndex(); + posIndex[i] = posIn.GetIndex(); } } + payloadPointer = new long[maxSkipLevels]; + payloadLength = new int[maxSkipLevels]; - _payloadPointer = new long[maxSkipLevels]; - _payloadLength = new int[maxSkipLevels]; - - _lastFreqIndex = freqIn != null ? freqIn.GetIndex() : null; - _lastDocIndex = docIn.GetIndex(); - _lastPosIndex = posIn != null ? posIn.GetIndex() : null; + if (freqIn != null) + { + lastFreqIndex = freqIn.GetIndex(); + } + else + { + lastFreqIndex = null; + } + lastDocIndex = docIn.GetIndex(); + if (posIn != null) + { + lastPosIndex = posIn.GetIndex(); + } + else + { + lastPosIndex = null; + } } - private IndexOptions _indexOptions; + internal IndexOptions indexOptions; - internal virtual void SetIndexOptions(IndexOptions v) + internal void SetIndexOptions(IndexOptions v) { - _indexOptions = v; + indexOptions = v; } - internal virtual void Init(long skipPointer, - Int32IndexInput.Index docBaseIndex, - Int32IndexInput.Index freqBaseIndex, - Int32IndexInput.Index posBaseIndex, - long payloadBasePointer, - int df, - bool storesPayloads) + internal void Init(long skipPointer, + Int32IndexInput.Index docBaseIndex, + Int32IndexInput.Index freqBaseIndex, + Int32IndexInput.Index posBaseIndex, + long payloadBasePointer, + int df, + bool storesPayloads) { + base.Init(skipPointer, df); - this._currentFieldStoresPayloads = storesPayloads; + this.currentFieldStoresPayloads = storesPayloads; - _lastPayloadPointer = payloadBasePointer; + lastPayloadPointer = payloadBasePointer; - for (var i = 0; i < m_maxNumberOfSkipLevels; i++) + for (int i = 0; i < m_maxNumberOfSkipLevels; i++) { - _docIndex[i].CopyFrom(docBaseIndex); - if (_freqIndex != null) + docIndex[i].CopyFrom(docBaseIndex); + if (freqIndex != null) { - _freqIndex[i].CopyFrom(freqBaseIndex); + freqIndex[i].CopyFrom(freqBaseIndex); } if (posBaseIndex != null) { - _posIndex[i].CopyFrom(posBaseIndex); + posIndex[i].CopyFrom(posBaseIndex); } } - Arrays.Fill(_payloadPointer, payloadBasePointer); - Arrays.Fill(_payloadLength, 0); + Arrays.Fill(payloadPointer, payloadBasePointer); + Arrays.Fill(payloadLength, 0); } - internal virtual long PayloadPointer + internal long PayloadPointer { - get { return _lastPayloadPointer; } + get { return lastPayloadPointer; } } /// <summary> /// Returns the payload length of the payload stored just before /// the doc to which the last call of <see cref="MultiLevelSkipListReader.SkipTo(int)"/> - /// has skipped. + /// has skipped. /// </summary> - internal virtual int PayloadLength + internal int PayloadLength { - get { return _lastPayloadLength; } + get { return lastPayloadLength; } } + /// <exception cref="System.IO.IOException"/> protected override void SeekChild(int level) { base.SeekChild(level); - _payloadPointer[level] = _lastPayloadPointer; - _payloadLength[level] = _lastPayloadLength; + payloadPointer[level] = lastPayloadPointer; + payloadLength[level] = lastPayloadLength; } protected override void SetLastSkipData(int level) { base.SetLastSkipData(level); - _lastPayloadPointer = _payloadPointer[level]; - _lastPayloadLength = _payloadLength[level]; - - if (_freqIndex != null) + lastPayloadPointer = payloadPointer[level]; + lastPayloadLength = payloadLength[level]; + if (freqIndex != null) { - _lastFreqIndex.CopyFrom(_freqIndex[level]); + lastFreqIndex.CopyFrom(freqIndex[level]); } - - _lastDocIndex.CopyFrom(_docIndex[level]); - - if (_lastPosIndex != null) + lastDocIndex.CopyFrom(docIndex[level]); + if (lastPosIndex != null) { - _lastPosIndex.CopyFrom(_posIndex[level]); + lastPosIndex.CopyFrom(posIndex[level]); } if (level > 0) { - if (_freqIndex != null) + if (freqIndex != null) { - _freqIndex[level - 1].CopyFrom(_freqIndex[level]); + freqIndex[level - 1].CopyFrom(freqIndex[level]); } - - _docIndex[level - 1].CopyFrom(_docIndex[level]); - - if (_posIndex != null) + docIndex[level - 1].CopyFrom(docIndex[level]); + if (posIndex != null) { - _posIndex[level - 1].CopyFrom(_posIndex[level]); + posIndex[level - 1].CopyFrom(posIndex[level]); } } } - internal virtual Int32IndexInput.Index FreqIndex + internal Int32IndexInput.Index FreqIndex { - get { return _lastFreqIndex; } + get { return lastFreqIndex; } } - internal virtual Int32IndexInput.Index PosIndex + internal Int32IndexInput.Index PosIndex { - get { return _lastPosIndex; } + get { return lastPosIndex; } } - internal virtual Int32IndexInput.Index DocIndex + internal Int32IndexInput.Index DocIndex { - get { return _lastDocIndex; } + get { return lastDocIndex; } } + /// <exception cref="System.IO.IOException"/> protected override int ReadSkipData(int level, IndexInput skipStream) { int delta; - Debug.Assert(_indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || - !_currentFieldStoresPayloads); - if (_currentFieldStoresPayloads) + Debug.Assert(indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !currentFieldStoresPayloads); + if (currentFieldStoresPayloads) { // the current field stores payloads. // if the doc delta is odd then we have @@ -206,28 +219,26 @@ namespace Lucene.Net.Codecs.Sep delta = skipStream.ReadVInt32(); if ((delta & 1) != 0) { - _payloadLength[level] = skipStream.ReadVInt32(); + payloadLength[level] = skipStream.ReadVInt32(); } - delta = (int) ((uint) delta >> 1); + //delta >>>= 1; + delta = (int)((uint)delta >> 1); } else { delta = skipStream.ReadVInt32(); } - - if (_indexOptions != IndexOptions.DOCS_ONLY) + if (indexOptions != IndexOptions.DOCS_ONLY) { - _freqIndex[level].Read(skipStream, false); + freqIndex[level].Read(skipStream, false); } - - _docIndex[level].Read(skipStream, false); - if (_indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) + docIndex[level].Read(skipStream, false); + if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { - _posIndex[level].Read(skipStream, false); - - if (_currentFieldStoresPayloads) + posIndex[level].Read(skipStream, false); + if (currentFieldStoresPayloads) { - _payloadPointer[level] += skipStream.ReadVInt32(); + payloadPointer[level] += skipStream.ReadVInt32(); } } http://git-wip-us.apache.org/repos/asf/lucenenet/blob/ffaa021f/src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs b/src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs index 5d3acb5..703b98d 100644 --- a/src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs +++ b/src/Lucene.Net.Codecs/Sep/SepSkipListWriter.cs @@ -33,121 +33,126 @@ namespace Lucene.Net.Codecs.Sep /// </summary> internal class SepSkipListWriter : MultiLevelSkipListWriter { - private readonly int[] _lastSkipDoc; - private readonly int[] _lastSkipPayloadLength; - private readonly long[] _lastSkipPayloadPointer; - - private readonly Int32IndexOutput.Index[] _docIndex; - private readonly Int32IndexOutput.Index[] _freqIndex; - private readonly Int32IndexOutput.Index[] _posIndex; - - private readonly Int32IndexOutput _freqOutput; - private Int32IndexOutput _posOutput; - private IndexOutput _payloadOutput; - - private int _curDoc; - private bool _curStorePayloads; - private int _curPayloadLength; - private long _curPayloadPointer; - - internal SepSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, - Int32IndexOutput freqOutput, - Int32IndexOutput docOutput, - Int32IndexOutput posOutput, - IndexOutput payloadOutput) + private int[] lastSkipDoc; + private int[] lastSkipPayloadLength; + private long[] lastSkipPayloadPointer; + + private Int32IndexOutput.Index[] docIndex; + private Int32IndexOutput.Index[] freqIndex; + private Int32IndexOutput.Index[] posIndex; + + private Int32IndexOutput freqOutput; + // TODO: -- private again + internal Int32IndexOutput posOutput; + // TODO: -- private again + internal IndexOutput payloadOutput; + + private int curDoc; + private bool curStorePayloads; + private int curPayloadLength; + private long curPayloadPointer; + + /// <exception cref="System.IO.IOException"/> + internal SepSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, + Int32IndexOutput freqOutput, + Int32IndexOutput docOutput, + Int32IndexOutput posOutput, + IndexOutput payloadOutput) : base(skipInterval, numberOfSkipLevels, docCount) { - this._freqOutput = freqOutput; - this._posOutput = posOutput; - this._payloadOutput = payloadOutput; - _lastSkipDoc = new int[numberOfSkipLevels]; - _lastSkipPayloadLength = new int[numberOfSkipLevels]; + this.freqOutput = freqOutput; + this.posOutput = posOutput; + this.payloadOutput = payloadOutput; + + lastSkipDoc = new int[numberOfSkipLevels]; + lastSkipPayloadLength = new int[numberOfSkipLevels]; // TODO: -- also cutover normal IndexOutput to use getIndex()? - _lastSkipPayloadPointer = new long[numberOfSkipLevels]; + lastSkipPayloadPointer = new long[numberOfSkipLevels]; - _freqIndex = new Int32IndexOutput.Index[numberOfSkipLevels]; - _docIndex = new Int32IndexOutput.Index[numberOfSkipLevels]; - _posIndex = new Int32IndexOutput.Index[numberOfSkipLevels]; + freqIndex = new Int32IndexOutput.Index[numberOfSkipLevels]; + docIndex = new Int32IndexOutput.Index[numberOfSkipLevels]; + posIndex = new Int32IndexOutput.Index[numberOfSkipLevels]; - for (var i = 0; i < numberOfSkipLevels; i++) + for (int i = 0; i < numberOfSkipLevels; i++) { if (freqOutput != null) { - _freqIndex[i] = freqOutput.GetIndex(); + freqIndex[i] = freqOutput.GetIndex(); } - _docIndex[i] = docOutput.GetIndex(); + docIndex[i] = docOutput.GetIndex(); if (posOutput != null) { - _posIndex[i] = posOutput.GetIndex(); + posIndex[i] = posOutput.GetIndex(); } } } - private IndexOptions _indexOptions; + IndexOptions indexOptions; - internal virtual void SetIndexOptions(IndexOptions v) + internal void SetIndexOptions(IndexOptions v) { - _indexOptions = v; + indexOptions = v; } - internal virtual void SetPosOutput(Int32IndexOutput posOutput) + /// <exception cref="System.IO.IOException"/> + internal void SetPosOutput(Int32IndexOutput posOutput) { - _posOutput = posOutput; - for (var i = 0; i < m_numberOfSkipLevels; i++) + this.posOutput = posOutput; + for (int i = 0; i < m_numberOfSkipLevels; i++) { - _posIndex[i] = posOutput.GetIndex(); + posIndex[i] = posOutput.GetIndex(); } } - internal virtual void SetPayloadOutput(IndexOutput payloadOutput) + internal void SetPayloadOutput(IndexOutput payloadOutput) { - _payloadOutput = payloadOutput; + this.payloadOutput = payloadOutput; } /// <summary> /// Sets the values for the current skip data. - /// Called @ every index interval (every 128th (by default) doc). /// </summary> - internal virtual void SetSkipData(int doc, bool storePayloads, int payloadLength) + // Called @ every index interval (every 128th (by default) + // doc) + internal void SetSkipData(int doc, bool storePayloads, int payloadLength) { - this._curDoc = doc; - this._curStorePayloads = storePayloads; - this._curPayloadLength = payloadLength; - if (_payloadOutput != null) + this.curDoc = doc; + this.curStorePayloads = storePayloads; + this.curPayloadLength = payloadLength; + if (payloadOutput != null) { - _curPayloadPointer = _payloadOutput.GetFilePointer(); + this.curPayloadPointer = payloadOutput.GetFilePointer(); } } - /// <summary> - /// Called @ start of new term. - /// </summary> - protected internal virtual void ResetSkip(Int32IndexOutput.Index topDocIndex, Int32IndexOutput.Index topFreqIndex, - Int32IndexOutput.Index topPosIndex) + // Called @ start of new term + /// <exception cref="System.IO.IOException"/> + protected internal void ResetSkip(Int32IndexOutput.Index topDocIndex, Int32IndexOutput.Index topFreqIndex, Int32IndexOutput.Index topPosIndex) { base.ResetSkip(); - Arrays.Fill(_lastSkipDoc, 0); - Arrays.Fill(_lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list + Arrays.Fill(lastSkipDoc, 0); + Arrays.Fill(lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list for (int i = 0; i < m_numberOfSkipLevels; i++) { - _docIndex[i].CopyFrom(topDocIndex, true); - if (_freqOutput != null) + docIndex[i].CopyFrom(topDocIndex, true); + if (freqOutput != null) { - _freqIndex[i].CopyFrom(topFreqIndex, true); + freqIndex[i].CopyFrom(topFreqIndex, true); } - if (_posOutput != null) + if (posOutput != null) { - _posIndex[i].CopyFrom(topPosIndex, true); + posIndex[i].CopyFrom(topPosIndex, true); } } - if (_payloadOutput != null) + if (payloadOutput != null) { - Arrays.Fill(_lastSkipPayloadPointer, _payloadOutput.GetFilePointer()); + Arrays.Fill(lastSkipPayloadPointer, payloadOutput.GetFilePointer()); } } + /// <exception cref="System.IO.IOException"/> protected override void WriteSkipData(int level, IndexOutput skipBuffer) { // To efficiently store payloads in the posting lists we do not store the length of @@ -171,12 +176,12 @@ namespace Lucene.Net.Codecs.Sep // current payload length equals the length at the previous // skip point - Debug.Assert(_indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !_curStorePayloads); + Debug.Assert(indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !curStorePayloads); - if (_curStorePayloads) + if (curStorePayloads) { - int delta = _curDoc - _lastSkipDoc[level]; - if (_curPayloadLength == _lastSkipPayloadLength[level]) + int delta = curDoc - lastSkipDoc[level]; + if (curPayloadLength == lastSkipPayloadLength[level]) { // the current payload length equals the length at the previous skip point, // so we don't store the length again @@ -187,35 +192,35 @@ namespace Lucene.Net.Codecs.Sep // the payload length is different from the previous one. We shift the DocSkip, // set the lowest bit and store the current payload length as VInt. skipBuffer.WriteVInt32(delta << 1 | 1); - skipBuffer.WriteVInt32(_curPayloadLength); - _lastSkipPayloadLength[level] = _curPayloadLength; + skipBuffer.WriteVInt32(curPayloadLength); + lastSkipPayloadLength[level] = curPayloadLength; } } else { // current field does not store payloads - skipBuffer.WriteVInt32(_curDoc - _lastSkipDoc[level]); + skipBuffer.WriteVInt32(curDoc - lastSkipDoc[level]); } - if (_indexOptions != IndexOptions.DOCS_ONLY) + if (indexOptions != IndexOptions.DOCS_ONLY) { - _freqIndex[level].Mark(); - _freqIndex[level].Write(skipBuffer, false); + freqIndex[level].Mark(); + freqIndex[level].Write(skipBuffer, false); } - _docIndex[level].Mark(); - _docIndex[level].Write(skipBuffer, false); - if (_indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) + docIndex[level].Mark(); + docIndex[level].Write(skipBuffer, false); + if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { - _posIndex[level].Mark(); - _posIndex[level].Write(skipBuffer, false); - if (_curStorePayloads) + posIndex[level].Mark(); + posIndex[level].Write(skipBuffer, false); + if (curStorePayloads) { - skipBuffer.WriteVInt32((int)(_curPayloadPointer - _lastSkipPayloadPointer[level])); + skipBuffer.WriteVInt32((int)(curPayloadPointer - lastSkipPayloadPointer[level])); } } - _lastSkipDoc[level] = _curDoc; - _lastSkipPayloadPointer[level] = _curPayloadPointer; + lastSkipDoc[level] = curDoc; + lastSkipPayloadPointer[level] = curPayloadPointer; } } } \ No newline at end of file
