Lucene.Net.Codecs: Renamed Intblock folder to IntBlock
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/0159ad2f Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/0159ad2f Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/0159ad2f Branch: refs/heads/api-work Commit: 0159ad2fcfc39dc4e662326ca01c414f0350fdae Parents: 5a5c215 Author: Shad Storhaug <[email protected]> Authored: Sun Jan 29 18:00:51 2017 +0700 Committer: Shad Storhaug <[email protected]> Committed: Sun Jan 29 18:00:51 2017 +0700 ---------------------------------------------------------------------- .../IntBlock/FixedIntBlockIndexInput.cs | 196 ++++++++++++++++ .../IntBlock/FixedIntBlockIndexOutput.cs | 151 ++++++++++++ .../IntBlock/VariableIntBlockIndexInput.cs | 229 +++++++++++++++++++ .../IntBlock/VariableIntBlockIndexOutput.cs | 162 +++++++++++++ .../Intblock/FixedIntBlockIndexInput.cs | 196 ---------------- .../Intblock/FixedIntBlockIndexOutput.cs | 151 ------------ .../Intblock/VariableIntBlockIndexInput.cs | 229 ------------------- .../Intblock/VariableIntBlockIndexOutput.cs | 162 ------------- 8 files changed, 738 insertions(+), 738 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexInput.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexInput.cs b/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexInput.cs new file mode 100644 index 0000000..991b5f0 --- /dev/null +++ b/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexInput.cs @@ -0,0 +1,196 @@ +using Lucene.Net.Codecs.Sep; +using Lucene.Net.Store; +using System.Diagnostics; + +namespace Lucene.Net.Codecs.IntBlock +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + // Naive int block API that writes vInts. This is + // expected to give poor performance; it's really only for + // testing the pluggability. One should typically use pfor instead. + + /// <summary> + /// Abstract base class that reads fixed-size blocks of ints + /// from an IndexInput. While this is a simple approach, a + /// more performant approach would directly create an impl + /// of IntIndexInput inside Directory. Wrapping a generic + /// IndexInput will likely cost performance. + /// + /// @lucene.experimental + /// </summary> + public abstract class FixedIntBlockIndexInput : IntIndexInput + { + private readonly IndexInput input; + protected readonly int m_blockSize; + + public FixedIntBlockIndexInput(IndexInput @in) + { + input = @in; + m_blockSize = @in.ReadVInt(); + } + + public override AbstractReader GetReader() + { + var buffer = new int[m_blockSize]; + var clone = (IndexInput)input.Clone(); + // TODO: can this be simplified? + return new Reader(clone, buffer, GetBlockReader(clone, buffer)); + } + + public override void Dispose() + { + input.Dispose(); + } + + public override AbstractIndex GetIndex() + { + return new Index(this); + } + + protected abstract IBlockReader GetBlockReader(IndexInput @in, int[] buffer); + + + /// <summary> + /// Interface for fixed-size block decoders. + /// <para> + /// Implementations should decode into the buffer in <see cref="ReadBlock()"/>. + /// </para> + /// </summary> + public interface IBlockReader + { + void ReadBlock(); + } + + private class Reader : AbstractReader + { + private readonly IndexInput input; + private readonly IBlockReader blockReader; + private readonly int blockSize; + private readonly int[] pending; + + private int upto; + private bool seekPending; + private long pendingFP; + private long lastBlockFP = -1; + + public Reader(IndexInput input, int[] pending, IBlockReader blockReader) + { + this.input = input; + this.pending = pending; + this.blockSize = pending.Length; + this.blockReader = blockReader; + upto = blockSize; + } + + internal virtual void Seek(long fp, int upto) + { + Debug.Assert(upto < blockSize); + if (seekPending || fp != lastBlockFP) + { + pendingFP = fp; + seekPending = true; + } + this.upto = upto; + } + + public override int Next() + { + if (seekPending) + { + // Seek & load new block + input.Seek(pendingFP); + lastBlockFP = pendingFP; + blockReader.ReadBlock(); + seekPending = false; + } + else if (upto == blockSize) + { + // Load new block + lastBlockFP = input.FilePointer; + blockReader.ReadBlock(); + upto = 0; + } + return pending[upto++]; + } + } + + private class Index : AbstractIndex + { + private readonly FixedIntBlockIndexInput outerInstance; + + public Index(FixedIntBlockIndexInput outerInstance) + { + this.outerInstance = outerInstance; + } + + private long fp; + private int upto; + + public override void Read(DataInput indexIn, bool absolute) + { + if (absolute) + { + upto = indexIn.ReadVInt(); + fp = indexIn.ReadVLong(); + } + else + { + int uptoDelta = indexIn.ReadVInt(); + if ((uptoDelta & 1) == 1) + { + // same block + upto += (int)((uint)uptoDelta >> 1); + } + else + { + // new block + upto = (int)((uint)uptoDelta >> 1); + fp += indexIn.ReadVLong(); + } + } + Debug.Assert(upto < outerInstance.m_blockSize); + } + + public override void Seek(AbstractReader other) + { + ((Reader)other).Seek(fp, upto); + } + + public override void CopyFrom(AbstractIndex other) + { + Index idx = (Index)other; + fp = idx.fp; + upto = idx.upto; + } + + public override AbstractIndex Clone() + { + Index other = new Index(outerInstance); + other.fp = fp; + other.upto = upto; + return other; + } + + public override string ToString() + { + return "fp=" + fp + " upto=" + upto; + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexOutput.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexOutput.cs b/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexOutput.cs new file mode 100644 index 0000000..544b644 --- /dev/null +++ b/src/Lucene.Net.Codecs/IntBlock/FixedIntBlockIndexOutput.cs @@ -0,0 +1,151 @@ +using Lucene.Net.Codecs.Sep; +using Lucene.Net.Store; +using System.Diagnostics; + +namespace Lucene.Net.Codecs.IntBlock +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Naive int block API that writes vInts. This is + /// expected to give poor performance; it's really only for + /// testing the pluggability. One should typically use pfor instead. + /// </summary> + + + /// <summary> + /// Abstract base class that writes fixed-size blocks of ints + /// to an IndexOutput. While this is a simple approach, a + /// more performant approach would directly create an impl + /// of IntIndexOutput inside Directory. Wrapping a generic + /// IndexInput will likely cost performance. + /// + /// @lucene.experimental + /// </summary> + public abstract class FixedIntBlockIndexOutput : IntIndexOutput + { + protected readonly IndexOutput m_output; + private readonly int blockSize; + protected readonly int[] m_buffer; + private int upto; + + protected FixedIntBlockIndexOutput(IndexOutput output, int fixedBlockSize) + { + blockSize = fixedBlockSize; + this.m_output = output; + output.WriteVInt(blockSize); + m_buffer = new int[blockSize]; + } + + protected abstract void FlushBlock(); + + public override AbstractIndex GetIndex() + { + return new Index(this); + } + + private class Index : AbstractIndex + { + private readonly FixedIntBlockIndexOutput outerInstance; + + public Index(FixedIntBlockIndexOutput outerInstance) + { + this.outerInstance = outerInstance; + } + + internal long fp; + internal int upto; + internal long lastFP; + internal int lastUpto; + + public override void Mark() + { + fp = outerInstance.m_output.FilePointer; + upto = outerInstance.upto; + } + + public override void CopyFrom(AbstractIndex other, bool copyLast) + { + Index idx = (Index)other; + fp = idx.fp; + upto = idx.upto; + if (copyLast) + { + lastFP = fp; + lastUpto = upto; + } + } + + public override void Write(DataOutput indexOut, bool absolute) + { + if (absolute) + { + indexOut.WriteVInt(upto); + indexOut.WriteVLong(fp); + } + else if (fp == lastFP) + { + // same block + Debug.Assert(upto >= lastUpto); + int uptoDelta = upto - lastUpto; + indexOut.WriteVInt(uptoDelta << 1 | 1); + } + else + { + // new block + indexOut.WriteVInt(upto << 1); + indexOut.WriteVLong(fp - lastFP); + } + lastUpto = upto; + lastFP = fp; + } + + public override string ToString() + { + return "fp=" + fp + " upto=" + upto; + } + } + + public override void Write(int v) + { + m_buffer[upto++] = v; + if (upto == blockSize) + { + FlushBlock(); + upto = 0; + } + } + + public override void Dispose() + { + try + { + if (upto > 0) + { + // NOTE: entries in the block after current upto are + // invalid + FlushBlock(); + } + } + finally + { + m_output.Dispose(); + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexInput.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexInput.cs b/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexInput.cs new file mode 100644 index 0000000..6607801 --- /dev/null +++ b/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexInput.cs @@ -0,0 +1,229 @@ +using Lucene.Net.Codecs.Sep; +using Lucene.Net.Store; +using Lucene.Net.Support; +using System.Diagnostics; + +namespace Lucene.Net.Codecs.IntBlock +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + // Naive int block API that writes vInts. This is + // expected to give poor performance; it's really only for + // testing the pluggability. One should typically use pfor instead. + + // TODO: much of this can be shared code w/ the fixed case + + /// <summary> + /// Abstract base class that reads variable-size blocks of ints + /// from an IndexInput. While this is a simple approach, a + /// more performant approach would directly create an impl + /// of IntIndexInput inside Directory. Wrapping a generic + /// IndexInput will likely cost performance. + /// + /// @lucene.experimental + /// </summary> + public abstract class VariableIntBlockIndexInput : IntIndexInput + { + private readonly IndexInput input; + protected readonly int m_maxBlockSize; + + protected internal VariableIntBlockIndexInput(IndexInput input) + { + this.input = input; + m_maxBlockSize = input.ReadInt(); + } + + public override AbstractReader GetReader() + { + var buffer = new int[m_maxBlockSize]; + var clone = (IndexInput)input.Clone(); + // TODO: can this be simplified? + return new Reader(clone, buffer, GetBlockReader(clone, buffer)); + } + + public override void Dispose() + { + input.Dispose(); + } + + public override AbstractIndex GetIndex() + { + return new Index(this); + } + + protected abstract IBlockReader GetBlockReader(IndexInput @in, int[] buffer); + + /// <summary> + /// Interface for variable-size block decoders. + /// <para> + /// Implementations should decode into the buffer in <see cref="ReadBlock()"/>. + /// </para> + /// </summary> + public interface IBlockReader + { + int ReadBlock(); + void Seek(long pos); + } + + private class Reader : AbstractReader + { + private readonly IndexInput input; + + [WritableArray] + public int[] Pending + { + get { return pending; } + } + private readonly int[] pending; + private int upto; + + private bool seekPending; + private long pendingFP; + private int pendingUpto; + private long lastBlockFP; + private int blockSize; + private readonly IBlockReader blockReader; + + public Reader(IndexInput input, int[] pending, IBlockReader blockReader) + { + this.input = input; + this.pending = pending; + this.blockReader = blockReader; + } + + internal virtual void Seek(long fp, int upto) + { + // TODO: should we do this in real-time, not lazy? + pendingFP = fp; + pendingUpto = upto; + Debug.Assert(pendingUpto >= 0, "pendingUpto=" + pendingUpto); + seekPending = true; + } + + private void MaybeSeek() + { + if (seekPending) + { + if (pendingFP != lastBlockFP) + { + // need new block + input.Seek(pendingFP); + blockReader.Seek(pendingFP); + lastBlockFP = pendingFP; + blockSize = blockReader.ReadBlock(); + } + upto = pendingUpto; + + // TODO: if we were more clever when writing the + // index, such that a seek point wouldn't be written + // until the int encoder "committed", we could avoid + // this (likely minor) inefficiency: + + // This is necessary for int encoders that are + // non-causal, ie must see future int values to + // encode the current ones. + while (upto >= blockSize) + { + upto -= blockSize; + lastBlockFP = input.FilePointer; + blockSize = blockReader.ReadBlock(); + } + seekPending = false; + } + } + + public override int Next() + { + this.MaybeSeek(); + if (upto == blockSize) + { + lastBlockFP = input.FilePointer; + blockSize = blockReader.ReadBlock(); + upto = 0; + } + + return pending[upto++]; + } + } + + private class Index : AbstractIndex + { + private readonly VariableIntBlockIndexInput outerInstance; + + public Index(VariableIntBlockIndexInput outerInstance) + { + this.outerInstance = outerInstance; + } + + private long fp; + private int upto; + + public override void Read(DataInput indexIn, bool absolute) + { + if (absolute) + { + upto = indexIn.ReadVInt(); + fp = indexIn.ReadVLong(); + } + else + { + int uptoDelta = indexIn.ReadVInt(); + if ((uptoDelta & 1) == 1) + { + // same block + upto += (int)((uint)uptoDelta >> 1); + } + else + { + // new block + upto = (int)((uint)uptoDelta >> 1); + fp += indexIn.ReadVLong(); + } + } + // TODO: we can't do this assert because non-causal + // int encoders can have upto over the buffer size + //assert upto < maxBlockSize: "upto=" + upto + " max=" + maxBlockSize; + } + + public override string ToString() + { + return "VarIntBlock.Index fp=" + fp + " upto=" + upto + " maxBlock=" + outerInstance.m_maxBlockSize; + } + + public override void Seek(AbstractReader other) + { + ((Reader)other).Seek(fp, upto); + } + + public override void CopyFrom(AbstractIndex other) + { + Index idx = (Index)other; + fp = idx.fp; + upto = idx.upto; + } + + public override AbstractIndex Clone() + { + Index other = new Index(outerInstance); + other.fp = fp; + other.upto = upto; + return other; + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexOutput.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexOutput.cs b/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexOutput.cs new file mode 100644 index 0000000..f756777 --- /dev/null +++ b/src/Lucene.Net.Codecs/IntBlock/VariableIntBlockIndexOutput.cs @@ -0,0 +1,162 @@ +using Lucene.Net.Codecs.Sep; +using Lucene.Net.Store; +using System.Diagnostics; + +namespace Lucene.Net.Codecs.IntBlock +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Naive int block API that writes vInts. This is + /// expected to give poor performance; it's really only for + /// testing the pluggability. One should typically use pfor instead. + /// </summary> + + + // TODO: much of this can be shared code w/ the fixed case + + /// <summary> + /// Abstract base class that writes variable-size blocks of ints + /// to an IndexOutput. While this is a simple approach, a + /// more performant approach would directly create an impl + /// of IntIndexOutput inside Directory. Wrapping a generic + /// IndexInput will likely cost performance. + /// + /// @lucene.experimental + /// </summary> + public abstract class VariableIntBlockIndexOutput : IntIndexOutput + { + protected readonly IndexOutput m_output; + + private int upto; + private bool hitExcDuringWrite; + + // TODO what Var-Var codecs exist in practice... and what are there blocksizes like? + // if its less than 128 we should set that as max and use byte? + + /// <summary> + /// NOTE: maxBlockSize must be the maximum block size + /// plus the max non-causal lookahead of your codec. EG Simple9 + /// requires lookahead=1 because on seeing the Nth value + /// it knows it must now encode the N-1 values before it. + /// </summary> + protected VariableIntBlockIndexOutput(IndexOutput output, int maxBlockSize) + { + this.m_output = output; + this.m_output.WriteInt(maxBlockSize); + } + + /// <summary> + /// Called one value at a time. Return the number of + /// buffered input values that have been written to out. + /// </summary> + protected abstract int Add(int value); + + public override AbstractIndex GetIndex() + { + return new Index(this); + } + + private class Index : AbstractIndex + { + private readonly VariableIntBlockIndexOutput outerInstance; + + public Index(VariableIntBlockIndexOutput outerInstance) + { + this.outerInstance = outerInstance; + } + + private long fp; + private int upto; + private long lastFP; + private int lastUpto; + + public override void Mark() + { + fp = outerInstance.m_output.FilePointer; + upto = outerInstance.upto; + } + + public override void CopyFrom(AbstractIndex other, bool copyLast) + { + Index idx = (Index)other; + fp = idx.fp; + upto = idx.upto; + if (copyLast) + { + lastFP = fp; + lastUpto = upto; + } + } + + public override void Write(DataOutput indexOut, bool absolute) + { + Debug.Assert(upto >= 0); + if (absolute) + { + indexOut.WriteVInt(upto); + indexOut.WriteVLong(fp); + } + else if (fp == lastFP) + { + // same block + Debug.Assert(upto >= lastUpto); + int uptoDelta = upto - lastUpto; + indexOut.WriteVInt(uptoDelta << 1 | 1); + } + else + { + // new block + indexOut.WriteVInt(upto << 1); + indexOut.WriteVLong(fp - lastFP); + } + lastUpto = upto; + lastFP = fp; + } + } + + public override void Write(int v) + { + hitExcDuringWrite = true; + upto -= Add(v) - 1; + hitExcDuringWrite = false; + Debug.Assert(upto >= 0); + } + + public override void Dispose() + { + try + { + if (hitExcDuringWrite) return; + + // stuff 0s in until the "real" data is flushed: + var stuffed = 0; + while (upto > stuffed) + { + upto -= Add(0) - 1; + Debug.Assert(upto >= 0); + stuffed += 1; + } + } + finally + { + m_output.Dispose(); + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexInput.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexInput.cs b/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexInput.cs deleted file mode 100644 index 991b5f0..0000000 --- a/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexInput.cs +++ /dev/null @@ -1,196 +0,0 @@ -using Lucene.Net.Codecs.Sep; -using Lucene.Net.Store; -using System.Diagnostics; - -namespace Lucene.Net.Codecs.IntBlock -{ - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - // Naive int block API that writes vInts. This is - // expected to give poor performance; it's really only for - // testing the pluggability. One should typically use pfor instead. - - /// <summary> - /// Abstract base class that reads fixed-size blocks of ints - /// from an IndexInput. While this is a simple approach, a - /// more performant approach would directly create an impl - /// of IntIndexInput inside Directory. Wrapping a generic - /// IndexInput will likely cost performance. - /// - /// @lucene.experimental - /// </summary> - public abstract class FixedIntBlockIndexInput : IntIndexInput - { - private readonly IndexInput input; - protected readonly int m_blockSize; - - public FixedIntBlockIndexInput(IndexInput @in) - { - input = @in; - m_blockSize = @in.ReadVInt(); - } - - public override AbstractReader GetReader() - { - var buffer = new int[m_blockSize]; - var clone = (IndexInput)input.Clone(); - // TODO: can this be simplified? - return new Reader(clone, buffer, GetBlockReader(clone, buffer)); - } - - public override void Dispose() - { - input.Dispose(); - } - - public override AbstractIndex GetIndex() - { - return new Index(this); - } - - protected abstract IBlockReader GetBlockReader(IndexInput @in, int[] buffer); - - - /// <summary> - /// Interface for fixed-size block decoders. - /// <para> - /// Implementations should decode into the buffer in <see cref="ReadBlock()"/>. - /// </para> - /// </summary> - public interface IBlockReader - { - void ReadBlock(); - } - - private class Reader : AbstractReader - { - private readonly IndexInput input; - private readonly IBlockReader blockReader; - private readonly int blockSize; - private readonly int[] pending; - - private int upto; - private bool seekPending; - private long pendingFP; - private long lastBlockFP = -1; - - public Reader(IndexInput input, int[] pending, IBlockReader blockReader) - { - this.input = input; - this.pending = pending; - this.blockSize = pending.Length; - this.blockReader = blockReader; - upto = blockSize; - } - - internal virtual void Seek(long fp, int upto) - { - Debug.Assert(upto < blockSize); - if (seekPending || fp != lastBlockFP) - { - pendingFP = fp; - seekPending = true; - } - this.upto = upto; - } - - public override int Next() - { - if (seekPending) - { - // Seek & load new block - input.Seek(pendingFP); - lastBlockFP = pendingFP; - blockReader.ReadBlock(); - seekPending = false; - } - else if (upto == blockSize) - { - // Load new block - lastBlockFP = input.FilePointer; - blockReader.ReadBlock(); - upto = 0; - } - return pending[upto++]; - } - } - - private class Index : AbstractIndex - { - private readonly FixedIntBlockIndexInput outerInstance; - - public Index(FixedIntBlockIndexInput outerInstance) - { - this.outerInstance = outerInstance; - } - - private long fp; - private int upto; - - public override void Read(DataInput indexIn, bool absolute) - { - if (absolute) - { - upto = indexIn.ReadVInt(); - fp = indexIn.ReadVLong(); - } - else - { - int uptoDelta = indexIn.ReadVInt(); - if ((uptoDelta & 1) == 1) - { - // same block - upto += (int)((uint)uptoDelta >> 1); - } - else - { - // new block - upto = (int)((uint)uptoDelta >> 1); - fp += indexIn.ReadVLong(); - } - } - Debug.Assert(upto < outerInstance.m_blockSize); - } - - public override void Seek(AbstractReader other) - { - ((Reader)other).Seek(fp, upto); - } - - public override void CopyFrom(AbstractIndex other) - { - Index idx = (Index)other; - fp = idx.fp; - upto = idx.upto; - } - - public override AbstractIndex Clone() - { - Index other = new Index(outerInstance); - other.fp = fp; - other.upto = upto; - return other; - } - - public override string ToString() - { - return "fp=" + fp + " upto=" + upto; - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexOutput.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexOutput.cs b/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexOutput.cs deleted file mode 100644 index 544b644..0000000 --- a/src/Lucene.Net.Codecs/Intblock/FixedIntBlockIndexOutput.cs +++ /dev/null @@ -1,151 +0,0 @@ -using Lucene.Net.Codecs.Sep; -using Lucene.Net.Store; -using System.Diagnostics; - -namespace Lucene.Net.Codecs.IntBlock -{ - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - /// <summary> - /// Naive int block API that writes vInts. This is - /// expected to give poor performance; it's really only for - /// testing the pluggability. One should typically use pfor instead. - /// </summary> - - - /// <summary> - /// Abstract base class that writes fixed-size blocks of ints - /// to an IndexOutput. While this is a simple approach, a - /// more performant approach would directly create an impl - /// of IntIndexOutput inside Directory. Wrapping a generic - /// IndexInput will likely cost performance. - /// - /// @lucene.experimental - /// </summary> - public abstract class FixedIntBlockIndexOutput : IntIndexOutput - { - protected readonly IndexOutput m_output; - private readonly int blockSize; - protected readonly int[] m_buffer; - private int upto; - - protected FixedIntBlockIndexOutput(IndexOutput output, int fixedBlockSize) - { - blockSize = fixedBlockSize; - this.m_output = output; - output.WriteVInt(blockSize); - m_buffer = new int[blockSize]; - } - - protected abstract void FlushBlock(); - - public override AbstractIndex GetIndex() - { - return new Index(this); - } - - private class Index : AbstractIndex - { - private readonly FixedIntBlockIndexOutput outerInstance; - - public Index(FixedIntBlockIndexOutput outerInstance) - { - this.outerInstance = outerInstance; - } - - internal long fp; - internal int upto; - internal long lastFP; - internal int lastUpto; - - public override void Mark() - { - fp = outerInstance.m_output.FilePointer; - upto = outerInstance.upto; - } - - public override void CopyFrom(AbstractIndex other, bool copyLast) - { - Index idx = (Index)other; - fp = idx.fp; - upto = idx.upto; - if (copyLast) - { - lastFP = fp; - lastUpto = upto; - } - } - - public override void Write(DataOutput indexOut, bool absolute) - { - if (absolute) - { - indexOut.WriteVInt(upto); - indexOut.WriteVLong(fp); - } - else if (fp == lastFP) - { - // same block - Debug.Assert(upto >= lastUpto); - int uptoDelta = upto - lastUpto; - indexOut.WriteVInt(uptoDelta << 1 | 1); - } - else - { - // new block - indexOut.WriteVInt(upto << 1); - indexOut.WriteVLong(fp - lastFP); - } - lastUpto = upto; - lastFP = fp; - } - - public override string ToString() - { - return "fp=" + fp + " upto=" + upto; - } - } - - public override void Write(int v) - { - m_buffer[upto++] = v; - if (upto == blockSize) - { - FlushBlock(); - upto = 0; - } - } - - public override void Dispose() - { - try - { - if (upto > 0) - { - // NOTE: entries in the block after current upto are - // invalid - FlushBlock(); - } - } - finally - { - m_output.Dispose(); - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexInput.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexInput.cs b/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexInput.cs deleted file mode 100644 index 6607801..0000000 --- a/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexInput.cs +++ /dev/null @@ -1,229 +0,0 @@ -using Lucene.Net.Codecs.Sep; -using Lucene.Net.Store; -using Lucene.Net.Support; -using System.Diagnostics; - -namespace Lucene.Net.Codecs.IntBlock -{ - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - // Naive int block API that writes vInts. This is - // expected to give poor performance; it's really only for - // testing the pluggability. One should typically use pfor instead. - - // TODO: much of this can be shared code w/ the fixed case - - /// <summary> - /// Abstract base class that reads variable-size blocks of ints - /// from an IndexInput. While this is a simple approach, a - /// more performant approach would directly create an impl - /// of IntIndexInput inside Directory. Wrapping a generic - /// IndexInput will likely cost performance. - /// - /// @lucene.experimental - /// </summary> - public abstract class VariableIntBlockIndexInput : IntIndexInput - { - private readonly IndexInput input; - protected readonly int m_maxBlockSize; - - protected internal VariableIntBlockIndexInput(IndexInput input) - { - this.input = input; - m_maxBlockSize = input.ReadInt(); - } - - public override AbstractReader GetReader() - { - var buffer = new int[m_maxBlockSize]; - var clone = (IndexInput)input.Clone(); - // TODO: can this be simplified? - return new Reader(clone, buffer, GetBlockReader(clone, buffer)); - } - - public override void Dispose() - { - input.Dispose(); - } - - public override AbstractIndex GetIndex() - { - return new Index(this); - } - - protected abstract IBlockReader GetBlockReader(IndexInput @in, int[] buffer); - - /// <summary> - /// Interface for variable-size block decoders. - /// <para> - /// Implementations should decode into the buffer in <see cref="ReadBlock()"/>. - /// </para> - /// </summary> - public interface IBlockReader - { - int ReadBlock(); - void Seek(long pos); - } - - private class Reader : AbstractReader - { - private readonly IndexInput input; - - [WritableArray] - public int[] Pending - { - get { return pending; } - } - private readonly int[] pending; - private int upto; - - private bool seekPending; - private long pendingFP; - private int pendingUpto; - private long lastBlockFP; - private int blockSize; - private readonly IBlockReader blockReader; - - public Reader(IndexInput input, int[] pending, IBlockReader blockReader) - { - this.input = input; - this.pending = pending; - this.blockReader = blockReader; - } - - internal virtual void Seek(long fp, int upto) - { - // TODO: should we do this in real-time, not lazy? - pendingFP = fp; - pendingUpto = upto; - Debug.Assert(pendingUpto >= 0, "pendingUpto=" + pendingUpto); - seekPending = true; - } - - private void MaybeSeek() - { - if (seekPending) - { - if (pendingFP != lastBlockFP) - { - // need new block - input.Seek(pendingFP); - blockReader.Seek(pendingFP); - lastBlockFP = pendingFP; - blockSize = blockReader.ReadBlock(); - } - upto = pendingUpto; - - // TODO: if we were more clever when writing the - // index, such that a seek point wouldn't be written - // until the int encoder "committed", we could avoid - // this (likely minor) inefficiency: - - // This is necessary for int encoders that are - // non-causal, ie must see future int values to - // encode the current ones. - while (upto >= blockSize) - { - upto -= blockSize; - lastBlockFP = input.FilePointer; - blockSize = blockReader.ReadBlock(); - } - seekPending = false; - } - } - - public override int Next() - { - this.MaybeSeek(); - if (upto == blockSize) - { - lastBlockFP = input.FilePointer; - blockSize = blockReader.ReadBlock(); - upto = 0; - } - - return pending[upto++]; - } - } - - private class Index : AbstractIndex - { - private readonly VariableIntBlockIndexInput outerInstance; - - public Index(VariableIntBlockIndexInput outerInstance) - { - this.outerInstance = outerInstance; - } - - private long fp; - private int upto; - - public override void Read(DataInput indexIn, bool absolute) - { - if (absolute) - { - upto = indexIn.ReadVInt(); - fp = indexIn.ReadVLong(); - } - else - { - int uptoDelta = indexIn.ReadVInt(); - if ((uptoDelta & 1) == 1) - { - // same block - upto += (int)((uint)uptoDelta >> 1); - } - else - { - // new block - upto = (int)((uint)uptoDelta >> 1); - fp += indexIn.ReadVLong(); - } - } - // TODO: we can't do this assert because non-causal - // int encoders can have upto over the buffer size - //assert upto < maxBlockSize: "upto=" + upto + " max=" + maxBlockSize; - } - - public override string ToString() - { - return "VarIntBlock.Index fp=" + fp + " upto=" + upto + " maxBlock=" + outerInstance.m_maxBlockSize; - } - - public override void Seek(AbstractReader other) - { - ((Reader)other).Seek(fp, upto); - } - - public override void CopyFrom(AbstractIndex other) - { - Index idx = (Index)other; - fp = idx.fp; - upto = idx.upto; - } - - public override AbstractIndex Clone() - { - Index other = new Index(outerInstance); - other.fp = fp; - other.upto = upto; - return other; - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0159ad2f/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexOutput.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexOutput.cs b/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexOutput.cs deleted file mode 100644 index f756777..0000000 --- a/src/Lucene.Net.Codecs/Intblock/VariableIntBlockIndexOutput.cs +++ /dev/null @@ -1,162 +0,0 @@ -using Lucene.Net.Codecs.Sep; -using Lucene.Net.Store; -using System.Diagnostics; - -namespace Lucene.Net.Codecs.IntBlock -{ - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - /// <summary> - /// Naive int block API that writes vInts. This is - /// expected to give poor performance; it's really only for - /// testing the pluggability. One should typically use pfor instead. - /// </summary> - - - // TODO: much of this can be shared code w/ the fixed case - - /// <summary> - /// Abstract base class that writes variable-size blocks of ints - /// to an IndexOutput. While this is a simple approach, a - /// more performant approach would directly create an impl - /// of IntIndexOutput inside Directory. Wrapping a generic - /// IndexInput will likely cost performance. - /// - /// @lucene.experimental - /// </summary> - public abstract class VariableIntBlockIndexOutput : IntIndexOutput - { - protected readonly IndexOutput m_output; - - private int upto; - private bool hitExcDuringWrite; - - // TODO what Var-Var codecs exist in practice... and what are there blocksizes like? - // if its less than 128 we should set that as max and use byte? - - /// <summary> - /// NOTE: maxBlockSize must be the maximum block size - /// plus the max non-causal lookahead of your codec. EG Simple9 - /// requires lookahead=1 because on seeing the Nth value - /// it knows it must now encode the N-1 values before it. - /// </summary> - protected VariableIntBlockIndexOutput(IndexOutput output, int maxBlockSize) - { - this.m_output = output; - this.m_output.WriteInt(maxBlockSize); - } - - /// <summary> - /// Called one value at a time. Return the number of - /// buffered input values that have been written to out. - /// </summary> - protected abstract int Add(int value); - - public override AbstractIndex GetIndex() - { - return new Index(this); - } - - private class Index : AbstractIndex - { - private readonly VariableIntBlockIndexOutput outerInstance; - - public Index(VariableIntBlockIndexOutput outerInstance) - { - this.outerInstance = outerInstance; - } - - private long fp; - private int upto; - private long lastFP; - private int lastUpto; - - public override void Mark() - { - fp = outerInstance.m_output.FilePointer; - upto = outerInstance.upto; - } - - public override void CopyFrom(AbstractIndex other, bool copyLast) - { - Index idx = (Index)other; - fp = idx.fp; - upto = idx.upto; - if (copyLast) - { - lastFP = fp; - lastUpto = upto; - } - } - - public override void Write(DataOutput indexOut, bool absolute) - { - Debug.Assert(upto >= 0); - if (absolute) - { - indexOut.WriteVInt(upto); - indexOut.WriteVLong(fp); - } - else if (fp == lastFP) - { - // same block - Debug.Assert(upto >= lastUpto); - int uptoDelta = upto - lastUpto; - indexOut.WriteVInt(uptoDelta << 1 | 1); - } - else - { - // new block - indexOut.WriteVInt(upto << 1); - indexOut.WriteVLong(fp - lastFP); - } - lastUpto = upto; - lastFP = fp; - } - } - - public override void Write(int v) - { - hitExcDuringWrite = true; - upto -= Add(v) - 1; - hitExcDuringWrite = false; - Debug.Assert(upto >= 0); - } - - public override void Dispose() - { - try - { - if (hitExcDuringWrite) return; - - // stuff 0s in until the "real" data is flushed: - var stuffed = 0; - while (upto > stuffed) - { - upto -= Add(0) - 1; - Debug.Assert(upto >= 0); - stuffed += 1; - } - } - finally - { - m_output.Dispose(); - } - } - } -} \ No newline at end of file
