This is an automated email from the ASF dual-hosted git repository. nightowl888 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/lucenenet.git
commit 4ce049d7dc8d91a2506371ab1948193e91bce117 Author: Shad Storhaug <[email protected]> AuthorDate: Sat Aug 29 17:58:23 2020 +0700 Lucene.Net.Suggest: Removed obsolete iterators/methods --- src/Lucene.Net.Suggest/Spell/Dictionary.cs | 6 - .../Spell/HighFrequencyDictionary.cs | 28 -- src/Lucene.Net.Suggest/Spell/LuceneDictionary.cs | 14 - .../Spell/PlainTextDictionary.cs | 44 --- src/Lucene.Net.Suggest/Spell/TermFreqIterator.cs | 41 --- .../Suggest/Analyzing/AnalyzingInfixSuggester.cs | 60 ---- .../Suggest/Analyzing/AnalyzingSuggester.cs | 235 ---------------- .../Suggest/Analyzing/FreeTextSuggester.cs | 162 ----------- .../Suggest/BufferedInputIterator.cs | 101 ------- .../Suggest/BufferingTermFreqIteratorWrapper.cs | 55 ---- .../Suggest/DocumentDictionary.cs | 82 ------ .../Suggest/DocumentValueSourceDictionary.cs | 64 ----- src/Lucene.Net.Suggest/Suggest/FileDictionary.cs | 73 ----- .../Suggest/Fst/BytesRefSorter.cs | 8 - .../Suggest/Fst/ExternalRefSorter.cs | 54 ---- .../Suggest/Fst/FSTCompletionLookup.cs | 104 ------- .../Suggest/Fst/WFSTCompletionLookup.cs | 73 ----- src/Lucene.Net.Suggest/Suggest/InMemorySorter.cs | 7 - src/Lucene.Net.Suggest/Suggest/InputIterator.cs | 82 ------ .../Suggest/Jaspell/JaspellLookup.cs | 37 --- src/Lucene.Net.Suggest/Suggest/Lookup.cs | 7 - .../Suggest/SortedInputIterator.cs | 311 --------------------- .../Suggest/SortedTermFreqIteratorWrapper.cs | 186 ------------ src/Lucene.Net.Suggest/Suggest/Tst/TSTLookup.cs | 35 --- .../Suggest/UnsortedInputIterator.cs | 82 ------ 25 files changed, 1951 deletions(-) diff --git a/src/Lucene.Net.Suggest/Spell/Dictionary.cs b/src/Lucene.Net.Suggest/Spell/Dictionary.cs index 584c5ae..0e45daa 100644 --- a/src/Lucene.Net.Suggest/Spell/Dictionary.cs +++ b/src/Lucene.Net.Suggest/Spell/Dictionary.cs @@ -32,11 +32,5 @@ namespace Lucene.Net.Search.Spell /// Returns an enumerator over all the entries.</summary> /// <returns> Enumerator </returns> IInputEnumerator GetEntryEnumerator(); - - /// <summary> - /// Returns an iterator over all the entries </summary> - /// <returns> Iterator </returns> - [Obsolete("Use GetEntryEnumerator(). This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - IInputIterator GetEntryIterator(); } } \ No newline at end of file diff --git a/src/Lucene.Net.Suggest/Spell/HighFrequencyDictionary.cs b/src/Lucene.Net.Suggest/Spell/HighFrequencyDictionary.cs index 939109d..6744418 100644 --- a/src/Lucene.Net.Suggest/Spell/HighFrequencyDictionary.cs +++ b/src/Lucene.Net.Suggest/Spell/HighFrequencyDictionary.cs @@ -59,16 +59,7 @@ namespace Lucene.Net.Search.Spell return new HighFrequencyEnumerator(this); } - [Obsolete("Use GetEntryEnumerator(). This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public IInputIterator GetEntryIterator() - { - return new HighFrequencyEnumerator(this); - } - internal sealed class HighFrequencyEnumerator : IInputEnumerator -#pragma warning disable CS0618 // Type or member is obsolete - , IInputIterator -#pragma warning restore CS0618 // Type or member is obsolete { internal readonly BytesRef spare = new BytesRef(); internal readonly TermsEnum termsEnum; @@ -97,25 +88,6 @@ namespace Lucene.Net.Search.Spell public long Weight => freq; - [Obsolete("Use MoveNext(), Current instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public BytesRef Next() - { - if (termsEnum != null) - { - BytesRef next; - while ((next = termsEnum.Next()) != null) - { - if (IsFrequent(termsEnum.DocFreq)) - { - freq = termsEnum.DocFreq; - spare.CopyBytes(next); - return spare; - } - } - } - return null; - } - public BytesRef Current => current; public bool MoveNext() diff --git a/src/Lucene.Net.Suggest/Spell/LuceneDictionary.cs b/src/Lucene.Net.Suggest/Spell/LuceneDictionary.cs index 568acd8..983c8aa 100644 --- a/src/Lucene.Net.Suggest/Spell/LuceneDictionary.cs +++ b/src/Lucene.Net.Suggest/Spell/LuceneDictionary.cs @@ -52,19 +52,5 @@ namespace Lucene.Net.Search.Spell return InputEnumerator.EMPTY; } } - - [Obsolete("Use GetEntryEnumerator(). This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public virtual IInputIterator GetEntryIterator() - { - Terms terms = MultiFields.GetTerms(reader, field); - if (terms != null) - { - return new InputIteratorWrapper(terms.GetIterator(null)); - } - else - { - return EmptyInputIterator.Instance; - } - } } } \ No newline at end of file diff --git a/src/Lucene.Net.Suggest/Spell/PlainTextDictionary.cs b/src/Lucene.Net.Suggest/Spell/PlainTextDictionary.cs index 2725d1a..b743492 100644 --- a/src/Lucene.Net.Suggest/Spell/PlainTextDictionary.cs +++ b/src/Lucene.Net.Suggest/Spell/PlainTextDictionary.cs @@ -72,16 +72,7 @@ namespace Lucene.Net.Search.Spell return new InputEnumeratorWrapper(new FileEnumerator(this)); } - [Obsolete("Use GetEntryEnumerator() instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public virtual IInputIterator GetEntryIterator() - { - return new InputIteratorWrapper(new FileEnumerator(this)); - } - internal sealed class FileEnumerator : IBytesRefEnumerator -#pragma warning disable CS0618 // Type or member is obsolete - , IBytesRefIterator -#pragma warning restore CS0618 // Type or member is obsolete { private readonly PlainTextDictionary outerInstance; @@ -94,41 +85,6 @@ namespace Lucene.Net.Search.Spell internal readonly BytesRef spare = new BytesRef(); private BytesRef current; - [Obsolete("Use MoveNext(), Current instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public BytesRef Next() - { - if (done) - { - return null; - } - bool success = false; - BytesRef result; - try - { - string line; - if ((line = [email protected]()) != null) - { - spare.CopyChars(line); - result = spare; - } - else - { - done = true; - IOUtils.Dispose(outerInstance.@in); - result = null; - } - success = true; - } - finally - { - if (!success) - { - IOUtils.DisposeWhileHandlingException(outerInstance.@in); - } - } - return result; - } - public BytesRef Current => current; public bool MoveNext() diff --git a/src/Lucene.Net.Suggest/Spell/TermFreqIterator.cs b/src/Lucene.Net.Suggest/Spell/TermFreqIterator.cs index bcc0b19..f2e586a 100644 --- a/src/Lucene.Net.Suggest/Spell/TermFreqIterator.cs +++ b/src/Lucene.Net.Suggest/Spell/TermFreqIterator.cs @@ -58,45 +58,4 @@ namespace Lucene.Net.Search.Spell public virtual IComparer<BytesRef> Comparer => wrapped.Comparer; } - - /// <summary> - /// Interface for enumerating term,weight pairs. - /// </summary> - [Obsolete("Use TermFreqEnumeratorWrapper instead. This class will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public interface ITermFreqIterator : IBytesRefIterator - { - - /// <summary> - /// Gets the term's weight, higher numbers mean better suggestions. - /// </summary> - long Weight { get; } - } - - /// <summary> - /// Wraps a BytesRefIterator as a TermFreqIterator, with all weights - /// set to <code>1</code> - /// </summary> - [Obsolete("Use TermFreqEnumeratorWrapper instead. This class will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public class TermFreqIteratorWrapper : ITermFreqIterator - { - internal IBytesRefIterator wrapped; - - /// <summary> - /// Creates a new wrapper, wrapping the specified iterator and - /// specifying a weight value of <code>1</code> for all terms. - /// </summary> - public TermFreqIteratorWrapper(IBytesRefIterator wrapped) - { - this.wrapped = wrapped; - } - - public virtual long Weight => 1; - - public virtual BytesRef Next() - { - return wrapped.Next(); - } - - public virtual IComparer<BytesRef> Comparer => wrapped.Comparer; - } } \ No newline at end of file diff --git a/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingInfixSuggester.cs b/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingInfixSuggester.cs index fcf28b8..1b532b7 100644 --- a/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingInfixSuggester.cs +++ b/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingInfixSuggester.cs @@ -246,66 +246,6 @@ namespace Lucene.Net.Search.Suggest.Analyzing } } - [Obsolete("Use Build(IInputEnumerator) instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public override void Build(IInputIterator iter) - { - if (m_searcherMgr != null) - { - m_searcherMgr.Dispose(); - m_searcherMgr = null; - } - - if (writer != null) - { - writer.Dispose(); - writer = null; - } - - AtomicReader r = null; - bool success = false; - try - { - // First pass: build a temporary normal Lucene index, - // just indexing the suggestions as they iterate: - writer = new IndexWriter(dir, GetIndexWriterConfig(matchVersion, GetGramAnalyzer(), OpenMode.CREATE)); - //long t0 = System.nanoTime(); - - // TODO: use threads? - BytesRef text; - while ((text = iter.Next()) != null) - { - BytesRef payload; - if (iter.HasPayloads) - { - payload = iter.Payload; - } - else - { - payload = null; - } - - Add(text, iter.Contexts, iter.Weight, payload); - } - - //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec"); - - m_searcherMgr = new SearcherManager(writer, true, null); - success = true; - } - finally - { - if (success) - { - IOUtils.Dispose(r); - } - else - { - IOUtils.DisposeWhileHandlingException(writer, r); - writer = null; - } - } - } - private Analyzer GetGramAnalyzer() => new AnalyzerWrapperAnonymousInnerClassHelper(this, Analyzer.PER_FIELD_REUSE_STRATEGY); diff --git a/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingSuggester.cs b/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingSuggester.cs index 73fc6c3..4ddb98e 100644 --- a/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingSuggester.cs +++ b/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingSuggester.cs @@ -628,241 +628,6 @@ namespace Lucene.Net.Search.Suggest.Analyzing } } - [Obsolete("Use Build(IInputEnumerator) instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public override void Build(IInputIterator iterator) - { - if (iterator.HasContexts) - { - throw new ArgumentException("this suggester doesn't support contexts"); - } - string prefix = this.GetType().Name; - var directory = OfflineSorter.DefaultTempDir(); - var tempInput = FileSupport.CreateTempFile(prefix, ".input", directory); - var tempSorted = FileSupport.CreateTempFile(prefix, ".sorted", directory); - - hasPayloads = iterator.HasPayloads; - - var writer = new OfflineSorter.ByteSequencesWriter(tempInput); - OfflineSorter.ByteSequencesReader reader = null; - var scratch = new BytesRef(); - - TokenStreamToAutomaton ts2a = GetTokenStreamToAutomaton(); - - bool success = false; - count = 0; - byte[] buffer = new byte[8]; - try - { - var output = new ByteArrayDataOutput(buffer); - BytesRef surfaceForm; - - while ((surfaceForm = iterator.Next()) != null) - { - ISet<Int32sRef> paths = ToFiniteStrings(surfaceForm, ts2a); - - maxAnalyzedPathsForOneInput = Math.Max(maxAnalyzedPathsForOneInput, paths.Count); - - foreach (Int32sRef path in paths) - { - - Util.Fst.Util.ToBytesRef(path, scratch); - - // length of the analyzed text (FST input) - if (scratch.Length > ushort.MaxValue - 2) - { - throw new ArgumentException("cannot handle analyzed forms > " + (ushort.MaxValue - 2) + - " in length (got " + scratch.Length + ")"); - } - ushort analyzedLength = (ushort)scratch.Length; - - // compute the required length: - // analyzed sequence + weight (4) + surface + analyzedLength (short) - int requiredLength = analyzedLength + 4 + surfaceForm.Length + 2; - - BytesRef payload; - - if (hasPayloads) - { - if (surfaceForm.Length > (ushort.MaxValue - 2)) - { - throw new ArgumentException("cannot handle surface form > " + (ushort.MaxValue - 2) + - " in length (got " + surfaceForm.Length + ")"); - } - payload = iterator.Payload; - // payload + surfaceLength (short) - requiredLength += payload.Length + 2; - } - else - { - payload = null; - } - - buffer = ArrayUtil.Grow(buffer, requiredLength); - - output.Reset(buffer); - - output.WriteInt16((short)analyzedLength); - - output.WriteBytes(scratch.Bytes, scratch.Offset, scratch.Length); - - output.WriteInt32(EncodeWeight(iterator.Weight)); - - if (hasPayloads) - { - for (int i = 0; i < surfaceForm.Length; i++) - { - if (surfaceForm.Bytes[i] == PAYLOAD_SEP) - { - throw new ArgumentException( - "surface form cannot contain unit separator character U+001F; this character is reserved"); - } - } - output.WriteInt16((short)surfaceForm.Length); - output.WriteBytes(surfaceForm.Bytes, surfaceForm.Offset, surfaceForm.Length); - output.WriteBytes(payload.Bytes, payload.Offset, payload.Length); - } - else - { - output.WriteBytes(surfaceForm.Bytes, surfaceForm.Offset, surfaceForm.Length); - } - - if (Debugging.AssertsEnabled) Debugging.Assert(output.Position == requiredLength, () => output.Position + " vs " + requiredLength); - - writer.Write(buffer, 0, output.Position); - } - count++; - } - writer.Dispose(); - - // Sort all input/output pairs (required by FST.Builder): - (new OfflineSorter(new AnalyzingComparer(hasPayloads))).Sort(tempInput, tempSorted); - - // Free disk space: - tempInput.Delete(); - - reader = new OfflineSorter.ByteSequencesReader(tempSorted); - - var outputs = new PairOutputs<long?, BytesRef>(PositiveInt32Outputs.Singleton, - ByteSequenceOutputs.Singleton); - var builder = new Builder<PairOutputs<long?, BytesRef>.Pair>(FST.INPUT_TYPE.BYTE1, outputs); - - // Build FST: - BytesRef previousAnalyzed = null; - BytesRef analyzed = new BytesRef(); - BytesRef surface = new BytesRef(); - Int32sRef scratchInts = new Int32sRef(); - var input = new ByteArrayDataInput(); - - // Used to remove duplicate surface forms (but we - // still index the hightest-weight one). We clear - // this when we see a new analyzed form, so it cannot - // grow unbounded (at most 256 entries): - var seenSurfaceForms = new JCG.HashSet<BytesRef>(); - - var dedup = 0; - while (reader.Read(scratch)) - { - input.Reset(scratch.Bytes, scratch.Offset, scratch.Length); - ushort analyzedLength = (ushort)input.ReadInt16(); - analyzed.Grow(analyzedLength + 2); - input.ReadBytes(analyzed.Bytes, 0, analyzedLength); - analyzed.Length = analyzedLength; - - long cost = input.ReadInt32(); - - surface.Bytes = scratch.Bytes; - if (hasPayloads) - { - surface.Length = (ushort)input.ReadInt16(); - surface.Offset = input.Position; - } - else - { - surface.Offset = input.Position; - surface.Length = scratch.Length - surface.Offset; - } - - if (previousAnalyzed == null) - { - previousAnalyzed = new BytesRef(); - previousAnalyzed.CopyBytes(analyzed); - seenSurfaceForms.Add(BytesRef.DeepCopyOf(surface)); - } - else if (analyzed.Equals(previousAnalyzed)) - { - dedup++; - if (dedup >= maxSurfaceFormsPerAnalyzedForm) - { - // More than maxSurfaceFormsPerAnalyzedForm - // dups: skip the rest: - continue; - } - if (seenSurfaceForms.Contains(surface)) - { - continue; - } - seenSurfaceForms.Add(BytesRef.DeepCopyOf(surface)); - } - else - { - dedup = 0; - previousAnalyzed.CopyBytes(analyzed); - seenSurfaceForms.Clear(); - seenSurfaceForms.Add(BytesRef.DeepCopyOf(surface)); - } - - // TODO: I think we can avoid the extra 2 bytes when - // there is no dup (dedup==0), but we'd have to fix - // the exactFirst logic ... which would be sort of - // hairy because we'd need to special case the two - // (dup/not dup)... - - // NOTE: must be byte 0 so we sort before whatever - // is next - analyzed.Bytes[analyzed.Offset + analyzed.Length] = 0; - analyzed.Bytes[analyzed.Offset + analyzed.Length + 1] = (byte)dedup; - analyzed.Length += 2; - - Util.Fst.Util.ToInt32sRef(analyzed, scratchInts); - //System.out.println("ADD: " + scratchInts + " -> " + cost + ": " + surface.utf8ToString()); - if (!hasPayloads) - { - builder.Add(scratchInts, outputs.NewPair(cost, BytesRef.DeepCopyOf(surface))); - } - else - { - int payloadOffset = input.Position + surface.Length; - int payloadLength = scratch.Length - payloadOffset; - BytesRef br = new BytesRef(surface.Length + 1 + payloadLength); - Array.Copy(surface.Bytes, surface.Offset, br.Bytes, 0, surface.Length); - br.Bytes[surface.Length] = PAYLOAD_SEP; - Array.Copy(scratch.Bytes, payloadOffset, br.Bytes, surface.Length + 1, payloadLength); - br.Length = br.Bytes.Length; - builder.Add(scratchInts, outputs.NewPair(cost, br)); - } - } - fst = builder.Finish(); - - //Util.dotToFile(fst, "/tmp/suggest.dot"); - - success = true; - } - finally - { - if (success) - { - IOUtils.Dispose(reader, writer); - } - else - { - IOUtils.DisposeWhileHandlingException(reader, writer); - } - - tempInput.Delete(); - tempSorted.Delete(); - } - } - public override bool Store(DataOutput output) { output.WriteVInt64(count); diff --git a/src/Lucene.Net.Suggest/Suggest/Analyzing/FreeTextSuggester.cs b/src/Lucene.Net.Suggest/Suggest/Analyzing/FreeTextSuggester.cs index f7d4e33..b641140 100644 --- a/src/Lucene.Net.Suggest/Suggest/Analyzing/FreeTextSuggester.cs +++ b/src/Lucene.Net.Suggest/Suggest/Analyzing/FreeTextSuggester.cs @@ -440,168 +440,6 @@ namespace Lucene.Net.Search.Suggest.Analyzing } } - [Obsolete("Use Build(IInputEnumerator) instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public override void Build(IInputIterator iterator) - { - Build(iterator, IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB); - } - - /// <summary> - /// Build the suggest index, using up to the specified - /// amount of temporary RAM while building. Note that - /// the weights for the suggestions are ignored. - /// </summary> - [Obsolete("Use Build(IInputEnumerator) instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public virtual void Build(IInputIterator iterator, double ramBufferSizeMB) - { - if (iterator.HasPayloads) - { - throw new ArgumentException("this suggester doesn't support payloads"); - } - if (iterator.HasContexts) - { - throw new ArgumentException("this suggester doesn't support contexts"); - } - - string prefix = this.GetType().Name; - var directory = OfflineSorter.DefaultTempDir(); - - // LUCENENET specific - using GetRandomFileName() instead of picking a random int - DirectoryInfo tempIndexPath = null; - while (true) - { - tempIndexPath = new DirectoryInfo(Path.Combine(directory.FullName, prefix + ".index." + Path.GetFileNameWithoutExtension(Path.GetRandomFileName()))); - tempIndexPath.Create(); - if (System.IO.Directory.Exists(tempIndexPath.FullName)) - { - break; - } - } - - Directory dir = FSDirectory.Open(tempIndexPath); - try - { -#pragma warning disable 612, 618 - IndexWriterConfig iwc = new IndexWriterConfig(LuceneVersion.LUCENE_CURRENT, indexAnalyzer); -#pragma warning restore 612, 618 - iwc.SetOpenMode(OpenMode.CREATE); - iwc.SetRAMBufferSizeMB(ramBufferSizeMB); - IndexWriter writer = new IndexWriter(dir, iwc); - - var ft = new FieldType(TextField.TYPE_NOT_STORED); - // TODO: if only we had IndexOptions.TERMS_ONLY... - ft.IndexOptions = IndexOptions.DOCS_AND_FREQS; - ft.OmitNorms = true; - ft.Freeze(); - - Document doc = new Document(); - Field field = new Field("body", "", ft); - doc.Add(field); - - totTokens = 0; - IndexReader reader = null; - - bool success = false; - count = 0; - try - { - while (true) - { - BytesRef surfaceForm = iterator.Next(); - if (surfaceForm == null) - { - break; - } - field.SetStringValue(surfaceForm.Utf8ToString()); - writer.AddDocument(doc); - count++; - } - reader = DirectoryReader.Open(writer, false); - - Terms terms = MultiFields.GetTerms(reader, "body"); - if (terms == null) - { - throw new ArgumentException("need at least one suggestion"); - } - - // Move all ngrams into an FST: - TermsEnum termsEnum = terms.GetIterator(null); - - Outputs<long?> outputs = PositiveInt32Outputs.Singleton; - Builder<long?> builder = new Builder<long?>(FST.INPUT_TYPE.BYTE1, outputs); - - Int32sRef scratchInts = new Int32sRef(); - while (true) - { - BytesRef term = termsEnum.Next(); - if (term == null) - { - break; - } - int ngramCount = CountGrams(term); - if (ngramCount > grams) - { - throw new ArgumentException("tokens must not contain separator byte; got token=" + term + " but gramCount=" + ngramCount + ", which is greater than expected max ngram size=" + grams); - } - if (ngramCount == 1) - { - totTokens += termsEnum.TotalTermFreq; - } - - builder.Add(Lucene.Net.Util.Fst.Util.ToInt32sRef(term, scratchInts), EncodeWeight(termsEnum.TotalTermFreq)); - } - - fst = builder.Finish(); - if (fst == null) - { - throw new ArgumentException("need at least one suggestion"); - } - //System.out.println("FST: " + fst.getNodeCount() + " nodes"); - - /* - PrintWriter pw = new PrintWriter("/x/tmp/out.dot"); - Util.toDot(fst, pw, true, true); - pw.close(); - */ - - success = true; - } - finally - { - if (success) - { - IOUtils.Dispose(writer, reader); - } - else - { - IOUtils.DisposeWhileHandlingException(writer, reader); - } - } - } - finally - { - try - { - IOUtils.Dispose(dir); - } - finally - { - // LUCENENET specific - since we are removing the entire directory anyway, - // it doesn't make sense to first do a loop in order remove the files. - // Let the System.IO.Directory.Delete() method handle that. - // We also need to dispose the Directory instance first before deleting from disk. - try - { - System.IO.Directory.Delete(tempIndexPath.FullName, true); - } - catch (Exception e) - { - throw new InvalidOperationException("failed to remove " + tempIndexPath, e); - } - } - } - } - public override bool Store(DataOutput output) { CodecUtil.WriteHeader(output, CODEC_NAME, VERSION_CURRENT); diff --git a/src/Lucene.Net.Suggest/Suggest/BufferedInputIterator.cs b/src/Lucene.Net.Suggest/Suggest/BufferedInputIterator.cs index a6b40a1..9b30605 100644 --- a/src/Lucene.Net.Suggest/Suggest/BufferedInputIterator.cs +++ b/src/Lucene.Net.Suggest/Suggest/BufferedInputIterator.cs @@ -124,105 +124,4 @@ namespace Lucene.Net.Search.Suggest public virtual bool HasContexts => hasContexts; } - - /// <summary> - /// This wrapper buffers incoming elements. - /// @lucene.experimental - /// </summary> - [Obsolete("Use BufferedInputEnumerator instead. This will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public class BufferedInputIterator : IInputIterator - { - // TODO keep this for now - /// <summary> - /// buffered term entries </summary> - protected BytesRefArray m_entries = new BytesRefArray(Counter.NewCounter()); - /// <summary> - /// buffered payload entries </summary> - protected BytesRefArray m_payloads = new BytesRefArray(Counter.NewCounter()); - /// <summary> - /// buffered context set entries </summary> - protected IList<ICollection<BytesRef>> m_contextSets = new List<ICollection<BytesRef>>(); - /// <summary> - /// current buffer position </summary> - protected int m_curPos = -1; - /// <summary> - /// buffered weights, parallel with <see cref="m_entries"/> </summary> - protected long[] m_freqs = new long[1]; - private readonly BytesRef spare = new BytesRef(); - private readonly BytesRef payloadSpare = new BytesRef(); - private readonly bool hasPayloads; - private readonly IComparer<BytesRef> comp; - - private readonly bool hasContexts; - - /// <summary> - /// Creates a new iterator, buffering entries from the specified iterator </summary> - public BufferedInputIterator(IInputIterator source) - { - BytesRef spare; - int freqIndex = 0; - hasPayloads = source.HasPayloads; - hasContexts = source.HasContexts; - while ((spare = source.Next()) != null) - { - m_entries.Append(spare); - if (hasPayloads) - { - m_payloads.Append(source.Payload); - } - if (hasContexts) - { - m_contextSets.Add(source.Contexts); - } - if (freqIndex >= m_freqs.Length) - { - m_freqs = ArrayUtil.Grow(m_freqs, m_freqs.Length + 1); - } - m_freqs[freqIndex++] = source.Weight; - } - comp = source.Comparer; - } - - public virtual long Weight => m_freqs[m_curPos]; - - public virtual BytesRef Next() - { - if (++m_curPos < m_entries.Length) - { - m_entries.Get(spare, m_curPos); - return spare; - } - return null; - } - - public virtual BytesRef Payload - { - get - { - if (hasPayloads && m_curPos < m_payloads.Length) - { - return m_payloads.Get(payloadSpare, m_curPos); - } - return null; - } - } - - public virtual bool HasPayloads => hasPayloads; - - public virtual IComparer<BytesRef> Comparer => comp; - - public virtual ICollection<BytesRef> Contexts - { - get - { - if (hasContexts && m_curPos < m_contextSets.Count) - { - return m_contextSets[m_curPos]; - } - return null; - } - } - - public virtual bool HasContexts => hasContexts; - } } \ No newline at end of file diff --git a/src/Lucene.Net.Suggest/Suggest/BufferingTermFreqIteratorWrapper.cs b/src/Lucene.Net.Suggest/Suggest/BufferingTermFreqIteratorWrapper.cs index f2ca663..7fc4572 100644 --- a/src/Lucene.Net.Suggest/Suggest/BufferingTermFreqIteratorWrapper.cs +++ b/src/Lucene.Net.Suggest/Suggest/BufferingTermFreqIteratorWrapper.cs @@ -80,59 +80,4 @@ namespace Lucene.Net.Search.Suggest public virtual IComparer<BytesRef> Comparer => comp; } - - /// <summary> - /// This wrapper buffers incoming elements. - /// @lucene.experimental - /// </summary> - [Obsolete("Use BufferingTermFreqEnumeratorWrapper instead. This class will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public class BufferingTermFreqIteratorWrapper : ITermFreqIterator - { - // TODO keep this for now - /// <summary> - /// buffered term entries </summary> - protected BytesRefArray m_entries = new BytesRefArray(Counter.NewCounter()); - /// <summary> - /// current buffer position </summary> - protected int m_curPos = -1; - /// <summary> - /// buffered weights, parallel with <see cref="m_entries"/> </summary> - protected long[] m_freqs = new long[1]; - private readonly BytesRef spare = new BytesRef(); - private readonly IComparer<BytesRef> comp; - - /// <summary> - /// Creates a new iterator, buffering entries from the specified iterator - /// </summary> - public BufferingTermFreqIteratorWrapper(ITermFreqIterator source) - { - this.comp = source.Comparer; - BytesRef spare; - int freqIndex = 0; - while ((spare = source.Next()) != null) - { - m_entries.Append(spare); - if (freqIndex >= m_freqs.Length) - { - m_freqs = ArrayUtil.Grow(m_freqs, m_freqs.Length + 1); - } - m_freqs[freqIndex++] = source.Weight; - } - - } - - public virtual long Weight => m_freqs[m_curPos]; - - public virtual BytesRef Next() - { - if (++m_curPos < m_entries.Length) - { - m_entries.Get(spare, m_curPos); - return spare; - } - return null; - } - - public virtual IComparer<BytesRef> Comparer => comp; - } } \ No newline at end of file diff --git a/src/Lucene.Net.Suggest/Suggest/DocumentDictionary.cs b/src/Lucene.Net.Suggest/Suggest/DocumentDictionary.cs index 123ab15..bacd70d 100644 --- a/src/Lucene.Net.Suggest/Suggest/DocumentDictionary.cs +++ b/src/Lucene.Net.Suggest/Suggest/DocumentDictionary.cs @@ -104,29 +104,6 @@ namespace Lucene.Net.Search.Suggest return new DocumentInputEnumerator(this, m_payloadField != null, m_contextsField != null); } - [Obsolete("Use GetEntryEnumerator() instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public virtual IInputIterator GetEntryIterator() - { - return new DocumentInputIterator(this, m_payloadField != null, m_contextsField != null); - } - - /// <summary> - /// Implements <see cref="IInputIterator"/> from stored fields. </summary> - [Obsolete("Use DocumentInputEnumerator instead. This class will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - protected internal class DocumentInputIterator : DocumentInputEnumerator, IInputIterator - { - /// <summary> - /// Creates an iterator over term, weight and payload fields from the lucene - /// index. Setting <paramref name="hasPayloads"/> to <c>false</c>, implies an iterator - /// over only term and weight. - /// </summary> - public DocumentInputIterator(DocumentDictionary outerInstance, bool hasPayloads, bool hasContexts) - : base(outerInstance, hasPayloads, hasContexts) - { } - - public virtual BytesRef Next() => base.Next(); - } - /// <summary> /// Implements <see cref="IInputEnumerator"/> from stored fields. </summary> protected internal class DocumentInputEnumerator : IInputEnumerator @@ -165,65 +142,6 @@ namespace Lucene.Net.Search.Suggest public virtual IComparer<BytesRef> Comparer => null; - [Obsolete("Use MoveNext(), Current instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public virtual BytesRef Next() - { - while (currentDocId < docCount) - { - currentDocId++; - if (liveDocs != null && !liveDocs.Get(currentDocId)) - { - continue; - } - - Document doc = outerInstance.m_reader.Document(currentDocId, relevantFields); - - BytesRef tempPayload = null; - BytesRef tempTerm = null; - ISet<BytesRef> tempContexts = new JCG.HashSet<BytesRef>(); - - if (hasPayloads) - { - IIndexableField payload = doc.GetField(outerInstance.m_payloadField); - if (payload == null || (payload.GetBinaryValue() == null && payload.GetStringValue() == null)) - { - continue; - } - tempPayload = payload.GetBinaryValue() ?? new BytesRef(payload.GetStringValue()); - } - - if (hasContexts) - { - IIndexableField[] contextFields = doc.GetFields(outerInstance.m_contextsField); - foreach (IIndexableField contextField in contextFields) - { - if (contextField.GetBinaryValue() == null && contextField.GetStringValue() == null) - { - continue; - } - else - { - tempContexts.Add(contextField.GetBinaryValue() ?? new BytesRef(contextField.GetStringValue())); - } - } - } - - IIndexableField fieldVal = doc.GetField(outerInstance.field); - if (fieldVal == null || (fieldVal.GetBinaryValue() == null && fieldVal.GetStringValue() == null)) - { - continue; - } - tempTerm = (fieldVal.GetStringValue() != null) ? new BytesRef(fieldVal.GetStringValue()) : fieldVal.GetBinaryValue(); - - currentPayload = tempPayload; - currentContexts = tempContexts; - currentWeight = GetWeight(doc, currentDocId); - - return tempTerm; - } - return null; - } - public BytesRef Current => current; public bool MoveNext() diff --git a/src/Lucene.Net.Suggest/Suggest/DocumentValueSourceDictionary.cs b/src/Lucene.Net.Suggest/Suggest/DocumentValueSourceDictionary.cs index 256b356..bcb5393 100644 --- a/src/Lucene.Net.Suggest/Suggest/DocumentValueSourceDictionary.cs +++ b/src/Lucene.Net.Suggest/Suggest/DocumentValueSourceDictionary.cs @@ -102,12 +102,6 @@ namespace Lucene.Net.Search.Suggest return new DocumentValueSourceInputEnumerator(this, m_payloadField != null, m_contextsField != null); } - [Obsolete("Use GetEntryEnumerator() instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public override IInputIterator GetEntryIterator() - { - return new DocumentValueSourceInputIterator(this, m_payloadField != null, m_contextsField != null); - } - internal sealed class DocumentValueSourceInputEnumerator : DocumentDictionary.DocumentInputEnumerator { private readonly DocumentValueSourceDictionary outerInstance; @@ -164,63 +158,5 @@ namespace Lucene.Net.Search.Suggest return currentWeightValues.Int64Val(docId - starts[subIndex]); } } - - [Obsolete("Use DocumentValueSourceInputEnumerator instead. This class will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - internal sealed class DocumentValueSourceInputIterator : DocumentDictionary.DocumentInputIterator - { - private readonly DocumentValueSourceDictionary outerInstance; - - - internal FunctionValues currentWeightValues; - /// <summary> - /// leaves of the reader </summary> - internal readonly IList<AtomicReaderContext> leaves; - /// <summary> - /// starting docIds of all the leaves </summary> - internal readonly int[] starts; - /// <summary> - /// current leave index </summary> - internal int currentLeafIndex = 0; - - public DocumentValueSourceInputIterator(DocumentValueSourceDictionary outerInstance, bool hasPayloads, bool hasContexts) - : base(outerInstance, hasPayloads, hasContexts) - { - this.outerInstance = outerInstance; - leaves = outerInstance.m_reader.Leaves; - starts = new int[leaves.Count + 1]; - for (int i = 0; i < leaves.Count; i++) - { - starts[i] = leaves[i].DocBase; - } - starts[leaves.Count] = outerInstance.m_reader.MaxDoc; - currentWeightValues = (leaves.Count > 0) ? outerInstance.weightsValueSource.GetValues(new Dictionary<string, object>(), leaves[currentLeafIndex]) : null; - } - - /// <summary> - /// Returns the weight for the current <paramref name="docId"/> as computed - /// by the <see cref="weightsValueSource"/> - /// </summary> - protected internal override long GetWeight(Document doc, int docId) - { - if (currentWeightValues == null) - { - return 0; - } - int subIndex = ReaderUtil.SubIndex(docId, starts); - if (subIndex != currentLeafIndex) - { - currentLeafIndex = subIndex; - try - { - currentWeightValues = outerInstance.weightsValueSource.GetValues(new Dictionary<string, object>(), leaves[currentLeafIndex]); - } - catch (IOException e) - { - throw new Exception(e.ToString(), e); - } - } - return currentWeightValues.Int64Val(docId - starts[subIndex]); - } - } } } \ No newline at end of file diff --git a/src/Lucene.Net.Suggest/Suggest/FileDictionary.cs b/src/Lucene.Net.Suggest/Suggest/FileDictionary.cs index 8030b81..bcf70b7 100644 --- a/src/Lucene.Net.Suggest/Suggest/FileDictionary.cs +++ b/src/Lucene.Net.Suggest/Suggest/FileDictionary.cs @@ -124,23 +124,7 @@ namespace Lucene.Net.Search.Suggest } } - [Obsolete("Use GetEntryEnumerator() instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public virtual IInputIterator GetEntryIterator() - { - try - { - return new FileEnumerator(this); - } - catch (IOException e) - { - throw new Exception(e.ToString(), e); - } - } - internal sealed class FileEnumerator : IInputEnumerator -#pragma warning disable CS0618 // Type or member is obsolete - , IInputIterator -#pragma warning restore CS0618 // Type or member is obsolete { private readonly FileDictionary outerInstance; @@ -189,63 +173,6 @@ namespace Lucene.Net.Search.Suggest public long Weight => curWeight; - [Obsolete("Use MoveNext(), Current instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public BytesRef Next() - { - if (outerInstance.done) - { - return null; - } - if (isFirstLine) - { - isFirstLine = false; - return spare; - } - outerInstance.line = [email protected](); - if (outerInstance.line != null) - { - string[] fields = outerInstance.line.Split(new string[] { outerInstance.fieldDelimiter }, StringSplitOptions.RemoveEmptyEntries); - if (fields.Length > 3) - { - throw new ArgumentException("More than 3 fields in one line"); - } // term, weight and payload - else if (fields.Length == 3) - { - spare.CopyChars(fields[0]); - ReadWeight(fields[1]); - if (hasPayloads) - { - curPayload.CopyChars(fields[2]); - } - } // term, weight - else if (fields.Length == 2) - { - spare.CopyChars(fields[0]); - ReadWeight(fields[1]); - if (hasPayloads) // have an empty payload - { - curPayload = new BytesRef(); - } - } // only term - else - { - spare.CopyChars(fields[0]); - curWeight = 1; - if (hasPayloads) - { - curPayload = new BytesRef(); - } - } - return spare; - } - else - { - outerInstance.done = true; - IOUtils.Dispose(outerInstance.@in); - return null; - } - } - public BytesRef Current => current; public bool MoveNext() diff --git a/src/Lucene.Net.Suggest/Suggest/Fst/BytesRefSorter.cs b/src/Lucene.Net.Suggest/Suggest/Fst/BytesRefSorter.cs index b079c16..5cf0c92 100644 --- a/src/Lucene.Net.Suggest/Suggest/Fst/BytesRefSorter.cs +++ b/src/Lucene.Net.Suggest/Suggest/Fst/BytesRefSorter.cs @@ -44,14 +44,6 @@ namespace Lucene.Net.Search.Suggest.Fst IBytesRefEnumerator GetEnumerator(); /// <summary> - /// Sorts the entries added in <see cref="Add(BytesRef)"/> and returns - /// an iterator over all sorted entries. - /// </summary> - /// <exception cref="IOException"> If an I/O exception occurs. </exception> - [Obsolete("Use GetEnumerator() instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - IBytesRefIterator GetIterator(); - - /// <summary> /// Comparer used to determine the sort order of entries. /// </summary> IComparer<BytesRef> Comparer { get; } diff --git a/src/Lucene.Net.Suggest/Suggest/Fst/ExternalRefSorter.cs b/src/Lucene.Net.Suggest/Suggest/Fst/ExternalRefSorter.cs index 60760af..95fa2f0 100644 --- a/src/Lucene.Net.Suggest/Suggest/Fst/ExternalRefSorter.cs +++ b/src/Lucene.Net.Suggest/Suggest/Fst/ExternalRefSorter.cs @@ -52,23 +52,6 @@ namespace Lucene.Net.Search.Suggest.Fst writer.Write(utf8); } - [Obsolete("Use GetEnumerator() instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public virtual IBytesRefIterator GetIterator() - { - if (sorted == null) - { - CloseWriter(); - - sorted = new FileInfo(Path.GetTempFileName()); - sort.Sort(input, sorted); - - input.Delete(); - input = null; - } - - return new ByteSequenceEnumerator(new OfflineSorter.ByteSequencesReader(sorted), sort.Comparer); - } - public virtual IBytesRefEnumerator GetEnumerator() { if (sorted == null) @@ -131,9 +114,6 @@ namespace Lucene.Net.Search.Suggest.Fst /// Iterate over byte refs in a file. /// </summary> internal class ByteSequenceEnumerator : IBytesRefEnumerator -#pragma warning disable CS0618 // Type or member is obsolete - , IBytesRefIterator -#pragma warning restore CS0618 // Type or member is obsolete { private readonly OfflineSorter.ByteSequencesReader reader; private BytesRef scratch = new BytesRef(); @@ -145,40 +125,6 @@ namespace Lucene.Net.Search.Suggest.Fst this.comparer = comparer; } - [Obsolete("Use MoveNext(), Current instead. This method will be removed in 4.8.0."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public virtual BytesRef Next() - { - if (scratch == null) - { - return null; - } - bool success = false; - try - { - byte[] next = reader.Read(); - if (next != null) - { - scratch.Bytes = next; - scratch.Length = next.Length; - scratch.Offset = 0; - } - else - { - IOUtils.Dispose(reader); - scratch = null; - } - success = true; - return scratch; - } - finally - { - if (!success) - { - IOUtils.DisposeWhileHandlingException(reader); - } - } - } - public BytesRef Current => scratch; public bool MoveNext() diff --git a/src/Lucene.Net.Suggest/Suggest/Fst/FSTCompletionLookup.cs b/src/Lucene.Net.Suggest/Suggest/Fst/FSTCompletionLookup.cs index 779619a..bc2f877 100644 --- a/src/Lucene.Net.Suggest/Suggest/Fst/FSTCompletionLookup.cs +++ b/src/Lucene.Net.Suggest/Suggest/Fst/FSTCompletionLookup.cs @@ -241,110 +241,6 @@ namespace Lucene.Net.Search.Suggest.Fst } } - [Obsolete("Use Build(IInputEnumerator) instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public override void Build(IInputIterator iterator) - { - if (iterator.HasPayloads) - { - throw new ArgumentException("this suggester doesn't support payloads"); - } - if (iterator.HasContexts) - { - throw new ArgumentException("this suggester doesn't support contexts"); - } - FileInfo tempInput = FileSupport.CreateTempFile(typeof(FSTCompletionLookup).Name, ".input", OfflineSorter.DefaultTempDir()); - FileInfo tempSorted = FileSupport.CreateTempFile(typeof(FSTCompletionLookup).Name, ".sorted", OfflineSorter.DefaultTempDir()); - - OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput); - OfflineSorter.ByteSequencesReader reader = null; - ExternalRefSorter sorter = null; - - // Push floats up front before sequences to sort them. For now, assume they are non-negative. - // If negative floats are allowed some trickery needs to be done to find their byte order. - bool success = false; - count = 0; - try - { - byte[] buffer = Arrays.Empty<byte>(); - ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); - BytesRef spare; - while ((spare = iterator.Next()) != null) - { - if (spare.Length + 4 >= buffer.Length) - { - buffer = ArrayUtil.Grow(buffer, spare.Length + 4); - } - - output.Reset(buffer); - output.WriteInt32(EncodeWeight(iterator.Weight)); - output.WriteBytes(spare.Bytes, spare.Offset, spare.Length); - writer.Write(buffer, 0, output.Position); - } - writer.Dispose(); - - // We don't know the distribution of scores and we need to bucket them, so we'll sort - // and divide into equal buckets. - OfflineSorter.SortInfo info = (new OfflineSorter()).Sort(tempInput, tempSorted); - tempInput.Delete(); - FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter = new ExternalRefSorter(new OfflineSorter()), sharedTailLength); - - int inputLines = info.Lines; - reader = new OfflineSorter.ByteSequencesReader(tempSorted); - long line = 0; - int previousBucket = 0; - int previousScore = 0; - ByteArrayDataInput input = new ByteArrayDataInput(); - BytesRef tmp1 = new BytesRef(); - BytesRef tmp2 = new BytesRef(); - while (reader.Read(tmp1)) - { - input.Reset(tmp1.Bytes); - int currentScore = input.ReadInt32(); - - int bucket; - if (line > 0 && currentScore == previousScore) - { - bucket = previousBucket; - } - else - { - bucket = (int)(line * buckets / inputLines); - } - previousScore = currentScore; - previousBucket = bucket; - - // Only append the input, discard the weight. - tmp2.Bytes = tmp1.Bytes; - tmp2.Offset = input.Position; - tmp2.Length = tmp1.Length - input.Position; - builder.Add(tmp2, bucket); - - line++; - count++; - } - - // The two FSTCompletions share the same automaton. - this.higherWeightsCompletion = builder.Build(); - this.normalCompletion = new FSTCompletion(higherWeightsCompletion.FST, false, exactMatchFirst); - - success = true; - } - finally - { - if (success) - { - IOUtils.Dispose(reader, writer, sorter); - } - else - { - IOUtils.DisposeWhileHandlingException(reader, writer, sorter); - } - - tempInput.Delete(); - tempSorted.Delete(); - } - } - /// <summary> /// weight -> cost </summary> private static int EncodeWeight(long value) diff --git a/src/Lucene.Net.Suggest/Suggest/Fst/WFSTCompletionLookup.cs b/src/Lucene.Net.Suggest/Suggest/Fst/WFSTCompletionLookup.cs index af95407..add314c 100644 --- a/src/Lucene.Net.Suggest/Suggest/Fst/WFSTCompletionLookup.cs +++ b/src/Lucene.Net.Suggest/Suggest/Fst/WFSTCompletionLookup.cs @@ -117,45 +117,6 @@ namespace Lucene.Net.Search.Suggest.Fst fst = builder.Finish(); } - [Obsolete("Use Build(IInputEnumerator) instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public override void Build(IInputIterator iterator) - { - if (iterator.HasPayloads) - { - throw new ArgumentException("this suggester doesn't support payloads"); - } - if (iterator.HasContexts) - { - throw new ArgumentException("this suggester doesn't support contexts"); - } - count = 0; - var scratch = new BytesRef(); - IInputIterator iter = new WFSTInputIterator(this, iterator); - var scratchInts = new Int32sRef(); - BytesRef previous = null; - var outputs = PositiveInt32Outputs.Singleton; - var builder = new Builder<long?>(FST.INPUT_TYPE.BYTE1, outputs); - while ((scratch = iter.Next()) != null) - { - long cost = iter.Weight; - - if (previous == null) - { - previous = new BytesRef(); - } - else if (scratch.Equals(previous)) - { - continue; // for duplicate suggestions, the best weight is actually - // added - } - Lucene.Net.Util.Fst.Util.ToInt32sRef(scratch, scratchInts); - builder.Add(scratchInts, cost); - previous.CopyBytes(scratch); - count++; - } - fst = builder.Finish(); - } - public override bool Store(DataOutput output) { output.WriteVInt64(count); @@ -354,40 +315,6 @@ namespace Lucene.Net.Search.Suggest.Fst } } - [Obsolete("Use WFSTInputEnumerator instead. This class will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - private sealed class WFSTInputIterator : SortedInputIterator - { - private readonly WFSTCompletionLookup outerInstance; - - - internal WFSTInputIterator(WFSTCompletionLookup outerInstance, IInputIterator source) - : base(source) - { - this.outerInstance = outerInstance; - if (Debugging.AssertsEnabled) Debugging.Assert(source.HasPayloads == false); - } - - protected internal override void Encode(OfflineSorter.ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, BytesRef payload, ICollection<BytesRef> contexts, long weight) - { - if (spare.Length + 4 >= buffer.Length) - { - buffer = ArrayUtil.Grow(buffer, spare.Length + 4); - } - output.Reset(buffer); - output.WriteBytes(spare.Bytes, spare.Offset, spare.Length); - output.WriteInt32(EncodeWeight(weight)); - writer.Write(buffer, 0, output.Position); - } - - protected internal override long Decode(BytesRef scratch, ByteArrayDataInput tmpInput) - { - scratch.Length -= 4; // int - // skip suggestion: - tmpInput.Reset(scratch.Bytes, scratch.Offset + scratch.Length, 4); - return tmpInput.ReadInt32(); - } - } - internal static readonly IComparer<long?> weightComparer = Comparer<long?>.Create((left, right) => Comparer<long?>.Default.Compare(left, right)); /// <summary> diff --git a/src/Lucene.Net.Suggest/Suggest/InMemorySorter.cs b/src/Lucene.Net.Suggest/Suggest/InMemorySorter.cs index 4371c9d..ee4e238 100644 --- a/src/Lucene.Net.Suggest/Suggest/InMemorySorter.cs +++ b/src/Lucene.Net.Suggest/Suggest/InMemorySorter.cs @@ -57,13 +57,6 @@ namespace Lucene.Net.Search.Suggest return buffer.GetEnumerator(comparer); } - [Obsolete("Use GetEnumerator() instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public IBytesRefIterator GetIterator() - { - closed = true; - return buffer.GetIterator(comparer); - } - public IComparer<BytesRef> Comparer => comparer; } } \ No newline at end of file diff --git a/src/Lucene.Net.Suggest/Suggest/InputIterator.cs b/src/Lucene.Net.Suggest/Suggest/InputIterator.cs index 882a538..383fabc 100644 --- a/src/Lucene.Net.Suggest/Suggest/InputIterator.cs +++ b/src/Lucene.Net.Suggest/Suggest/InputIterator.cs @@ -96,87 +96,5 @@ namespace Lucene.Net.Search.Suggest public virtual ICollection<BytesRef> Contexts => null; public virtual bool HasContexts => false; - - - } - - - /// <summary> - /// Interface for enumerating term,weight,payload triples for suggester consumption; - /// currently only <see cref="Analyzing.AnalyzingSuggester"/>, <see cref="Analyzing.FuzzySuggester"/> - /// and <see cref="Analyzing.AnalyzingInfixSuggester"/> support payloads. - /// </summary> - [Obsolete("This interface will be removed in 4.8.0 release candidate"), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public interface IInputIterator : IBytesRefIterator - { - - /// <summary> - /// A term's weight, higher numbers mean better suggestions. </summary> - long Weight { get; } - - /// <summary> - /// An arbitrary byte[] to record per suggestion. See - /// <see cref="Lookup.LookupResult.Payload"/> to retrieve the payload - /// for each suggestion. - /// </summary> - BytesRef Payload { get; } - - /// <summary> - /// Returns true if the iterator has payloads </summary> - bool HasPayloads { get; } - - /// <summary> - /// A term's contexts context can be used to filter suggestions. - /// May return null, if suggest entries do not have any context - /// </summary> - ICollection<BytesRef> Contexts { get; } - - /// <summary> - /// Returns true if the iterator has contexts </summary> - bool HasContexts { get; } - } - - /// <summary> - /// Singleton <see cref="IInputIterator"/> that iterates over 0 BytesRefs. - /// </summary> - [Obsolete("This class will be removed in 4.8.0 release candidate"), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public static class EmptyInputIterator - { - public static readonly IInputIterator Instance = new InputIteratorWrapper(BytesRefIterator.EMPTY); - } - - /// <summary> - /// Wraps a <see cref="IBytesRefIterator"/> as a suggester <see cref="IInputIterator"/>, with all weights - /// set to <c>1</c> and carries no payload - /// </summary> - [Obsolete("This interface will be removed in 4.8.0 release candidate"), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public class InputIteratorWrapper : IInputIterator - { - internal readonly IBytesRefIterator wrapped; - - /// <summary> - /// Creates a new wrapper, wrapping the specified iterator and - /// specifying a weight value of <c>1</c> for all terms - /// and nullifies associated payloads. - /// </summary> - public InputIteratorWrapper(IBytesRefIterator wrapped) - { - this.wrapped = wrapped; - } - - public virtual long Weight => 1; - - public virtual BytesRef Next() - => wrapped.Next(); - - public virtual BytesRef Payload => null; - - public virtual bool HasPayloads => false; - - public virtual IComparer<BytesRef> Comparer => wrapped.Comparer; - - public virtual ICollection<BytesRef> Contexts => null; - - public virtual bool HasContexts => false; } } \ No newline at end of file diff --git a/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellLookup.cs b/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellLookup.cs index c0fc19a..d3396d5 100644 --- a/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellLookup.cs +++ b/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellLookup.cs @@ -81,43 +81,6 @@ namespace Lucene.Net.Search.Suggest.Jaspell } } - [Obsolete("Use Build(IInputEnumerator) instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public override void Build(IInputIterator tfit) - { - if (tfit.HasPayloads) - { - throw new ArgumentException("this suggester doesn't support payloads"); - } - if (tfit.Comparer != null) - { - // make sure it's unsorted - // WTF - this could result in yet another sorted iteration.... - tfit = new UnsortedInputIterator(tfit); - } - if (tfit.HasContexts) - { - throw new ArgumentException("this suggester doesn't support contexts"); - } - count = 0; - trie = new JaspellTernarySearchTrie { MatchAlmostDiff = editDistance }; - BytesRef spare; - - var charsSpare = new CharsRef(); - - while ((spare = tfit.Next()) != null) - { - - long weight = tfit.Weight; - if (spare.Length == 0) - { - continue; - } - charsSpare.Grow(spare.Length); - UnicodeUtil.UTF8toUTF16(spare.Bytes, spare.Offset, spare.Length, charsSpare); - trie.Put(charsSpare.ToString(), weight); - } - } - /// <summary> /// Adds a new node if <code>key</code> already exists, /// otherwise replaces its value. diff --git a/src/Lucene.Net.Suggest/Suggest/Lookup.cs b/src/Lucene.Net.Suggest/Suggest/Lookup.cs index 7c85546..ca74e4f 100644 --- a/src/Lucene.Net.Suggest/Suggest/Lookup.cs +++ b/src/Lucene.Net.Suggest/Suggest/Lookup.cs @@ -251,13 +251,6 @@ namespace Lucene.Net.Search.Suggest public abstract void Build(IInputEnumerator inputEnumerator); /// <summary> - /// Builds up a new internal <see cref="Lookup"/> representation based on the given <see cref="IInputIterator"/>. - /// The implementation might re-sort the data internally. - /// </summary> - [Obsolete("Use Build(IInputEnumerator) instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public abstract void Build(IInputIterator inputIterator); - - /// <summary> /// Look up a key and return possible completion for this key. </summary> /// <param name="key"> lookup key. Depending on the implementation this may be /// a prefix, misspelling, or even infix. </param> diff --git a/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs b/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs index 295b9c5..5154192 100644 --- a/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs +++ b/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs @@ -337,315 +337,4 @@ namespace Lucene.Net.Search.Suggest return payloadScratch; } } - - /// <summary> - /// This wrapper buffers incoming elements and makes sure they are sorted based on given comparer. - /// @lucene.experimental - /// </summary> - [Obsolete("Use SortedInputEnumerator instead. This class will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public class SortedInputIterator : IInputIterator - { - private readonly IInputIterator source; - private FileInfo tempInput; - private FileInfo tempSorted; - private readonly OfflineSorter.ByteSequencesReader reader; - private readonly IComparer<BytesRef> comparer; - private readonly bool hasPayloads; - private readonly bool hasContexts; - private bool done = false; - - private long weight; - private readonly BytesRef scratch = new BytesRef(); - private BytesRef payload = new BytesRef(); - private ISet<BytesRef> contexts = null; - - /// <summary> - /// Creates a new sorted wrapper, using <see cref="BytesRef.UTF8SortedAsUnicodeComparer"/> - /// for sorting. - /// </summary> - public SortedInputIterator(IInputIterator source) - : this(source, BytesRef.UTF8SortedAsUnicodeComparer) - { - } - - /// <summary> - /// Creates a new sorted wrapper, sorting by BytesRef - /// (ascending) then cost (ascending). - /// </summary> - public SortedInputIterator(IInputIterator source, IComparer<BytesRef> comparer) - { - this.tieBreakByCostComparer = Comparer<BytesRef>.Create((left, right) => - { - SortedInputIterator outerInstance = this; - - BytesRef leftScratch = new BytesRef(); - BytesRef rightScratch = new BytesRef(); - ByteArrayDataInput input = new ByteArrayDataInput(); - // Make shallow copy in case decode changes the BytesRef: - leftScratch.Bytes = left.Bytes; - leftScratch.Offset = left.Offset; - leftScratch.Length = left.Length; - rightScratch.Bytes = right.Bytes; - rightScratch.Offset = right.Offset; - rightScratch.Length = right.Length; - long leftCost = outerInstance.Decode(leftScratch, input); - long rightCost = outerInstance.Decode(rightScratch, input); - if (outerInstance.HasPayloads) - { - outerInstance.DecodePayload(leftScratch, input); - outerInstance.DecodePayload(rightScratch, input); - } - if (outerInstance.HasContexts) - { - outerInstance.DecodeContexts(leftScratch, input); - outerInstance.DecodeContexts(rightScratch, input); - } - // LUCENENET NOTE: outerInstance.Comparer != outerInstance.comparer!! - int cmp = outerInstance.comparer.Compare(leftScratch, rightScratch); - if (cmp != 0) - { - return cmp; - } - if (leftCost < rightCost) - { - return -1; - } - else if (leftCost > rightCost) - { - return 1; - } - else - { - return 0; - } - }); - - this.hasPayloads = source.HasPayloads; - this.hasContexts = source.HasContexts; - this.source = source; - this.comparer = comparer; - this.reader = Sort(); - } - - public virtual BytesRef Next() - { - bool success = false; - if (done) - { - return null; - } - try - { - var input = new ByteArrayDataInput(); - if (reader.Read(scratch)) - { - weight = Decode(scratch, input); - if (hasPayloads) - { - payload = DecodePayload(scratch, input); - } - if (hasContexts) - { - contexts = DecodeContexts(scratch, input); - } - success = true; - return scratch; - } - Dispose(); - success = done = true; - return null; - } - finally - { - if (!success) - { - done = true; - Dispose(); - } - } - } - - public virtual long Weight => weight; - - public virtual BytesRef Payload - { - get - { - if (hasPayloads) - { - return payload; - } - return null; - } - } - - public virtual bool HasPayloads => hasPayloads; - - public virtual ICollection<BytesRef> Contexts => contexts; - - public virtual IComparer<BytesRef> Comparer => tieBreakByCostComparer; - - public virtual bool HasContexts => hasContexts; - - /// <summary> - /// Sortes by BytesRef (ascending) then cost (ascending). </summary> - private readonly IComparer<BytesRef> tieBreakByCostComparer; - - private OfflineSorter.ByteSequencesReader Sort() - { - string prefix = this.GetType().Name; - DirectoryInfo directory = OfflineSorter.DefaultTempDir(); - tempInput = FileSupport.CreateTempFile(prefix, ".input", directory); - tempSorted = FileSupport.CreateTempFile(prefix, ".sorted", directory); - - var writer = new OfflineSorter.ByteSequencesWriter(tempInput); - bool success = false; - try - { - BytesRef spare; - byte[] buffer = Arrays.Empty<byte>(); - var output = new ByteArrayDataOutput(buffer); - - while ((spare = source.Next()) != null) - { - Encode(writer, output, buffer, spare, source.Payload, source.Contexts, source.Weight); - } - writer.Dispose(); - (new OfflineSorter(tieBreakByCostComparer)).Sort(tempInput, tempSorted); - var reader = new OfflineSorter.ByteSequencesReader(tempSorted); - success = true; - return reader; - - } - finally - { - if (success) - { - IOUtils.Dispose(writer); - } - else - { - try - { - IOUtils.DisposeWhileHandlingException(writer); - } - finally - { - Dispose(); - } - } - } - } - - private void Dispose() - { - IOUtils.Dispose(reader); - if (tempInput != null) - { - tempInput.Delete(); - } - if (tempSorted != null) - { - tempSorted.Delete(); - } - } - - /// <summary> - /// encodes an entry (bytes+(contexts)+(payload)+weight) to the provided writer - /// </summary> - protected internal virtual void Encode(OfflineSorter.ByteSequencesWriter writer, - ByteArrayDataOutput output, byte[] buffer, BytesRef spare, BytesRef payload, - ICollection<BytesRef> contexts, long weight) - { - int requiredLength = spare.Length + 8 + ((hasPayloads) ? 2 + payload.Length : 0); - if (hasContexts) - { - foreach (BytesRef ctx in contexts) - { - requiredLength += 2 + ctx.Length; - } - requiredLength += 2; // for length of contexts - } - if (requiredLength >= buffer.Length) - { - buffer = ArrayUtil.Grow(buffer, requiredLength); - } - output.Reset(buffer); - output.WriteBytes(spare.Bytes, spare.Offset, spare.Length); - if (hasContexts) - { - foreach (BytesRef ctx in contexts) - { - output.WriteBytes(ctx.Bytes, ctx.Offset, ctx.Length); - output.WriteInt16((short)ctx.Length); - } - output.WriteInt16((short)contexts.Count); - } - if (hasPayloads) - { - output.WriteBytes(payload.Bytes, payload.Offset, payload.Length); - output.WriteInt16((short)payload.Length); - } - output.WriteInt64(weight); - writer.Write(buffer, 0, output.Position); - } - - /// <summary> - /// decodes the weight at the current position </summary> - protected internal virtual long Decode(BytesRef scratch, ByteArrayDataInput tmpInput) - { - tmpInput.Reset(scratch.Bytes); - tmpInput.SkipBytes(scratch.Length - 8); // suggestion - scratch.Length -= 8; // long - return tmpInput.ReadInt64(); - } - - /// <summary> - /// decodes the contexts at the current position </summary> - protected internal virtual ISet<BytesRef> DecodeContexts(BytesRef scratch, ByteArrayDataInput tmpInput) - { - tmpInput.Reset(scratch.Bytes); - tmpInput.SkipBytes(scratch.Length - 2); //skip to context set size - ushort ctxSetSize = (ushort)tmpInput.ReadInt16(); - scratch.Length -= 2; - - var contextSet = new JCG.HashSet<BytesRef>(); - for (ushort i = 0; i < ctxSetSize; i++) - { - tmpInput.Position = scratch.Length - 2; - ushort curContextLength = (ushort)tmpInput.ReadInt16(); - scratch.Length -= 2; - tmpInput.Position = scratch.Length - curContextLength; - BytesRef contextSpare = new BytesRef(curContextLength); - tmpInput.ReadBytes(contextSpare.Bytes, 0, curContextLength); - contextSpare.Length = curContextLength; - contextSet.Add(contextSpare); - scratch.Length -= curContextLength; - } - - // LUCENENET TODO: We are writing the data forward. - // Not sure exactly why, but when we read it back it - // is reversed. So, we need to fix that before returning the result. - // If the underlying problem is found and fixed, then this line can just be - // return contextSet; - return new JCG.HashSet<BytesRef>(contextSet.Reverse()); - } - - /// <summary> - /// decodes the payload at the current position - /// </summary> - protected internal virtual BytesRef DecodePayload(BytesRef scratch, ByteArrayDataInput tmpInput) - { - tmpInput.Reset(scratch.Bytes); - tmpInput.SkipBytes(scratch.Length - 2); // skip to payload size - ushort payloadLength = (ushort)tmpInput.ReadInt16(); // read payload size - tmpInput.Position = scratch.Length - 2 - payloadLength; // setPosition to start of payload - BytesRef payloadScratch = new BytesRef(payloadLength); - tmpInput.ReadBytes(payloadScratch.Bytes, 0, payloadLength); // read payload - payloadScratch.Length = payloadLength; - scratch.Length -= 2; // payload length info (short) - scratch.Length -= payloadLength; // payload - return payloadScratch; - } - } } \ No newline at end of file diff --git a/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs b/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs index a02e5a8..bb764e7 100644 --- a/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs +++ b/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs @@ -209,190 +209,4 @@ namespace Lucene.Net.Search.Suggest return tmpInput.ReadInt64(); } } - - /// <summary> - /// This wrapper buffers incoming elements and makes sure they are sorted based on given comparer. - /// @lucene.experimental - /// </summary> - [Obsolete("Use SortedTermFreqEnumeratorWrapper instead. This class will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public class SortedTermFreqIteratorWrapper : ITermFreqIterator - { - - private readonly ITermFreqIterator source; - private FileInfo tempInput; - private FileInfo tempSorted; - private readonly OfflineSorter.ByteSequencesReader reader; - private readonly IComparer<BytesRef> comparer; - private bool done = false; - - private long weight; - private readonly BytesRef scratch = new BytesRef(); - - /// <summary> - /// Creates a new sorted wrapper, using <see cref="BytesRef.UTF8SortedAsUnicodeComparer"/> - /// for sorting. - /// </summary> - public SortedTermFreqIteratorWrapper(ITermFreqIterator source) - : this(source, BytesRef.UTF8SortedAsUnicodeComparer) - { - } - - /// <summary> - /// Creates a new sorted wrapper, sorting by BytesRef - /// (ascending) then cost (ascending). - /// </summary> - public SortedTermFreqIteratorWrapper(ITermFreqIterator source, IComparer<BytesRef> comparer) - { - this.source = source; - this.comparer = comparer; - this.reader = Sort(); - this.tieBreakByCostComparer = Comparer<BytesRef>.Create((left, right) => - { - SortedTermFreqIteratorWrapper outerInstance = this; - BytesRef leftScratch = new BytesRef(); - BytesRef rightScratch = new BytesRef(); - - ByteArrayDataInput input = new ByteArrayDataInput(); - // Make shallow copy in case decode changes the BytesRef: - leftScratch.Bytes = left.Bytes; - leftScratch.Offset = left.Offset; - leftScratch.Length = left.Length; - rightScratch.Bytes = right.Bytes; - rightScratch.Offset = right.Offset; - rightScratch.Length = right.Length; - long leftCost = outerInstance.Decode(leftScratch, input); - long rightCost = outerInstance.Decode(rightScratch, input); - int cmp = outerInstance.comparer.Compare(leftScratch, rightScratch); - if (cmp != 0) - { - return cmp; - } - return leftCost.CompareTo(rightCost); - }); - } - - public virtual IComparer<BytesRef> Comparer => comparer; - - public virtual BytesRef Next() - { - bool success = false; - if (done) - { - return null; - } - try - { - var input = new ByteArrayDataInput(); - if (reader.Read(scratch)) - { - weight = Decode(scratch, input); - success = true; - return scratch; - } - Dispose(); - success = done = true; - return null; - } - finally - { - if (!success) - { - done = true; - Dispose(); - } - } - } - - public virtual long Weight => weight; - - /// <summary> - /// Sortes by BytesRef (ascending) then cost (ascending). - /// </summary> - private readonly IComparer<BytesRef> tieBreakByCostComparer; - - private OfflineSorter.ByteSequencesReader Sort() - { - string prefix = this.GetType().Name; - DirectoryInfo directory = OfflineSorter.DefaultTempDir(); - tempInput = FileSupport.CreateTempFile(prefix, ".input", directory); - tempSorted = FileSupport.CreateTempFile(prefix, ".sorted", directory); - - var writer = new OfflineSorter.ByteSequencesWriter(tempInput); - bool success = false; - try - { - BytesRef spare; - byte[] buffer = Arrays.Empty<byte>(); - ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); - - while ((spare = source.Next()) != null) - { - Encode(writer, output, buffer, spare, source.Weight); - } - writer.Dispose(); - (new OfflineSorter(tieBreakByCostComparer)).Sort(tempInput, tempSorted); - OfflineSorter.ByteSequencesReader reader = new OfflineSorter.ByteSequencesReader(tempSorted); - success = true; - return reader; - - } - finally - { - if (success) - { - IOUtils.Dispose(writer); - } - else - { - try - { - IOUtils.DisposeWhileHandlingException(writer); - } - finally - { - Dispose(); - } - } - } - } - - private void Dispose() - { - IOUtils.Dispose(reader); - if (tempInput != null) - { - tempInput.Delete(); - } - if (tempSorted != null) - { - tempSorted.Delete(); - } - } - - /// <summary> - /// encodes an entry (bytes+weight) to the provided writer - /// </summary> - protected internal virtual void Encode(OfflineSorter.ByteSequencesWriter writer, - ByteArrayDataOutput output, byte[] buffer, BytesRef spare, long weight) - { - if (spare.Length + 8 >= buffer.Length) - { - buffer = ArrayUtil.Grow(buffer, spare.Length + 8); - } - output.Reset(buffer); - output.WriteBytes(spare.Bytes, spare.Offset, spare.Length); - output.WriteInt64(weight); - writer.Write(buffer, 0, output.Position); - } - - /// <summary> - /// decodes the weight at the current position </summary> - protected internal virtual long Decode(BytesRef scratch, ByteArrayDataInput tmpInput) - { - tmpInput.Reset(scratch.Bytes); - tmpInput.SkipBytes(scratch.Length - 8); // suggestion - scratch.Length -= 8; // long - return tmpInput.ReadInt64(); - } - } } \ No newline at end of file diff --git a/src/Lucene.Net.Suggest/Suggest/Tst/TSTLookup.cs b/src/Lucene.Net.Suggest/Suggest/Tst/TSTLookup.cs index 61f604f..ea40852 100644 --- a/src/Lucene.Net.Suggest/Suggest/Tst/TSTLookup.cs +++ b/src/Lucene.Net.Suggest/Suggest/Tst/TSTLookup.cs @@ -79,41 +79,6 @@ namespace Lucene.Net.Search.Suggest.Tst autocomplete.BalancedTree(tokens.ToArray(), vals.ToArray(), 0, tokens.Count - 1, root); } - [Obsolete("Use Build(IInputEnumerator) instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public override void Build(IInputIterator tfit) - { - if (tfit.HasPayloads) - { - throw new ArgumentException("this suggester doesn't support payloads"); - } - if (tfit.HasContexts) - { - throw new ArgumentException("this suggester doesn't support contexts"); - } - root = new TernaryTreeNode(); - // buffer first -#pragma warning disable 612, 618 - if (tfit.Comparer != BytesRef.UTF8SortedAsUTF16Comparer) - { - // make sure it's sorted and the comparer uses UTF16 sort order - tfit = new SortedInputIterator(tfit, BytesRef.UTF8SortedAsUTF16Comparer); - } -#pragma warning restore 612, 618 - - List<string> tokens = new List<string>(); - List<object> vals = new List<object>(); - BytesRef spare; - CharsRef charsSpare = new CharsRef(); - while ((spare = tfit.Next()) != null) - { - charsSpare.Grow(spare.Length); - UnicodeUtil.UTF8toUTF16(spare.Bytes, spare.Offset, spare.Length, charsSpare); - tokens.Add(charsSpare.ToString()); - vals.Add(tfit.Weight); - } - autocomplete.BalancedTree(tokens.ToArray(), vals.ToArray(), 0, tokens.Count - 1, root); - } - /// <summary> /// Adds a new node if <code>key</code> already exists, /// otherwise replaces its value. diff --git a/src/Lucene.Net.Suggest/Suggest/UnsortedInputIterator.cs b/src/Lucene.Net.Suggest/Suggest/UnsortedInputIterator.cs index 69527d2..6b1707f 100644 --- a/src/Lucene.Net.Suggest/Suggest/UnsortedInputIterator.cs +++ b/src/Lucene.Net.Suggest/Suggest/UnsortedInputIterator.cs @@ -104,86 +104,4 @@ namespace Lucene.Net.Search.Suggest } } } - - /// <summary> - /// This wrapper buffers the incoming elements and makes sure they are in - /// random order. - /// @lucene.experimental - /// </summary> - [Obsolete("Use UnsortedInputEnumerator instead. This method will be removed in 4.8.0 release candidate."), System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - public class UnsortedInputIterator : BufferedInputIterator - { - // TODO keep this for now - private readonly int[] ords; - private int currentOrd = -1; - private readonly BytesRef spare = new BytesRef(); - private readonly BytesRef payloadSpare = new BytesRef(); - - /// <summary> - /// Creates a new iterator, wrapping the specified iterator and - /// returning elements in a random order. - /// </summary> - public UnsortedInputIterator(IInputIterator source) - : base(source) - { - ords = new int[m_entries.Length]; - Random random = new Random(); - for (int i = 0; i < ords.Length; i++) - { - ords[i] = i; - } - for (int i = 0; i < ords.Length; i++) - { - int randomPosition = random.Next(ords.Length); - int temp = ords[i]; - ords[i] = ords[randomPosition]; - ords[randomPosition] = temp; - } - } - - public override long Weight - { - get - { - if (Debugging.AssertsEnabled) Debugging.Assert(currentOrd == ords[m_curPos]); - return m_freqs[currentOrd]; - } - } - - public override BytesRef Next() - { - if (++m_curPos < m_entries.Length) - { - currentOrd = ords[m_curPos]; - return m_entries.Get(spare, currentOrd); - } - return null; - } - - public override BytesRef Payload - { - get - { - if (HasPayloads && m_curPos < m_payloads.Length) - { - if (Debugging.AssertsEnabled) Debugging.Assert(currentOrd == ords[m_curPos]); - return m_payloads.Get(payloadSpare, currentOrd); - } - return null; - } - } - - public override ICollection<BytesRef> Contexts - { - get - { - if (HasContexts && m_curPos < m_contextSets.Count) - { - if (Debugging.AssertsEnabled) Debugging.Assert(currentOrd == ords[m_curPos]); - return m_contextSets[currentOrd]; - } - return null; - } - } - } } \ No newline at end of file
