This is an automated email from the ASF dual-hosted git repository. nightowl888 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/lucenenet.git
commit 6eba1e0648e3460cabc7032fdc976a1f44999465 Author: Shad Storhaug <[email protected]> AuthorDate: Mon Aug 24 15:10:10 2020 +0700 TO REVERT: Lucene.Net.ICU: Added locking to ICUTokenizer to only allow a single thread to manipulate the BreakIterator at a time. This can be reverted when the BreakIterator issue is fixed. --- .../Analysis/Icu/Segmentation/ICUTokenizer.cs | 29 ++++++++++++++-------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs b/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs index 2b37cde..7b31755 100644 --- a/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs +++ b/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs @@ -57,6 +57,8 @@ namespace Lucene.Net.Analysis.Icu.Segmentation private readonly ITypeAttribute typeAtt; private readonly IScriptAttribute scriptAtt; + private static readonly object syncLock = new object(); // LUCENENET specific - workaround until BreakIterator is made thread safe (LUCENENET TODO: TO REVERT) + /// <summary> /// Construct a new <see cref="ICUTokenizer"/> that breaks text into words from the given /// <see cref="TextReader"/>. @@ -109,23 +111,27 @@ namespace Lucene.Net.Analysis.Icu.Segmentation public override bool IncrementToken() { - ClearAttributes(); - if (length == 0) - Refill(); - while (!IncrementTokenBuffer()) + lock (syncLock) { - Refill(); - if (length <= 0) // no more bytes to read; - return false; + ClearAttributes(); + if (length == 0) + Refill(); + while (!IncrementTokenBuffer()) + { + Refill(); + if (length <= 0) // no more bytes to read; + return false; + } + return true; } - return true; } public override void Reset() { base.Reset(); - breaker.SetText(buffer, 0, 0); + lock (syncLock) + breaker.SetText(buffer, 0, 0); length = usableLength = offset = 0; } @@ -187,7 +193,8 @@ namespace Lucene.Net.Analysis.Icu.Segmentation */ } - breaker.SetText(buffer, 0, Math.Max(0, usableLength)); + lock (syncLock) + breaker.SetText(buffer, 0, Math.Max(0, usableLength)); } // TODO: refactor to a shared readFully somewhere @@ -236,7 +243,7 @@ namespace Lucene.Net.Analysis.Icu.Segmentation offsetAtt.SetOffset(CorrectOffset(offset + start), CorrectOffset(offset + end)); typeAtt.Type = config.GetType(breaker.ScriptCode, breaker.RuleStatus); scriptAtt.Code = breaker.ScriptCode; - + return true; } }
