BUG: Lucene.Net.Core.Search (FieldCache + FieldComparer): fixed special case for "positive 0" and "negative 0" for float and double, and added unit tests to verify that float support works.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/eb1b5fcb Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/eb1b5fcb Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/eb1b5fcb Branch: refs/heads/api-work Commit: eb1b5fcbdfc004c4866f4ba1eec677f5c117ff81 Parents: 33b5db4 Author: Shad Storhaug <[email protected]> Authored: Fri Mar 10 17:01:27 2017 +0700 Committer: Shad Storhaug <[email protected]> Committed: Fri Mar 10 17:01:27 2017 +0700 ---------------------------------------------------------------------- src/Lucene.Net.Core/Lucene.Net.csproj | 1 + src/Lucene.Net.Core/Search/FieldCache.cs | 18 ++++++++-- src/Lucene.Net.Core/Search/FieldComparator.cs | 35 ++++++++++++++++++-- .../Support/SignedZeroComparer.cs | 31 +++++++++++++++++ src/Lucene.Net.Tests/Search/TestSort.cs | 31 +++++++++++++++++ .../Search/TestSortDocValues.cs | 33 ++++++++++++++++++ 6 files changed, 145 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucenenet/blob/eb1b5fcb/src/Lucene.Net.Core/Lucene.Net.csproj ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Lucene.Net.csproj b/src/Lucene.Net.Core/Lucene.Net.csproj index 40006b3..d094f47 100644 --- a/src/Lucene.Net.Core/Lucene.Net.csproj +++ b/src/Lucene.Net.Core/Lucene.Net.csproj @@ -666,6 +666,7 @@ <Compile Include="Support\ReentrantLock.cs" /> <Compile Include="Support\SafeTextWriterWrapper.cs" /> <Compile Include="Support\SetExtensions.cs" /> + <Compile Include="Support\SignedZeroComparer.cs" /> <Compile Include="Support\StreamUtils.cs" /> <Compile Include="Support\StringBuilderExtensions.cs" /> <Compile Include="Support\StringCharacterIterator.cs" /> http://git-wip-us.apache.org/repos/asf/lucenenet/blob/eb1b5fcb/src/Lucene.Net.Core/Search/FieldCache.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Search/FieldCache.cs b/src/Lucene.Net.Core/Search/FieldCache.cs index ebc5939..1ba36e8 100644 --- a/src/Lucene.Net.Core/Search/FieldCache.cs +++ b/src/Lucene.Net.Core/Search/FieldCache.cs @@ -653,7 +653,10 @@ namespace Lucene.Net.Search // LUCENENET: We parse to double first and then cast to float, which allows us to parse // double.MaxValue.ToString("R") (resulting in Infinity). This is how it worked in Java // and the TestFieldCache.TestInfoStream() test depends on this behavior to pass. - return (float)double.Parse(term.Utf8ToString(), NumberStyles.Float, CultureInfo.InvariantCulture); + + // We also need to use the same logic as DEFAULT_DOUBLE_PARSER to ensure we have signed zero + // support, so just call it directly rather than duplicating the logic here. + return (float)DEFAULT_DOUBLE_PARSER.ParseDouble(term); } public TermsEnum TermsEnum(Terms terms) @@ -707,7 +710,18 @@ namespace Lucene.Net.Search // UTF8 bytes... but really users should use // DoubleField, instead, which already decodes // directly from byte[] - return double.Parse(term.Utf8ToString(), NumberStyles.Float, CultureInfo.InvariantCulture); + string text = term.Utf8ToString(); + double value = double.Parse(text, NumberStyles.Float, CultureInfo.InvariantCulture); + + // LUCENENET specific special case - check whether a negative + // zero was passed in and, if so, convert the sign. Unfotunately, double.Parse() + // doesn't take care of this for us. + if (value == 0 && text.TrimStart().StartsWith("-", StringComparison.Ordinal)) + { + value = -0d; // Hard-coding the value in case double.Parse() works right someday (which would break if we did value * -1) + } + + return value; } public TermsEnum TermsEnum(Terms terms) http://git-wip-us.apache.org/repos/asf/lucenenet/blob/eb1b5fcb/src/Lucene.Net.Core/Search/FieldComparator.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Search/FieldComparator.cs b/src/Lucene.Net.Core/Search/FieldComparator.cs index 4c255aa..e2517a8 100644 --- a/src/Lucene.Net.Core/Search/FieldComparator.cs +++ b/src/Lucene.Net.Core/Search/FieldComparator.cs @@ -317,6 +317,11 @@ namespace Lucene.Net.Search /// <returns> value in this slot </returns> public abstract IComparable this[int slot] { get; } + + internal static readonly IComparer<double> SIGNED_ZERO_COMPARER = new SignedZeroComparer(); + + + /// <summary> /// Base FieldComparer class for numeric types /// </summary> @@ -461,7 +466,20 @@ namespace Lucene.Net.Search public override int Compare(int slot1, int slot2) { - return values[slot1].CompareTo(values[slot2]); + double v1 = values[slot1]; + double v2 = values[slot2]; + + int c = v1.CompareTo(v2); + + if (c == 0 && v1 == 0) + { + // LUCENENET specific special case: + // In case of zero, we may have a "positive 0" or "negative 0" + // to tie-break. + c = SIGNED_ZERO_COMPARER.Compare(v1, v2); + } + + return c; } public override int CompareBottom(int doc) @@ -549,7 +567,20 @@ namespace Lucene.Net.Search public override int Compare(int slot1, int slot2) { - return values[slot1].CompareTo(values[slot2]); + float v1 = values[slot1]; + float v2 = values[slot2]; + + int c = v1.CompareTo(v2); + + if (c == 0 && v1 == 0) + { + // LUCENENET specific special case: + // In case of zero, we may have a "positive 0" or "negative 0" + // to tie-break. + c = SIGNED_ZERO_COMPARER.Compare(v1, v2); + } + + return c; } public override int CompareBottom(int doc) http://git-wip-us.apache.org/repos/asf/lucenenet/blob/eb1b5fcb/src/Lucene.Net.Core/Support/SignedZeroComparer.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Support/SignedZeroComparer.cs b/src/Lucene.Net.Core/Support/SignedZeroComparer.cs new file mode 100644 index 0000000..2ebb9f6 --- /dev/null +++ b/src/Lucene.Net.Core/Support/SignedZeroComparer.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; + +namespace Lucene.Net.Support +{ + /// <summary> + /// LUCENENET specific comparer to handle the special case + /// of comparing negative zero with positive zero. + /// <para/> + /// For IEEE floating-point numbers, there is a distinction of negative and positive zero. + /// Reference: http://stackoverflow.com/a/3139636 + /// </summary> + public class SignedZeroComparer : IComparer<double> + { + public int Compare(double v1, double v2) + { + long a = BitConverter.DoubleToInt64Bits(v1); + long b = BitConverter.DoubleToInt64Bits(v2); + if (a > b) + { + return 1; + } + else if (a < b) + { + return -1; + } + + return 0; + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/eb1b5fcb/src/Lucene.Net.Tests/Search/TestSort.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Search/TestSort.cs b/src/Lucene.Net.Tests/Search/TestSort.cs index c3036b1..9c6d99b 100644 --- a/src/Lucene.Net.Tests/Search/TestSort.cs +++ b/src/Lucene.Net.Tests/Search/TestSort.cs @@ -1,3 +1,4 @@ +using Lucene.Net.Attributes; using Lucene.Net.Documents; using Lucene.Net.Randomized.Generators; using Lucene.Net.Support; @@ -1110,6 +1111,36 @@ namespace Lucene.Net.Search } /// <summary> + /// Tests sorting on type double with +/- zero </summary> + [Test, LuceneNetSpecific] + public virtual void TestFloatSignedZero() + { + Directory dir = NewDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, Similarity, TimeZone); + Document doc = new Document(); + doc.Add(NewStringField("value", "+0", Field.Store.YES)); + writer.AddDocument(doc); + doc = new Document(); + doc.Add(NewStringField("value", "-0", Field.Store.YES)); + writer.AddDocument(doc); + doc = new Document(); + IndexReader ir = writer.Reader; + writer.Dispose(); + + IndexSearcher searcher = NewSearcher(ir); + Sort sort = new Sort(new SortField("value", SortFieldType.SINGLE)); + + TopDocs td = searcher.Search(new MatchAllDocsQuery(), 10, sort); + Assert.AreEqual(2, td.TotalHits); + // numeric order + Assert.AreEqual("-0", searcher.Doc(td.ScoreDocs[0].Doc).Get("value")); + Assert.AreEqual("+0", searcher.Doc(td.ScoreDocs[1].Doc).Get("value")); + + ir.Dispose(); + dir.Dispose(); + } + + /// <summary> /// Tests sorting on type float with a missing value </summary> [Test] public virtual void TestFloatMissing() http://git-wip-us.apache.org/repos/asf/lucenenet/blob/eb1b5fcb/src/Lucene.Net.Tests/Search/TestSortDocValues.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Search/TestSortDocValues.cs b/src/Lucene.Net.Tests/Search/TestSortDocValues.cs index fc2489a..3a7a480 100644 --- a/src/Lucene.Net.Tests/Search/TestSortDocValues.cs +++ b/src/Lucene.Net.Tests/Search/TestSortDocValues.cs @@ -1,3 +1,4 @@ +using Lucene.Net.Attributes; using Lucene.Net.Documents; namespace Lucene.Net.Search @@ -731,6 +732,38 @@ namespace Lucene.Net.Search } /// <summary> + /// Tests sorting on type double with +/- zero </summary> + [Test, LuceneNetSpecific] + public virtual void TestFloatSignedZero() + { + Directory dir = NewDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, Similarity, TimeZone); + Document doc = new Document(); + doc.Add(new SingleDocValuesField("value", +0f)); + doc.Add(NewStringField("value", "+0", Field.Store.YES)); + writer.AddDocument(doc); + doc = new Document(); + doc.Add(new SingleDocValuesField("value", -0f)); + doc.Add(NewStringField("value", "-0", Field.Store.YES)); + writer.AddDocument(doc); + doc = new Document(); + IndexReader ir = writer.Reader; + writer.Dispose(); + + IndexSearcher searcher = NewSearcher(ir); + Sort sort = new Sort(new SortField("value", SortFieldType.SINGLE)); + + TopDocs td = searcher.Search(new MatchAllDocsQuery(), 10, sort); + Assert.AreEqual(2, td.TotalHits); + // numeric order + Assert.AreEqual("-0", searcher.Doc(td.ScoreDocs[0].Doc).Get("value")); + Assert.AreEqual("+0", searcher.Doc(td.ScoreDocs[1].Doc).Get("value")); + + ir.Dispose(); + dir.Dispose(); + } + + /// <summary> /// Tests sorting on type float in reverse </summary> [Test] public virtual void TestFloatReverse()
