Added test for compatibility with Lucene 4.8.0 index that has binary doc values.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/afb23de6 Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/afb23de6 Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/afb23de6 Branch: refs/heads/master Commit: afb23de6b6c46f0a9f0a58c86aa07a1c7eb3b6f2 Parents: 0cc3cfe Author: Shad Storhaug <[email protected]> Authored: Tue May 16 17:02:35 2017 +0700 Committer: Shad Storhaug <[email protected]> Committed: Tue May 16 19:17:11 2017 +0700 ---------------------------------------------------------------------- .../Index/TestBinaryDocValuesUpdates.cs | 89 +++++++++++++++++++ .../Index/index.48.w-binary-doc-values.zip | Bin 0 -> 39790 bytes .../Index/taxo.48.w-binary-doc-values.zip | Bin 0 -> 4869 bytes src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 2 + src/Lucene.Net.Tests/project.json | 2 + 5 files changed, 93 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs b/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs index 7f6a817..3b5e738 100644 --- a/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs +++ b/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs @@ -1,13 +1,19 @@ using Lucene.Net.Attributes; using Lucene.Net.Codecs; using Lucene.Net.Documents; +using Lucene.Net.Facet; +using Lucene.Net.Facet.Taxonomy; +using Lucene.Net.Facet.Taxonomy.Directory; using Lucene.Net.Randomized.Generators; +using Lucene.Net.Search; +using Lucene.Net.Store; using Lucene.Net.Support; using Lucene.Net.Support.Threading; using NUnit.Framework; using System; using System.Collections.Generic; using System.IO; +using System.Reflection; using System.Threading; namespace Lucene.Net.Index @@ -1776,5 +1782,88 @@ namespace Lucene.Net.Index dir.Dispose(); } + + /// <summary> + /// Using the exact translation from Lucene, we had an issue where loss of float precision was causing + /// index corruption when using binary doc values in combination with a 32 bit application to write the index. + /// Consequently, a 64 bit application could not read an index generated by a 32 bit application and a + /// 32 bit application could not read an index written by Java Lucene (regardless of bitness). + /// <para/> + /// This test is to verify that the current test environment (be it 32 or 64 bit) can read an index with + /// binary doc values that was written by Lucene in Java. + /// <para/> + /// To be thorough, we should setup the testing to run both in 32 bit and 64 bit on each platform. It would + /// be better if we could somehow generate the file under one bitness and consume it under the other, but + /// that would be difficult to set up. So, we are just reading a known good index and if we cannot read it + /// we know there is a problem. If the reading is fixed and the writing is different, it will fail several tests anyway. + /// </summary> + [Test] + [LuceneNetSpecific] + public virtual void TestReadIndexBitness() + { + string[] alphabet = new string[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" }; + int iterations = 500; + FacetsConfig facetConfig = GetFacetConfig(); + + // Unzip index + DirectoryInfo indexDir = CreateTempDir("index"); + using (Stream zipFileStream = this.GetType().GetTypeInfo().Assembly.FindAndGetManifestResourceStream(GetType(), "index.48.w-binary-doc-values.zip")) + { + TestUtil.Unzip(zipFileStream, indexDir); + } + + // Unzip taxonomy + DirectoryInfo taxoDir = CreateTempDir("taxo"); + using (Stream zipFileStream = this.GetType().GetTypeInfo().Assembly.FindAndGetManifestResourceStream(GetType(), "taxo.48.w-binary-doc-values.zip")) + { + TestUtil.Unzip(zipFileStream, taxoDir); + } + + + SimpleFSDirectory nFsd = new SimpleFSDirectory(indexDir); + IndexReader indexReader = DirectoryReader.Open(nFsd); + IndexSearcher searcher = new IndexSearcher(indexReader); + + DirectoryTaxonomyReader taxoReader = + new DirectoryTaxonomyReader(FSDirectory.Open(taxoDir)); + + try + { + foreach (string letter in alphabet) + { + FacetsCollector c = new FacetsCollector(); + searcher.Search(new TermQuery(new Term("Field1", letter)), c); + + Facets facets = new FastTaxonomyFacetCounts(taxoReader, facetConfig, c); + FacetResult result = facets.GetTopChildren(int.MaxValue, "facetField1"); + + assertEquals(iterations, System.Convert.ToInt32(result.LabelValues[0].Value)); + + FacetResult result2 = facets.GetTopChildren(int.MaxValue, "facetField2"); + + + for (int i = 0; i < iterations; i++) + { + assertEquals(i, System.Convert.ToInt32(result2.LabelValues[i].Label)); + } + } + + System.Console.WriteLine("Success"); + } + finally + { + IOUtils.Close(indexReader, taxoReader, nFsd); + } + } + + private FacetsConfig GetFacetConfig() + { + FacetsConfig facetConfig = new FacetsConfig(); + FacetsConfig config = new FacetsConfig(); + + config.SetRequireDimCount("facetField1", true); + config.SetRequireDimCount("facetField2", true); + return facetConfig; + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip b/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip new file mode 100644 index 0000000..c18e39e Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip differ http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip b/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip new file mode 100644 index 0000000..413cb54 Binary files /dev/null and b/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip differ http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj index bc1008b..a913b37 100644 --- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj +++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj @@ -113,6 +113,8 @@ <EmbeddedResource Include="Index\unsupported.24.nocfs.zip" /> <EmbeddedResource Include="Index\unsupported.29.cfs.zip" /> <EmbeddedResource Include="Index\unsupported.29.nocfs.zip" /> + <EmbeddedResource Include="Index\index.48.w-binary-doc-values.zip" /> + <EmbeddedResource Include="Index\taxo.48.w-binary-doc-values.zip" /> <None Include="Lucene.Net.snk" /> <None Include="Lucene.Net.Tests.project.json" /> </ItemGroup> http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/project.json ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/project.json b/src/Lucene.Net.Tests/project.json index c8b25ae..b1cf1a4 100644 --- a/src/Lucene.Net.Tests/project.json +++ b/src/Lucene.Net.Tests/project.json @@ -33,7 +33,9 @@ "Index/index.45.nocfs.zip", "Index/index.461.cfs.zip", "Index/index.461.nocfs.zip", + "Index/index.48.w-binary-doc-values.zip", "Index/moreterms.40.zip", + "Index/taxo.48.w-binary-doc-values.zip", "Index/unsupported.19.cfs.zip", "Index/unsupported.19.nocfs.zip", "Index/unsupported.20.cfs.zip",
