http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestOmitPositions.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestOmitPositions.cs b/src/Lucene.Net.Tests/Index/TestOmitPositions.cs new file mode 100644 index 0000000..ff8ae7d --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestOmitPositions.cs @@ -0,0 +1,294 @@ +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using Lucene.Net.Randomized.Generators; + using NUnit.Framework; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using Analyzer = Lucene.Net.Analysis.Analyzer; + using BytesRef = Lucene.Net.Util.BytesRef; + using Directory = Lucene.Net.Store.Directory; + using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator; + using Document = Documents.Document; + using Field = Field; + using FieldType = FieldType; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using TestUtil = Lucene.Net.Util.TestUtil; + using TextField = TextField; + + /// + /// <summary> + /// @lucene.experimental + /// </summary> + [TestFixture] + public class TestOmitPositions : LuceneTestCase + { + [Test] + public virtual void TestBasic() + { + Directory dir = NewDirectory(); + RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Similarity, TimeZone); + Document doc = new Document(); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); + ft.IndexOptions = IndexOptions.DOCS_AND_FREQS; + Field f = NewField("foo", "this is a test test", ft); + doc.Add(f); + for (int i = 0; i < 100; i++) + { + w.AddDocument(doc); + } + + IndexReader reader = w.Reader; + w.Dispose(); + + Assert.IsNull(MultiFields.GetTermPositionsEnum(reader, null, "foo", new BytesRef("test"))); + + DocsEnum de = TestUtil.Docs(Random(), reader, "foo", new BytesRef("test"), null, null, DocsEnum.FLAG_FREQS); + while (de.NextDoc() != DocIdSetIterator.NO_MORE_DOCS) + { + Assert.AreEqual(2, de.Freq); + } + + reader.Dispose(); + dir.Dispose(); + } + + // Tests whether the DocumentWriter correctly enable the + // omitTermFreqAndPositions bit in the FieldInfo + [Test] + public virtual void TestPositions() + { + Directory ram = NewDirectory(); + Analyzer analyzer = new MockAnalyzer(Random()); + IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); + Document d = new Document(); + + // f1,f2,f3: docs only + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); + ft.IndexOptions = IndexOptions.DOCS_ONLY; + + Field f1 = NewField("f1", "this field has docs only", ft); + d.Add(f1); + + Field f2 = NewField("f2", "this field has docs only", ft); + d.Add(f2); + + Field f3 = NewField("f3", "this field has docs only", ft); + d.Add(f3); + + FieldType ft2 = new FieldType(TextField.TYPE_NOT_STORED); + ft2.IndexOptions = IndexOptions.DOCS_AND_FREQS; + + // f4,f5,f6 docs and freqs + Field f4 = NewField("f4", "this field has docs and freqs", ft2); + d.Add(f4); + + Field f5 = NewField("f5", "this field has docs and freqs", ft2); + d.Add(f5); + + Field f6 = NewField("f6", "this field has docs and freqs", ft2); + d.Add(f6); + + FieldType ft3 = new FieldType(TextField.TYPE_NOT_STORED); + ft3.IndexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + + // f7,f8,f9 docs/freqs/positions + Field f7 = NewField("f7", "this field has docs and freqs and positions", ft3); + d.Add(f7); + + Field f8 = NewField("f8", "this field has docs and freqs and positions", ft3); + d.Add(f8); + + Field f9 = NewField("f9", "this field has docs and freqs and positions", ft3); + d.Add(f9); + + writer.AddDocument(d); + writer.ForceMerge(1); + + // now we add another document which has docs-only for f1, f4, f7, docs/freqs for f2, f5, f8, + // and docs/freqs/positions for f3, f6, f9 + d = new Document(); + + // f1,f4,f7: docs only + f1 = NewField("f1", "this field has docs only", ft); + d.Add(f1); + + f4 = NewField("f4", "this field has docs only", ft); + d.Add(f4); + + f7 = NewField("f7", "this field has docs only", ft); + d.Add(f7); + + // f2, f5, f8: docs and freqs + f2 = NewField("f2", "this field has docs and freqs", ft2); + d.Add(f2); + + f5 = NewField("f5", "this field has docs and freqs", ft2); + d.Add(f5); + + f8 = NewField("f8", "this field has docs and freqs", ft2); + d.Add(f8); + + // f3, f6, f9: docs and freqs and positions + f3 = NewField("f3", "this field has docs and freqs and positions", ft3); + d.Add(f3); + + f6 = NewField("f6", "this field has docs and freqs and positions", ft3); + d.Add(f6); + + f9 = NewField("f9", "this field has docs and freqs and positions", ft3); + d.Add(f9); + + writer.AddDocument(d); + + // force merge + writer.ForceMerge(1); + // flush + writer.Dispose(); + + SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram)); + FieldInfos fi = reader.FieldInfos; + // docs + docs = docs + Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f1").IndexOptions); + // docs + docs/freqs = docs + Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f2").IndexOptions); + // docs + docs/freqs/pos = docs + Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f3").IndexOptions); + // docs/freqs + docs = docs + Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f4").IndexOptions); + // docs/freqs + docs/freqs = docs/freqs + Assert.AreEqual(IndexOptions.DOCS_AND_FREQS, fi.FieldInfo("f5").IndexOptions); + // docs/freqs + docs/freqs/pos = docs/freqs + Assert.AreEqual(IndexOptions.DOCS_AND_FREQS, fi.FieldInfo("f6").IndexOptions); + // docs/freqs/pos + docs = docs + Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f7").IndexOptions); + // docs/freqs/pos + docs/freqs = docs/freqs + Assert.AreEqual(IndexOptions.DOCS_AND_FREQS, fi.FieldInfo("f8").IndexOptions); + // docs/freqs/pos + docs/freqs/pos = docs/freqs/pos + Assert.AreEqual(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.FieldInfo("f9").IndexOptions); + + reader.Dispose(); + ram.Dispose(); + } + + private void AssertNoPrx(Directory dir) + { + string[] files = dir.ListAll(); + for (int i = 0; i < files.Length; i++) + { + Assert.IsFalse(files[i].EndsWith(".prx")); + Assert.IsFalse(files[i].EndsWith(".pos")); + } + } + + // Verifies no *.prx exists when all fields omit term positions: + [Test] + public virtual void TestNoPrxFile() + { + Directory ram = NewDirectory(); + Analyzer analyzer = new MockAnalyzer(Random()); + IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(3).SetMergePolicy(NewLogMergePolicy())); + LogMergePolicy lmp = (LogMergePolicy)writer.Config.MergePolicy; + lmp.MergeFactor = 2; + lmp.NoCFSRatio = 0.0; + Document d = new Document(); + + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); + ft.IndexOptions = IndexOptions.DOCS_AND_FREQS; + Field f1 = NewField("f1", "this field has term freqs", ft); + d.Add(f1); + + for (int i = 0; i < 30; i++) + { + writer.AddDocument(d); + } + + writer.Commit(); + + AssertNoPrx(ram); + + // now add some documents with positions, and check there is no prox after optimization + d = new Document(); + f1 = NewTextField("f1", "this field has positions", Field.Store.NO); + d.Add(f1); + + for (int i = 0; i < 30; i++) + { + writer.AddDocument(d); + } + + // force merge + writer.ForceMerge(1); + // flush + writer.Dispose(); + + AssertNoPrx(ram); + ram.Dispose(); + } + + /// <summary> + /// make sure we downgrade positions and payloads correctly </summary> + [Test] + public virtual void TestMixing() + { + // no positions + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); + ft.IndexOptions = IndexOptions.DOCS_AND_FREQS; + + Directory dir = NewDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, Similarity, TimeZone); + + for (int i = 0; i < 20; i++) + { + Document doc = new Document(); + if (i < 19 && Random().NextBoolean()) + { + for (int j = 0; j < 50; j++) + { + doc.Add(new TextField("foo", "i have positions", Field.Store.NO)); + } + } + else + { + for (int j = 0; j < 50; j++) + { + doc.Add(new Field("foo", "i have no positions", ft)); + } + } + iw.AddDocument(doc); + iw.Commit(); + } + + if (Random().NextBoolean()) + { + iw.ForceMerge(1); + } + + DirectoryReader ir = iw.Reader; + FieldInfos fis = MultiFields.GetMergedFieldInfos(ir); + Assert.AreEqual(IndexOptions.DOCS_AND_FREQS, fis.FieldInfo("foo").IndexOptions); + Assert.IsFalse(fis.FieldInfo("foo").HasPayloads); + iw.Dispose(); + ir.Dispose(); + dir.Dispose(); // checkindex + } + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestOmitTf.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestOmitTf.cs b/src/Lucene.Net.Tests/Index/TestOmitTf.cs new file mode 100644 index 0000000..3286d4b --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestOmitTf.cs @@ -0,0 +1,588 @@ +using System; +using System.Text; +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using NUnit.Framework; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using Analyzer = Lucene.Net.Analysis.Analyzer; + using BooleanQuery = Lucene.Net.Search.BooleanQuery; + using BytesRef = Lucene.Net.Util.BytesRef; + using CollectionStatistics = Lucene.Net.Search.CollectionStatistics; + using ICollector = Lucene.Net.Search.ICollector; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using Explanation = Lucene.Net.Search.Explanation; + using Field = Field; + using FieldType = FieldType; + using IndexSearcher = Lucene.Net.Search.IndexSearcher; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using Occur = Lucene.Net.Search.Occur; + using PhraseQuery = Lucene.Net.Search.PhraseQuery; + using Scorer = Lucene.Net.Search.Scorer; + using TermQuery = Lucene.Net.Search.TermQuery; + using TermStatistics = Lucene.Net.Search.TermStatistics; + using TextField = TextField; + using TFIDFSimilarity = Lucene.Net.Search.Similarities.TFIDFSimilarity; + + [TestFixture] + public class TestOmitTf : LuceneTestCase + { + public class SimpleSimilarity : TFIDFSimilarity + { + public override float DecodeNormValue(long norm) + { + return norm; + } + + public override long EncodeNormValue(float f) + { + return (long)f; + } + + public override float QueryNorm(float sumOfSquaredWeights) + { + return 1.0f; + } + + public override float Coord(int overlap, int maxOverlap) + { + return 1.0f; + } + + public override float LengthNorm(FieldInvertState state) + { + return state.Boost; + } + + public override float Tf(float freq) + { + return freq; + } + + public override float SloppyFreq(int distance) + { + return 2.0f; + } + + public override float Idf(long docFreq, long numDocs) + { + return 1.0f; + } + + public override Explanation IdfExplain(CollectionStatistics collectionStats, TermStatistics[] termStats) + { + return new Explanation(1.0f, "Inexplicable"); + } + + public override float ScorePayload(int doc, int start, int end, BytesRef payload) + { + return 1.0f; + } + } + + private static readonly FieldType OmitType = new FieldType(TextField.TYPE_NOT_STORED); + private static readonly FieldType NormalType = new FieldType(TextField.TYPE_NOT_STORED); + + static TestOmitTf() + { + OmitType.IndexOptions = IndexOptions.DOCS_ONLY; + } + + // Tests whether the DocumentWriter correctly enable the + // omitTermFreqAndPositions bit in the FieldInfo + [Test] + public virtual void TestOmitTermFreqAndPositions() + { + Directory ram = NewDirectory(); + Analyzer analyzer = new MockAnalyzer(Random()); + IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); + Document d = new Document(); + + // this field will have Tf + Field f1 = NewField("f1", "this field has term freqs", NormalType); + d.Add(f1); + + // this field will NOT have Tf + Field f2 = NewField("f2", "this field has NO Tf in all docs", OmitType); + d.Add(f2); + + writer.AddDocument(d); + writer.ForceMerge(1); + // now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger + // keep things constant + d = new Document(); + + // Reverse + f1 = NewField("f1", "this field has term freqs", OmitType); + d.Add(f1); + + f2 = NewField("f2", "this field has NO Tf in all docs", NormalType); + d.Add(f2); + + writer.AddDocument(d); + + // force merge + writer.ForceMerge(1); + // flush + writer.Dispose(); + + SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram)); + FieldInfos fi = reader.FieldInfos; + Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f1").IndexOptions, "OmitTermFreqAndPositions field bit should be set."); + Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f2").IndexOptions, "OmitTermFreqAndPositions field bit should be set."); + + reader.Dispose(); + ram.Dispose(); + } + + // Tests whether merging of docs that have different + // omitTermFreqAndPositions for the same field works + [Test] + public virtual void TestMixedMerge() + { + Directory ram = NewDirectory(); + Analyzer analyzer = new MockAnalyzer(Random()); + IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(3).SetMergePolicy(NewLogMergePolicy(2))); + Document d = new Document(); + + // this field will have Tf + Field f1 = NewField("f1", "this field has term freqs", NormalType); + d.Add(f1); + + // this field will NOT have Tf + Field f2 = NewField("f2", "this field has NO Tf in all docs", OmitType); + d.Add(f2); + + for (int i = 0; i < 30; i++) + { + writer.AddDocument(d); + } + + // now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger + // keep things constant + d = new Document(); + + // Reverese + f1 = NewField("f1", "this field has term freqs", OmitType); + d.Add(f1); + + f2 = NewField("f2", "this field has NO Tf in all docs", NormalType); + d.Add(f2); + + for (int i = 0; i < 30; i++) + { + writer.AddDocument(d); + } + + // force merge + writer.ForceMerge(1); + // flush + writer.Dispose(); + + SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram)); + FieldInfos fi = reader.FieldInfos; + Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f1").IndexOptions, "OmitTermFreqAndPositions field bit should be set."); + Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f2").IndexOptions, "OmitTermFreqAndPositions field bit should be set."); + + reader.Dispose(); + ram.Dispose(); + } + + // Make sure first adding docs that do not omitTermFreqAndPositions for + // field X, then adding docs that do omitTermFreqAndPositions for that same + // field, + [Test] + public virtual void TestMixedRAM() + { + Directory ram = NewDirectory(); + Analyzer analyzer = new MockAnalyzer(Random()); + IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(10).SetMergePolicy(NewLogMergePolicy(2))); + Document d = new Document(); + + // this field will have Tf + Field f1 = NewField("f1", "this field has term freqs", NormalType); + d.Add(f1); + + // this field will NOT have Tf + Field f2 = NewField("f2", "this field has NO Tf in all docs", OmitType); + d.Add(f2); + + for (int i = 0; i < 5; i++) + { + writer.AddDocument(d); + } + + for (int i = 0; i < 20; i++) + { + writer.AddDocument(d); + } + + // force merge + writer.ForceMerge(1); + + // flush + writer.Dispose(); + + SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram)); + FieldInfos fi = reader.FieldInfos; + Assert.AreEqual(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.FieldInfo("f1").IndexOptions, "OmitTermFreqAndPositions field bit should not be set."); + Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f2").IndexOptions, "OmitTermFreqAndPositions field bit should be set."); + + reader.Dispose(); + ram.Dispose(); + } + + private void AssertNoPrx(Directory dir) + { + string[] files = dir.ListAll(); + for (int i = 0; i < files.Length; i++) + { + Assert.IsFalse(files[i].EndsWith(".prx")); + Assert.IsFalse(files[i].EndsWith(".pos")); + } + } + + // Verifies no *.prx exists when all fields omit term freq: + [Test] + public virtual void TestNoPrxFile() + { + Directory ram = NewDirectory(); + Analyzer analyzer = new MockAnalyzer(Random()); + IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(3).SetMergePolicy(NewLogMergePolicy())); + LogMergePolicy lmp = (LogMergePolicy)writer.Config.MergePolicy; + lmp.MergeFactor = 2; + lmp.NoCFSRatio = 0.0; + Document d = new Document(); + + Field f1 = NewField("f1", "this field has term freqs", OmitType); + d.Add(f1); + + for (int i = 0; i < 30; i++) + { + writer.AddDocument(d); + } + + writer.Commit(); + + AssertNoPrx(ram); + + // now add some documents with positions, and check + // there is no prox after full merge + d = new Document(); + f1 = NewTextField("f1", "this field has positions", Field.Store.NO); + d.Add(f1); + + for (int i = 0; i < 30; i++) + { + writer.AddDocument(d); + } + + // force merge + writer.ForceMerge(1); + // flush + writer.Dispose(); + + AssertNoPrx(ram); + ram.Dispose(); + } + + // Test scores with one field with Term Freqs and one without, otherwise with equal content + [Test] + public virtual void TestBasic() + { + Directory dir = NewDirectory(); + Analyzer analyzer = new MockAnalyzer(Random()); + IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(2).SetSimilarity(new SimpleSimilarity()).SetMergePolicy(NewLogMergePolicy(2))); + + StringBuilder sb = new StringBuilder(265); + string term = "term"; + for (int i = 0; i < 30; i++) + { + Document doc = new Document(); + sb.Append(term).Append(" "); + string content = sb.ToString(); + Field noTf = NewField("noTf", content + (i % 2 == 0 ? "" : " notf"), OmitType); + doc.Add(noTf); + + Field tf = NewField("tf", content + (i % 2 == 0 ? " tf" : ""), NormalType); + doc.Add(tf); + + writer.AddDocument(doc); + //System.out.println(d); + } + + writer.ForceMerge(1); + // flush + writer.Dispose(); + + /* + * Verify the index + */ + IndexReader reader = DirectoryReader.Open(dir); + IndexSearcher searcher = NewSearcher(reader); + searcher.Similarity = new SimpleSimilarity(); + + Term a = new Term("noTf", term); + Term b = new Term("tf", term); + Term c = new Term("noTf", "notf"); + Term d = new Term("tf", "tf"); + TermQuery q1 = new TermQuery(a); + TermQuery q2 = new TermQuery(b); + TermQuery q3 = new TermQuery(c); + TermQuery q4 = new TermQuery(d); + + PhraseQuery pq = new PhraseQuery(); + pq.Add(a); + pq.Add(c); + try + { + searcher.Search(pq, 10); + Assert.Fail("did not hit expected exception"); + } + catch (Exception e) + { + Exception cause = e; + // If the searcher uses an executor service, the IAE is wrapped into other exceptions + while (cause.InnerException != null) + { + cause = cause.InnerException; + } + if (!(cause is InvalidOperationException)) + { + throw new InvalidOperationException("Expected an IAE", e); + } // else OK because positions are not indexed + } + + searcher.Search(q1, new CountingHitCollectorAnonymousInnerClassHelper(this)); + //System.out.println(CountingHitCollector.getCount()); + + searcher.Search(q2, new CountingHitCollectorAnonymousInnerClassHelper2(this)); + //System.out.println(CountingHitCollector.getCount()); + + searcher.Search(q3, new CountingHitCollectorAnonymousInnerClassHelper3(this)); + //System.out.println(CountingHitCollector.getCount()); + + searcher.Search(q4, new CountingHitCollectorAnonymousInnerClassHelper4(this)); + //System.out.println(CountingHitCollector.getCount()); + + BooleanQuery bq = new BooleanQuery(); + bq.Add(q1, Occur.MUST); + bq.Add(q4, Occur.MUST); + + searcher.Search(bq, new CountingHitCollectorAnonymousInnerClassHelper5(this)); + Assert.AreEqual(15, CountingHitCollector.Count); + + reader.Dispose(); + dir.Dispose(); + } + + private class CountingHitCollectorAnonymousInnerClassHelper : CountingHitCollector + { + private readonly TestOmitTf OuterInstance; + + public CountingHitCollectorAnonymousInnerClassHelper(TestOmitTf outerInstance) + { + this.OuterInstance = outerInstance; + } + + private Scorer scorer; + + public override sealed void SetScorer(Scorer scorer) + { + this.scorer = scorer; + } + + public override sealed void Collect(int doc) + { + //System.out.println("Q1: Doc=" + doc + " score=" + score); + float score = scorer.GetScore(); + Assert.IsTrue(score == 1.0f, "got score=" + score); + base.Collect(doc); + } + } + + private class CountingHitCollectorAnonymousInnerClassHelper2 : CountingHitCollector + { + private readonly TestOmitTf OuterInstance; + + public CountingHitCollectorAnonymousInnerClassHelper2(TestOmitTf outerInstance) + { + this.OuterInstance = outerInstance; + } + + private Scorer scorer; + + public override sealed void SetScorer(Scorer scorer) + { + this.scorer = scorer; + } + + public override sealed void Collect(int doc) + { + //System.out.println("Q2: Doc=" + doc + " score=" + score); + float score = scorer.GetScore(); + Assert.AreEqual(1.0f + doc, score, 0.00001f); + base.Collect(doc); + } + } + + private class CountingHitCollectorAnonymousInnerClassHelper3 : CountingHitCollector + { + private readonly TestOmitTf OuterInstance; + + public CountingHitCollectorAnonymousInnerClassHelper3(TestOmitTf outerInstance) + { + this.OuterInstance = outerInstance; + } + + private Scorer scorer; + + public override sealed void SetScorer(Scorer scorer) + { + this.scorer = scorer; + } + + public override sealed void Collect(int doc) + { + //System.out.println("Q1: Doc=" + doc + " score=" + score); + float score = scorer.GetScore(); + Assert.IsTrue(score == 1.0f); + Assert.IsFalse(doc % 2 == 0); + base.Collect(doc); + } + } + + private class CountingHitCollectorAnonymousInnerClassHelper4 : CountingHitCollector + { + private readonly TestOmitTf OuterInstance; + + public CountingHitCollectorAnonymousInnerClassHelper4(TestOmitTf outerInstance) + { + this.OuterInstance = outerInstance; + } + + private Scorer scorer; + + public override sealed void SetScorer(Scorer scorer) + { + this.scorer = scorer; + } + + public override sealed void Collect(int doc) + { + float score = scorer.GetScore(); + //System.out.println("Q1: Doc=" + doc + " score=" + score); + Assert.IsTrue(score == 1.0f); + Assert.IsTrue(doc % 2 == 0); + base.Collect(doc); + } + } + + private class CountingHitCollectorAnonymousInnerClassHelper5 : CountingHitCollector + { + private readonly TestOmitTf OuterInstance; + + public CountingHitCollectorAnonymousInnerClassHelper5(TestOmitTf outerInstance) + { + this.OuterInstance = outerInstance; + } + + public override sealed void Collect(int doc) + { + //System.out.println("BQ: Doc=" + doc + " score=" + score); + base.Collect(doc); + } + } + + public class CountingHitCollector : ICollector + { + internal static int Count_Renamed = 0; + internal static int Sum_Renamed = 0; + internal int DocBase = -1; + + internal CountingHitCollector() + { + Count_Renamed = 0; + Sum_Renamed = 0; + } + + public virtual void SetScorer(Scorer scorer) + { + } + + public virtual void Collect(int doc) + { + Count_Renamed++; + Sum_Renamed += doc + DocBase; // use it to avoid any possibility of being merged away + } + + public static int Count + { + get + { + return Count_Renamed; + } + } + + public static int Sum + { + get + { + return Sum_Renamed; + } + } + + public virtual void SetNextReader(AtomicReaderContext context) + { + DocBase = context.DocBase; + } + + public virtual bool AcceptsDocsOutOfOrder + { + get { return true; } + } + } + + /// <summary> + /// test that when freqs are omitted, that totalTermFreq and sumTotalTermFreq are -1 </summary> + [Test] + public virtual void TestStats() + { + Directory dir = NewDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + Document doc = new Document(); + FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); + ft.IndexOptions = IndexOptions.DOCS_ONLY; + ft.Freeze(); + Field f = NewField("foo", "bar", ft); + doc.Add(f); + iw.AddDocument(doc); + IndexReader ir = iw.Reader; + iw.Dispose(); + Assert.AreEqual(-1, ir.TotalTermFreq(new Term("foo", new BytesRef("bar")))); + Assert.AreEqual(-1, ir.GetSumTotalTermFreq("foo")); + ir.Dispose(); + dir.Dispose(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestParallelAtomicReader.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestParallelAtomicReader.cs b/src/Lucene.Net.Tests/Index/TestParallelAtomicReader.cs new file mode 100644 index 0000000..c6e896a --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestParallelAtomicReader.cs @@ -0,0 +1,357 @@ +using System; +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using Lucene.Net.Randomized.Generators; + using Lucene.Net.Search; + using NUnit.Framework; + using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using Field = Field; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using Occur = Lucene.Net.Search.Occur; + using TestUtil = Lucene.Net.Util.TestUtil; + + [TestFixture] + public class TestParallelAtomicReader : LuceneTestCase + { + private IndexSearcher Parallel_Renamed, Single_Renamed; + private Directory Dir, Dir1, Dir2; + + [Test] + public virtual void TestQueries() + { + Single_Renamed = Single(Random()); + Parallel_Renamed = Parallel(Random()); + + QueryTest(new TermQuery(new Term("f1", "v1"))); + QueryTest(new TermQuery(new Term("f1", "v2"))); + QueryTest(new TermQuery(new Term("f2", "v1"))); + QueryTest(new TermQuery(new Term("f2", "v2"))); + QueryTest(new TermQuery(new Term("f3", "v1"))); + QueryTest(new TermQuery(new Term("f3", "v2"))); + QueryTest(new TermQuery(new Term("f4", "v1"))); + QueryTest(new TermQuery(new Term("f4", "v2"))); + + BooleanQuery bq1 = new BooleanQuery(); + bq1.Add(new TermQuery(new Term("f1", "v1")), Occur.MUST); + bq1.Add(new TermQuery(new Term("f4", "v1")), Occur.MUST); + QueryTest(bq1); + + Single_Renamed.IndexReader.Dispose(); + Single_Renamed = null; + Parallel_Renamed.IndexReader.Dispose(); + Parallel_Renamed = null; + Dir.Dispose(); + Dir = null; + Dir1.Dispose(); + Dir1 = null; + Dir2.Dispose(); + Dir2 = null; + } + + [Test] + public virtual void TestFieldNames() + { + Directory dir1 = GetDir1(Random()); + Directory dir2 = GetDir2(Random()); + ParallelAtomicReader pr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1)), SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir2))); + FieldInfos fieldInfos = pr.FieldInfos; + Assert.AreEqual(4, fieldInfos.Count); + Assert.IsNotNull(fieldInfos.FieldInfo("f1")); + Assert.IsNotNull(fieldInfos.FieldInfo("f2")); + Assert.IsNotNull(fieldInfos.FieldInfo("f3")); + Assert.IsNotNull(fieldInfos.FieldInfo("f4")); + pr.Dispose(); + dir1.Dispose(); + dir2.Dispose(); + } + + [Test] + public virtual void TestRefCounts1() + { + Directory dir1 = GetDir1(Random()); + Directory dir2 = GetDir2(Random()); + AtomicReader ir1, ir2; + // close subreaders, ParallelReader will not change refCounts, but close on its own close + ParallelAtomicReader pr = new ParallelAtomicReader(ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1)), ir2 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir2))); + + // check RefCounts + Assert.AreEqual(1, ir1.RefCount); + Assert.AreEqual(1, ir2.RefCount); + pr.Dispose(); + Assert.AreEqual(0, ir1.RefCount); + Assert.AreEqual(0, ir2.RefCount); + dir1.Dispose(); + dir2.Dispose(); + } + + [Test] + public virtual void TestRefCounts2() + { + Directory dir1 = GetDir1(Random()); + Directory dir2 = GetDir2(Random()); + AtomicReader ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1)); + AtomicReader ir2 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir2)); + // don't close subreaders, so ParallelReader will increment refcounts + ParallelAtomicReader pr = new ParallelAtomicReader(false, ir1, ir2); + // check RefCounts + Assert.AreEqual(2, ir1.RefCount); + Assert.AreEqual(2, ir2.RefCount); + pr.Dispose(); + Assert.AreEqual(1, ir1.RefCount); + Assert.AreEqual(1, ir2.RefCount); + ir1.Dispose(); + ir2.Dispose(); + Assert.AreEqual(0, ir1.RefCount); + Assert.AreEqual(0, ir2.RefCount); + dir1.Dispose(); + dir2.Dispose(); + } + + [Test] + public virtual void TestCloseInnerReader() + { + Directory dir1 = GetDir1(Random()); + AtomicReader ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1)); + + // with overlapping + ParallelAtomicReader pr = new ParallelAtomicReader(true, new AtomicReader[] { ir1 }, new AtomicReader[] { ir1 }); + + ir1.Dispose(); + + try + { + pr.Document(0); + Assert.Fail("ParallelAtomicReader should be already closed because inner reader was closed!"); + } +#pragma warning disable 168 + catch (AlreadyClosedException e) +#pragma warning restore 168 + { + // pass + } + + // noop: + pr.Dispose(); + dir1.Dispose(); + } + + [Test] + public virtual void TestIncompatibleIndexes() + { + // two documents: + Directory dir1 = GetDir1(Random()); + + // one document only: + Directory dir2 = NewDirectory(); + IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + Document d3 = new Document(); + + d3.Add(NewTextField("f3", "v1", Field.Store.YES)); + w2.AddDocument(d3); + w2.Dispose(); + + AtomicReader ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1)); + AtomicReader ir2 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir2)); + + try + { + new ParallelAtomicReader(ir1, ir2); + Assert.Fail("didn't get exptected exception: indexes don't have same number of documents"); + } +#pragma warning disable 168 + catch (System.ArgumentException e) +#pragma warning restore 168 + { + // expected exception + } + + try + { + new ParallelAtomicReader(Random().NextBoolean(), new AtomicReader[] { ir1, ir2 }, new AtomicReader[] { ir1, ir2 }); + Assert.Fail("didn't get expected exception: indexes don't have same number of documents"); + } +#pragma warning disable 168 + catch (System.ArgumentException e) +#pragma warning restore 168 + { + // expected exception + } + // check RefCounts + Assert.AreEqual(1, ir1.RefCount); + Assert.AreEqual(1, ir2.RefCount); + ir1.Dispose(); + ir2.Dispose(); + dir1.Dispose(); + dir2.Dispose(); + } + + [Test] + public virtual void TestIgnoreStoredFields() + { + Directory dir1 = GetDir1(Random()); + Directory dir2 = GetDir2(Random()); + AtomicReader ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1)); + AtomicReader ir2 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir2)); + + // with overlapping + ParallelAtomicReader pr = new ParallelAtomicReader(false, new AtomicReader[] { ir1, ir2 }, new AtomicReader[] { ir1 }); + Assert.AreEqual("v1", pr.Document(0).Get("f1")); + Assert.AreEqual("v1", pr.Document(0).Get("f2")); + Assert.IsNull(pr.Document(0).Get("f3")); + Assert.IsNull(pr.Document(0).Get("f4")); + // check that fields are there + Assert.IsNotNull(pr.Terms("f1")); + Assert.IsNotNull(pr.Terms("f2")); + Assert.IsNotNull(pr.Terms("f3")); + Assert.IsNotNull(pr.Terms("f4")); + pr.Dispose(); + + // no stored fields at all + pr = new ParallelAtomicReader(false, new AtomicReader[] { ir2 }, new AtomicReader[0]); + Assert.IsNull(pr.Document(0).Get("f1")); + Assert.IsNull(pr.Document(0).Get("f2")); + Assert.IsNull(pr.Document(0).Get("f3")); + Assert.IsNull(pr.Document(0).Get("f4")); + // check that fields are there + Assert.IsNull(pr.Terms("f1")); + Assert.IsNull(pr.Terms("f2")); + Assert.IsNotNull(pr.Terms("f3")); + Assert.IsNotNull(pr.Terms("f4")); + pr.Dispose(); + + // without overlapping + pr = new ParallelAtomicReader(true, new AtomicReader[] { ir2 }, new AtomicReader[] { ir1 }); + Assert.AreEqual("v1", pr.Document(0).Get("f1")); + Assert.AreEqual("v1", pr.Document(0).Get("f2")); + Assert.IsNull(pr.Document(0).Get("f3")); + Assert.IsNull(pr.Document(0).Get("f4")); + // check that fields are there + Assert.IsNull(pr.Terms("f1")); + Assert.IsNull(pr.Terms("f2")); + Assert.IsNotNull(pr.Terms("f3")); + Assert.IsNotNull(pr.Terms("f4")); + pr.Dispose(); + + // no main readers + try + { + new ParallelAtomicReader(true, new AtomicReader[0], new AtomicReader[] { ir1 }); + Assert.Fail("didn't get expected exception: need a non-empty main-reader array"); + } +#pragma warning disable 168 + catch (System.ArgumentException iae) +#pragma warning restore 168 + { + // pass + } + + dir1.Dispose(); + dir2.Dispose(); + } + + private void QueryTest(Query query) + { + ScoreDoc[] parallelHits = Parallel_Renamed.Search(query, null, 1000).ScoreDocs; + ScoreDoc[] singleHits = Single_Renamed.Search(query, null, 1000).ScoreDocs; + Assert.AreEqual(parallelHits.Length, singleHits.Length); + for (int i = 0; i < parallelHits.Length; i++) + { + Assert.AreEqual(parallelHits[i].Score, singleHits[i].Score, 0.001f); + Document docParallel = Parallel_Renamed.Doc(parallelHits[i].Doc); + Document docSingle = Single_Renamed.Doc(singleHits[i].Doc); + Assert.AreEqual(docParallel.Get("f1"), docSingle.Get("f1")); + Assert.AreEqual(docParallel.Get("f2"), docSingle.Get("f2")); + Assert.AreEqual(docParallel.Get("f3"), docSingle.Get("f3")); + Assert.AreEqual(docParallel.Get("f4"), docSingle.Get("f4")); + } + } + + // Fields 1-4 indexed together: + private IndexSearcher Single(Random random) + { + Dir = NewDirectory(); + IndexWriter w = new IndexWriter(Dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + Document d1 = new Document(); + d1.Add(NewTextField("f1", "v1", Field.Store.YES)); + d1.Add(NewTextField("f2", "v1", Field.Store.YES)); + d1.Add(NewTextField("f3", "v1", Field.Store.YES)); + d1.Add(NewTextField("f4", "v1", Field.Store.YES)); + w.AddDocument(d1); + Document d2 = new Document(); + d2.Add(NewTextField("f1", "v2", Field.Store.YES)); + d2.Add(NewTextField("f2", "v2", Field.Store.YES)); + d2.Add(NewTextField("f3", "v2", Field.Store.YES)); + d2.Add(NewTextField("f4", "v2", Field.Store.YES)); + w.AddDocument(d2); + w.Dispose(); + + DirectoryReader ir = DirectoryReader.Open(Dir); + return NewSearcher(ir); + } + + // Fields 1 & 2 in one index, 3 & 4 in other, with ParallelReader: + private IndexSearcher Parallel(Random random) + { + Dir1 = GetDir1(random); + Dir2 = GetDir2(random); + ParallelAtomicReader pr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(Dir1)), SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(Dir2))); + TestUtil.CheckReader(pr); + return NewSearcher(pr); + } + + private Directory GetDir1(Random random) + { + Directory dir1 = NewDirectory(); + IndexWriter w1 = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + Document d1 = new Document(); + d1.Add(NewTextField("f1", "v1", Field.Store.YES)); + d1.Add(NewTextField("f2", "v1", Field.Store.YES)); + w1.AddDocument(d1); + Document d2 = new Document(); + d2.Add(NewTextField("f1", "v2", Field.Store.YES)); + d2.Add(NewTextField("f2", "v2", Field.Store.YES)); + w1.AddDocument(d2); + w1.Dispose(); + return dir1; + } + + private Directory GetDir2(Random random) + { + Directory dir2 = NewDirectory(); + IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + Document d3 = new Document(); + d3.Add(NewTextField("f3", "v1", Field.Store.YES)); + d3.Add(NewTextField("f4", "v1", Field.Store.YES)); + w2.AddDocument(d3); + Document d4 = new Document(); + d4.Add(NewTextField("f3", "v2", Field.Store.YES)); + d4.Add(NewTextField("f4", "v2", Field.Store.YES)); + w2.AddDocument(d4); + w2.Dispose(); + return dir2; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestParallelCompositeReader.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestParallelCompositeReader.cs b/src/Lucene.Net.Tests/Index/TestParallelCompositeReader.cs new file mode 100644 index 0000000..8b7da0e --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestParallelCompositeReader.cs @@ -0,0 +1,666 @@ +using System; +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using Lucene.Net.Randomized.Generators; + using Lucene.Net.Search; + using NUnit.Framework; + using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using Field = Field; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using Occur = Lucene.Net.Search.Occur; + using IReaderClosedListener = Lucene.Net.Index.IndexReader.IReaderClosedListener; + + [TestFixture] + public class TestParallelCompositeReader : LuceneTestCase + { + private IndexSearcher Parallel_Renamed, Single_Renamed; + private Directory Dir, Dir1, Dir2; + + [Test] + public virtual void TestQueries() + { + Single_Renamed = Single(Random(), false); + Parallel_Renamed = Parallel(Random(), false); + + Queries(); + + Single_Renamed.IndexReader.Dispose(); + Single_Renamed = null; + Parallel_Renamed.IndexReader.Dispose(); + Parallel_Renamed = null; + Dir.Dispose(); + Dir = null; + Dir1.Dispose(); + Dir1 = null; + Dir2.Dispose(); + Dir2 = null; + } + + [Test] + public virtual void TestQueriesCompositeComposite() + { + Single_Renamed = Single(Random(), true); + Parallel_Renamed = Parallel(Random(), true); + + Queries(); + + Single_Renamed.IndexReader.Dispose(); + Single_Renamed = null; + Parallel_Renamed.IndexReader.Dispose(); + Parallel_Renamed = null; + Dir.Dispose(); + Dir = null; + Dir1.Dispose(); + Dir1 = null; + Dir2.Dispose(); + Dir2 = null; + } + + private void Queries() + { + QueryTest(new TermQuery(new Term("f1", "v1"))); + QueryTest(new TermQuery(new Term("f1", "v2"))); + QueryTest(new TermQuery(new Term("f2", "v1"))); + QueryTest(new TermQuery(new Term("f2", "v2"))); + QueryTest(new TermQuery(new Term("f3", "v1"))); + QueryTest(new TermQuery(new Term("f3", "v2"))); + QueryTest(new TermQuery(new Term("f4", "v1"))); + QueryTest(new TermQuery(new Term("f4", "v2"))); + + BooleanQuery bq1 = new BooleanQuery(); + bq1.Add(new TermQuery(new Term("f1", "v1")), Occur.MUST); + bq1.Add(new TermQuery(new Term("f4", "v1")), Occur.MUST); + QueryTest(bq1); + } + + [Test] + public virtual void TestRefCounts1() + { + Directory dir1 = GetDir1(Random()); + Directory dir2 = GetDir2(Random()); + DirectoryReader ir1, ir2; + // close subreaders, ParallelReader will not change refCounts, but close on its own close + ParallelCompositeReader pr = new ParallelCompositeReader(ir1 = DirectoryReader.Open(dir1), ir2 = DirectoryReader.Open(dir2)); + IndexReader psub1 = pr.GetSequentialSubReaders()[0]; + // check RefCounts + Assert.AreEqual(1, ir1.RefCount); + Assert.AreEqual(1, ir2.RefCount); + Assert.AreEqual(1, psub1.RefCount); + pr.Dispose(); + Assert.AreEqual(0, ir1.RefCount); + Assert.AreEqual(0, ir2.RefCount); + Assert.AreEqual(0, psub1.RefCount); + dir1.Dispose(); + dir2.Dispose(); + } + + [Test] + public virtual void TestRefCounts2() + { + Directory dir1 = GetDir1(Random()); + Directory dir2 = GetDir2(Random()); + DirectoryReader ir1 = DirectoryReader.Open(dir1); + DirectoryReader ir2 = DirectoryReader.Open(dir2); + + // don't close subreaders, so ParallelReader will increment refcounts + ParallelCompositeReader pr = new ParallelCompositeReader(false, ir1, ir2); + IndexReader psub1 = pr.GetSequentialSubReaders()[0]; + // check RefCounts + Assert.AreEqual(2, ir1.RefCount); + Assert.AreEqual(2, ir2.RefCount); + Assert.AreEqual(1, psub1.RefCount, "refCount must be 1, as the synthetic reader was created by ParallelCompositeReader"); + pr.Dispose(); + Assert.AreEqual(1, ir1.RefCount); + Assert.AreEqual(1, ir2.RefCount); + Assert.AreEqual(0, psub1.RefCount, "refcount must be 0 because parent was closed"); + ir1.Dispose(); + ir2.Dispose(); + Assert.AreEqual(0, ir1.RefCount); + Assert.AreEqual(0, ir2.RefCount); + Assert.AreEqual(0, psub1.RefCount, "refcount should not change anymore"); + dir1.Dispose(); + dir2.Dispose(); + } + + // closeSubreaders=false + [Test] + public virtual void TestReaderClosedListener1() + { + Directory dir1 = GetDir1(Random()); + CompositeReader ir1 = DirectoryReader.Open(dir1); + + // with overlapping + ParallelCompositeReader pr = new ParallelCompositeReader(false, new CompositeReader[] { ir1 }, new CompositeReader[] { ir1 }); + + int[] listenerClosedCount = new int[1]; + + Assert.AreEqual(3, pr.Leaves.Count); + + foreach (AtomicReaderContext cxt in pr.Leaves) + { + cxt.Reader.AddReaderClosedListener(new ReaderClosedListenerAnonymousInnerClassHelper(this, listenerClosedCount)); + } + pr.Dispose(); + ir1.Dispose(); + Assert.AreEqual(3, listenerClosedCount[0]); + dir1.Dispose(); + } + + private class ReaderClosedListenerAnonymousInnerClassHelper : IReaderClosedListener + { + private readonly TestParallelCompositeReader OuterInstance; + + private int[] ListenerClosedCount; + + public ReaderClosedListenerAnonymousInnerClassHelper(TestParallelCompositeReader outerInstance, int[] listenerClosedCount) + { + this.OuterInstance = outerInstance; + this.ListenerClosedCount = listenerClosedCount; + } + + public void OnClose(IndexReader reader) + { + ListenerClosedCount[0]++; + } + } + + // closeSubreaders=true + [Test] + public virtual void TestReaderClosedListener2() + { + Directory dir1 = GetDir1(Random()); + CompositeReader ir1 = DirectoryReader.Open(dir1); + + // with overlapping + ParallelCompositeReader pr = new ParallelCompositeReader(true, new CompositeReader[] { ir1 }, new CompositeReader[] { ir1 }); + + int[] listenerClosedCount = new int[1]; + + Assert.AreEqual(3, pr.Leaves.Count); + + foreach (AtomicReaderContext cxt in pr.Leaves) + { + cxt.Reader.AddReaderClosedListener(new ReaderClosedListenerAnonymousInnerClassHelper2(this, listenerClosedCount)); + } + pr.Dispose(); + Assert.AreEqual(3, listenerClosedCount[0]); + dir1.Dispose(); + } + + private class ReaderClosedListenerAnonymousInnerClassHelper2 : IReaderClosedListener + { + private readonly TestParallelCompositeReader OuterInstance; + + private int[] ListenerClosedCount; + + public ReaderClosedListenerAnonymousInnerClassHelper2(TestParallelCompositeReader outerInstance, int[] listenerClosedCount) + { + this.OuterInstance = outerInstance; + this.ListenerClosedCount = listenerClosedCount; + } + + public void OnClose(IndexReader reader) + { + ListenerClosedCount[0]++; + } + } + + [Test] + public virtual void TestCloseInnerReader() + { + Directory dir1 = GetDir1(Random()); + CompositeReader ir1 = DirectoryReader.Open(dir1); + Assert.AreEqual(1, ir1.GetSequentialSubReaders()[0].RefCount); + + // with overlapping + ParallelCompositeReader pr = new ParallelCompositeReader(true, new CompositeReader[] { ir1 }, new CompositeReader[] { ir1 }); + + IndexReader psub = pr.GetSequentialSubReaders()[0]; + Assert.AreEqual(1, psub.RefCount); + + ir1.Dispose(); + + Assert.AreEqual(1, psub.RefCount, "refCount of synthetic subreader should be unchanged"); + try + { + psub.Document(0); + Assert.Fail("Subreader should be already closed because inner reader was closed!"); + } +#pragma warning disable 168 + catch (AlreadyClosedException e) +#pragma warning restore 168 + { + // pass + } + + try + { + pr.Document(0); + Assert.Fail("ParallelCompositeReader should be already closed because inner reader was closed!"); + } +#pragma warning disable 168 + catch (AlreadyClosedException e) +#pragma warning restore 168 + { + // pass + } + + // noop: + pr.Dispose(); + Assert.AreEqual(0, psub.RefCount); + dir1.Dispose(); + } + + [Test] + public virtual void TestIncompatibleIndexes1() + { + // two documents: + Directory dir1 = GetDir1(Random()); + + // one document only: + Directory dir2 = NewDirectory(); + IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + Document d3 = new Document(); + + d3.Add(NewTextField("f3", "v1", Field.Store.YES)); + w2.AddDocument(d3); + w2.Dispose(); + + DirectoryReader ir1 = DirectoryReader.Open(dir1), ir2 = DirectoryReader.Open(dir2); + try + { + new ParallelCompositeReader(ir1, ir2); + Assert.Fail("didn't get expected exception: indexes don't have same number of documents"); + } +#pragma warning disable 168 + catch (System.ArgumentException e) +#pragma warning restore 168 + { + // expected exception + } + try + { + new ParallelCompositeReader(Random().NextBoolean(), ir1, ir2); + Assert.Fail("didn't get expected exception: indexes don't have same number of documents"); + } +#pragma warning disable 168 + catch (System.ArgumentException e) +#pragma warning restore 168 + { + // expected exception + } + Assert.AreEqual(1, ir1.RefCount); + Assert.AreEqual(1, ir2.RefCount); + ir1.Dispose(); + ir2.Dispose(); + Assert.AreEqual(0, ir1.RefCount); + Assert.AreEqual(0, ir2.RefCount); + dir1.Dispose(); + dir2.Dispose(); + } + + [Test] + public virtual void TestIncompatibleIndexes2() + { + Directory dir1 = GetDir1(Random()); + Directory dir2 = GetInvalidStructuredDir2(Random()); + + DirectoryReader ir1 = DirectoryReader.Open(dir1), ir2 = DirectoryReader.Open(dir2); + CompositeReader[] readers = new CompositeReader[] { ir1, ir2 }; + try + { + new ParallelCompositeReader(readers); + Assert.Fail("didn't get expected exception: indexes don't have same subreader structure"); + } +#pragma warning disable 168 + catch (System.ArgumentException e) +#pragma warning restore 168 + { + // expected exception + } + try + { + new ParallelCompositeReader(Random().NextBoolean(), readers, readers); + Assert.Fail("didn't get expected exception: indexes don't have same subreader structure"); + } +#pragma warning disable 168 + catch (System.ArgumentException e) +#pragma warning restore 168 + { + // expected exception + } + Assert.AreEqual(1, ir1.RefCount); + Assert.AreEqual(1, ir2.RefCount); + ir1.Dispose(); + ir2.Dispose(); + Assert.AreEqual(0, ir1.RefCount); + Assert.AreEqual(0, ir2.RefCount); + dir1.Dispose(); + dir2.Dispose(); + } + + [Test] + public virtual void TestIncompatibleIndexes3() + { + Directory dir1 = GetDir1(Random()); + Directory dir2 = GetDir2(Random()); + + CompositeReader ir1 = new MultiReader(DirectoryReader.Open(dir1), SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1))), ir2 = new MultiReader(DirectoryReader.Open(dir2), DirectoryReader.Open(dir2)); + CompositeReader[] readers = new CompositeReader[] { ir1, ir2 }; + try + { + new ParallelCompositeReader(readers); + Assert.Fail("didn't get expected exception: indexes don't have same subreader structure"); + } +#pragma warning disable 168 + catch (System.ArgumentException e) +#pragma warning restore 168 + { + // expected exception + } + try + { + new ParallelCompositeReader(Random().NextBoolean(), readers, readers); + Assert.Fail("didn't get expected exception: indexes don't have same subreader structure"); + } +#pragma warning disable 168 + catch (System.ArgumentException e) +#pragma warning restore 168 + { + // expected exception + } + Assert.AreEqual(1, ir1.RefCount); + Assert.AreEqual(1, ir2.RefCount); + ir1.Dispose(); + ir2.Dispose(); + Assert.AreEqual(0, ir1.RefCount); + Assert.AreEqual(0, ir2.RefCount); + dir1.Dispose(); + dir2.Dispose(); + } + + [Test] + public virtual void TestIgnoreStoredFields() + { + Directory dir1 = GetDir1(Random()); + Directory dir2 = GetDir2(Random()); + CompositeReader ir1 = DirectoryReader.Open(dir1); + CompositeReader ir2 = DirectoryReader.Open(dir2); + + // with overlapping + ParallelCompositeReader pr = new ParallelCompositeReader(false, new CompositeReader[] { ir1, ir2 }, new CompositeReader[] { ir1 }); + Assert.AreEqual("v1", pr.Document(0).Get("f1")); + Assert.AreEqual("v1", pr.Document(0).Get("f2")); + Assert.IsNull(pr.Document(0).Get("f3")); + Assert.IsNull(pr.Document(0).Get("f4")); + // check that fields are there + AtomicReader slow = SlowCompositeReaderWrapper.Wrap(pr); + Assert.IsNotNull(slow.Terms("f1")); + Assert.IsNotNull(slow.Terms("f2")); + Assert.IsNotNull(slow.Terms("f3")); + Assert.IsNotNull(slow.Terms("f4")); + pr.Dispose(); + + // no stored fields at all + pr = new ParallelCompositeReader(false, new CompositeReader[] { ir2 }, new CompositeReader[0]); + Assert.IsNull(pr.Document(0).Get("f1")); + Assert.IsNull(pr.Document(0).Get("f2")); + Assert.IsNull(pr.Document(0).Get("f3")); + Assert.IsNull(pr.Document(0).Get("f4")); + // check that fields are there + slow = SlowCompositeReaderWrapper.Wrap(pr); + Assert.IsNull(slow.Terms("f1")); + Assert.IsNull(slow.Terms("f2")); + Assert.IsNotNull(slow.Terms("f3")); + Assert.IsNotNull(slow.Terms("f4")); + pr.Dispose(); + + // without overlapping + pr = new ParallelCompositeReader(true, new CompositeReader[] { ir2 }, new CompositeReader[] { ir1 }); + Assert.AreEqual("v1", pr.Document(0).Get("f1")); + Assert.AreEqual("v1", pr.Document(0).Get("f2")); + Assert.IsNull(pr.Document(0).Get("f3")); + Assert.IsNull(pr.Document(0).Get("f4")); + // check that fields are there + slow = SlowCompositeReaderWrapper.Wrap(pr); + Assert.IsNull(slow.Terms("f1")); + Assert.IsNull(slow.Terms("f2")); + Assert.IsNotNull(slow.Terms("f3")); + Assert.IsNotNull(slow.Terms("f4")); + pr.Dispose(); + + // no main readers + try + { + new ParallelCompositeReader(true, new CompositeReader[0], new CompositeReader[] { ir1 }); + Assert.Fail("didn't get expected exception: need a non-empty main-reader array"); + } +#pragma warning disable 168 + catch (System.ArgumentException iae) +#pragma warning restore 168 + { + // pass + } + + dir1.Dispose(); + dir2.Dispose(); + } + + [Test] + public virtual void TestToString() + { + Directory dir1 = GetDir1(Random()); + CompositeReader ir1 = DirectoryReader.Open(dir1); + ParallelCompositeReader pr = new ParallelCompositeReader(new CompositeReader[] { ir1 }); + + string s = pr.ToString(); + Assert.IsTrue(s.StartsWith("ParallelCompositeReader(ParallelAtomicReader("), "toString incorrect: " + s); + + pr.Dispose(); + dir1.Dispose(); + } + + [Test] + public virtual void TestToStringCompositeComposite() + { + Directory dir1 = GetDir1(Random()); + CompositeReader ir1 = DirectoryReader.Open(dir1); + ParallelCompositeReader pr = new ParallelCompositeReader(new CompositeReader[] { new MultiReader(ir1) }); + + string s = pr.ToString(); + + Assert.IsTrue(s.StartsWith("ParallelCompositeReader(ParallelCompositeReaderAnonymousInnerClassHelper(ParallelAtomicReader("), "toString incorrect: " + s); + + pr.Dispose(); + dir1.Dispose(); + } + + private void QueryTest(Query query) + { + ScoreDoc[] parallelHits = Parallel_Renamed.Search(query, null, 1000).ScoreDocs; + ScoreDoc[] singleHits = Single_Renamed.Search(query, null, 1000).ScoreDocs; + Assert.AreEqual(parallelHits.Length, singleHits.Length); + for (int i = 0; i < parallelHits.Length; i++) + { + Assert.AreEqual(parallelHits[i].Score, singleHits[i].Score, 0.001f); + Document docParallel = Parallel_Renamed.Doc(parallelHits[i].Doc); + Document docSingle = Single_Renamed.Doc(singleHits[i].Doc); + Assert.AreEqual(docParallel.Get("f1"), docSingle.Get("f1")); + Assert.AreEqual(docParallel.Get("f2"), docSingle.Get("f2")); + Assert.AreEqual(docParallel.Get("f3"), docSingle.Get("f3")); + Assert.AreEqual(docParallel.Get("f4"), docSingle.Get("f4")); + } + } + + // Fields 1-4 indexed together: + private IndexSearcher Single(Random random, bool compositeComposite) + { + Dir = NewDirectory(); + IndexWriter w = new IndexWriter(Dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + Document d1 = new Document(); + d1.Add(NewTextField("f1", "v1", Field.Store.YES)); + d1.Add(NewTextField("f2", "v1", Field.Store.YES)); + d1.Add(NewTextField("f3", "v1", Field.Store.YES)); + d1.Add(NewTextField("f4", "v1", Field.Store.YES)); + w.AddDocument(d1); + Document d2 = new Document(); + d2.Add(NewTextField("f1", "v2", Field.Store.YES)); + d2.Add(NewTextField("f2", "v2", Field.Store.YES)); + d2.Add(NewTextField("f3", "v2", Field.Store.YES)); + d2.Add(NewTextField("f4", "v2", Field.Store.YES)); + w.AddDocument(d2); + Document d3 = new Document(); + d3.Add(NewTextField("f1", "v3", Field.Store.YES)); + d3.Add(NewTextField("f2", "v3", Field.Store.YES)); + d3.Add(NewTextField("f3", "v3", Field.Store.YES)); + d3.Add(NewTextField("f4", "v3", Field.Store.YES)); + w.AddDocument(d3); + Document d4 = new Document(); + d4.Add(NewTextField("f1", "v4", Field.Store.YES)); + d4.Add(NewTextField("f2", "v4", Field.Store.YES)); + d4.Add(NewTextField("f3", "v4", Field.Store.YES)); + d4.Add(NewTextField("f4", "v4", Field.Store.YES)); + w.AddDocument(d4); + w.Dispose(); + + CompositeReader ir; + if (compositeComposite) + { + ir = new MultiReader(DirectoryReader.Open(Dir), DirectoryReader.Open(Dir)); + } + else + { + ir = DirectoryReader.Open(Dir); + } + return NewSearcher(ir); + } + + // Fields 1 & 2 in one index, 3 & 4 in other, with ParallelReader: + private IndexSearcher Parallel(Random random, bool compositeComposite) + { + Dir1 = GetDir1(random); + Dir2 = GetDir2(random); + CompositeReader rd1, rd2; + if (compositeComposite) + { + rd1 = new MultiReader(DirectoryReader.Open(Dir1), DirectoryReader.Open(Dir1)); + rd2 = new MultiReader(DirectoryReader.Open(Dir2), DirectoryReader.Open(Dir2)); + Assert.AreEqual(2, rd1.Context.Children.Count); + Assert.AreEqual(2, rd2.Context.Children.Count); + } + else + { + rd1 = DirectoryReader.Open(Dir1); + rd2 = DirectoryReader.Open(Dir2); + Assert.AreEqual(3, rd1.Context.Children.Count); + Assert.AreEqual(3, rd2.Context.Children.Count); + } + ParallelCompositeReader pr = new ParallelCompositeReader(rd1, rd2); + return NewSearcher(pr); + } + + // subreader structure: (1,2,1) + private Directory GetDir1(Random random) + { + Directory dir1 = NewDirectory(); + IndexWriter w1 = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES)); + Document d1 = new Document(); + d1.Add(NewTextField("f1", "v1", Field.Store.YES)); + d1.Add(NewTextField("f2", "v1", Field.Store.YES)); + w1.AddDocument(d1); + w1.Commit(); + Document d2 = new Document(); + d2.Add(NewTextField("f1", "v2", Field.Store.YES)); + d2.Add(NewTextField("f2", "v2", Field.Store.YES)); + w1.AddDocument(d2); + Document d3 = new Document(); + d3.Add(NewTextField("f1", "v3", Field.Store.YES)); + d3.Add(NewTextField("f2", "v3", Field.Store.YES)); + w1.AddDocument(d3); + w1.Commit(); + Document d4 = new Document(); + d4.Add(NewTextField("f1", "v4", Field.Store.YES)); + d4.Add(NewTextField("f2", "v4", Field.Store.YES)); + w1.AddDocument(d4); + w1.Dispose(); + return dir1; + } + + // subreader structure: (1,2,1) + private Directory GetDir2(Random random) + { + Directory dir2 = NewDirectory(); + IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES)); + Document d1 = new Document(); + d1.Add(NewTextField("f3", "v1", Field.Store.YES)); + d1.Add(NewTextField("f4", "v1", Field.Store.YES)); + w2.AddDocument(d1); + w2.Commit(); + Document d2 = new Document(); + d2.Add(NewTextField("f3", "v2", Field.Store.YES)); + d2.Add(NewTextField("f4", "v2", Field.Store.YES)); + w2.AddDocument(d2); + Document d3 = new Document(); + d3.Add(NewTextField("f3", "v3", Field.Store.YES)); + d3.Add(NewTextField("f4", "v3", Field.Store.YES)); + w2.AddDocument(d3); + w2.Commit(); + Document d4 = new Document(); + d4.Add(NewTextField("f3", "v4", Field.Store.YES)); + d4.Add(NewTextField("f4", "v4", Field.Store.YES)); + w2.AddDocument(d4); + w2.Dispose(); + return dir2; + } + + // this dir has a different subreader structure (1,1,2); + private Directory GetInvalidStructuredDir2(Random random) + { + Directory dir2 = NewDirectory(); + IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES)); + Document d1 = new Document(); + d1.Add(NewTextField("f3", "v1", Field.Store.YES)); + d1.Add(NewTextField("f4", "v1", Field.Store.YES)); + w2.AddDocument(d1); + w2.Commit(); + Document d2 = new Document(); + d2.Add(NewTextField("f3", "v2", Field.Store.YES)); + d2.Add(NewTextField("f4", "v2", Field.Store.YES)); + w2.AddDocument(d2); + w2.Commit(); + Document d3 = new Document(); + d3.Add(NewTextField("f3", "v3", Field.Store.YES)); + d3.Add(NewTextField("f4", "v3", Field.Store.YES)); + w2.AddDocument(d3); + Document d4 = new Document(); + d4.Add(NewTextField("f3", "v4", Field.Store.YES)); + d4.Add(NewTextField("f4", "v4", Field.Store.YES)); + w2.AddDocument(d4); + w2.Dispose(); + return dir2; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestParallelReaderEmptyIndex.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestParallelReaderEmptyIndex.cs b/src/Lucene.Net.Tests/Index/TestParallelReaderEmptyIndex.cs new file mode 100644 index 0000000..f51128e --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestParallelReaderEmptyIndex.cs @@ -0,0 +1,162 @@ +using System; +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using NUnit.Framework; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using Field = Field; + using FieldType = FieldType; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using TextField = TextField; + + /// <summary> + /// Some tests for <seealso cref="ParallelAtomicReader"/>s with empty indexes + /// </summary> + [TestFixture] + public class TestParallelReaderEmptyIndex : LuceneTestCase + { + /// <summary> + /// Creates two empty indexes and wraps a ParallelReader around. Adding this + /// reader to a new index should not throw any exception. + /// </summary> + [Test] + public virtual void TestEmptyIndex() + { + Directory rd1 = NewDirectory(); + IndexWriter iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + iw.Dispose(); + // create a copy: + Directory rd2 = NewDirectory(rd1); + + Directory rdOut = NewDirectory(); + + IndexWriter iwOut = new IndexWriter(rdOut, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + + ParallelAtomicReader apr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(rd1)), SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(rd2))); + + // When unpatched, Lucene crashes here with a NoSuchElementException (caused by ParallelTermEnum) + iwOut.AddIndexes(apr); + iwOut.ForceMerge(1); + + // 2nd try with a readerless parallel reader + iwOut.AddIndexes(new ParallelAtomicReader()); + iwOut.ForceMerge(1); + + ParallelCompositeReader cpr = new ParallelCompositeReader(DirectoryReader.Open(rd1), DirectoryReader.Open(rd2)); + + // When unpatched, Lucene crashes here with a NoSuchElementException (caused by ParallelTermEnum) + iwOut.AddIndexes(cpr); + iwOut.ForceMerge(1); + + // 2nd try with a readerless parallel reader + iwOut.AddIndexes(new ParallelCompositeReader()); + iwOut.ForceMerge(1); + + iwOut.Dispose(); + rdOut.Dispose(); + rd1.Dispose(); + rd2.Dispose(); + } + + /// <summary> + /// this method creates an empty index (numFields=0, numDocs=0) but is marked + /// to have TermVectors. Adding this index to another index should not throw + /// any exception. + /// </summary> + [Test] + public virtual void TestEmptyIndexWithVectors() + { + Directory rd1 = NewDirectory(); + { + if (VERBOSE) + { + Console.WriteLine("\nTEST: make 1st writer"); + } + IndexWriter iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + Document doc = new Document(); + Field idField = NewTextField("id", "", Field.Store.NO); + doc.Add(idField); + FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); + customType.StoreTermVectors = true; + doc.Add(NewField("test", "", customType)); + idField.SetStringValue("1"); + iw.AddDocument(doc); + doc.Add(NewTextField("test", "", Field.Store.NO)); + idField.SetStringValue("2"); + iw.AddDocument(doc); + iw.Dispose(); + + IndexWriterConfig dontMergeConfig = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMergePolicy(NoMergePolicy.COMPOUND_FILES); + if (VERBOSE) + { + Console.WriteLine("\nTEST: make 2nd writer"); + } + IndexWriter writer = new IndexWriter(rd1, dontMergeConfig); + + writer.DeleteDocuments(new Term("id", "1")); + writer.Dispose(); + IndexReader ir = DirectoryReader.Open(rd1); + Assert.AreEqual(2, ir.MaxDoc); + Assert.AreEqual(1, ir.NumDocs); + ir.Dispose(); + + iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.APPEND)); + iw.ForceMerge(1); + iw.Dispose(); + } + + Directory rd2 = NewDirectory(); + { + IndexWriter iw = new IndexWriter(rd2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + Document doc = new Document(); + iw.AddDocument(doc); + iw.Dispose(); + } + + Directory rdOut = NewDirectory(); + + IndexWriter iwOut = new IndexWriter(rdOut, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + DirectoryReader reader1, reader2; + ParallelAtomicReader pr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(reader1 = DirectoryReader.Open(rd1)), SlowCompositeReaderWrapper.Wrap(reader2 = DirectoryReader.Open(rd2))); + + // When unpatched, Lucene crashes here with an ArrayIndexOutOfBoundsException (caused by TermVectorsWriter) + iwOut.AddIndexes(pr); + + // ParallelReader closes any IndexReader you added to it: + pr.Dispose(); + + // assert subreaders were closed + Assert.AreEqual(0, reader1.RefCount); + Assert.AreEqual(0, reader2.RefCount); + + rd1.Dispose(); + rd2.Dispose(); + + iwOut.ForceMerge(1); + iwOut.Dispose(); + + rdOut.Dispose(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestParallelTermEnum.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestParallelTermEnum.cs b/src/Lucene.Net.Tests/Index/TestParallelTermEnum.cs new file mode 100644 index 0000000..9b6ac85 --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestParallelTermEnum.cs @@ -0,0 +1,127 @@ +using System.Collections.Generic; +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using NUnit.Framework; + using IBits = Lucene.Net.Util.IBits; + using BytesRef = Lucene.Net.Util.BytesRef; + using Directory = Lucene.Net.Store.Directory; + using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator; + using Document = Documents.Document; + using Field = Field; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using TestUtil = Lucene.Net.Util.TestUtil; + + [TestFixture] + public class TestParallelTermEnum : LuceneTestCase + { + private AtomicReader Ir1; + private AtomicReader Ir2; + private Directory Rd1; + private Directory Rd2; + + [SetUp] + public override void SetUp() + { + base.SetUp(); + Document doc; + Rd1 = NewDirectory(); + IndexWriter iw1 = new IndexWriter(Rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + + doc = new Document(); + doc.Add(NewTextField("field1", "the quick brown fox jumps", Field.Store.YES)); + doc.Add(NewTextField("field2", "the quick brown fox jumps", Field.Store.YES)); + iw1.AddDocument(doc); + + iw1.Dispose(); + Rd2 = NewDirectory(); + IndexWriter iw2 = new IndexWriter(Rd2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + + doc = new Document(); + doc.Add(NewTextField("field1", "the fox jumps over the lazy dog", Field.Store.YES)); + doc.Add(NewTextField("field3", "the fox jumps over the lazy dog", Field.Store.YES)); + iw2.AddDocument(doc); + + iw2.Dispose(); + + this.Ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(Rd1)); + this.Ir2 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(Rd2)); + } + + [TearDown] + public override void TearDown() + { + Ir1.Dispose(); + Ir2.Dispose(); + Rd1.Dispose(); + Rd2.Dispose(); + base.TearDown(); + } + + private void CheckTerms(Terms terms, IBits liveDocs, params string[] termsList) + { + Assert.IsNotNull(terms); + TermsEnum te = terms.GetIterator(null); + + foreach (string t in termsList) + { + BytesRef b = te.Next(); + Assert.IsNotNull(b); + Assert.AreEqual(t, b.Utf8ToString()); + DocsEnum td = TestUtil.Docs(Random(), te, liveDocs, null, DocsEnum.FLAG_NONE); + Assert.IsTrue(td.NextDoc() != DocIdSetIterator.NO_MORE_DOCS); + Assert.AreEqual(0, td.DocID); + Assert.AreEqual(td.NextDoc(), DocIdSetIterator.NO_MORE_DOCS); + } + Assert.IsNull(te.Next()); + } + + [Test] + public virtual void Test1() + { + ParallelAtomicReader pr = new ParallelAtomicReader(Ir1, Ir2); + + IBits liveDocs = pr.LiveDocs; + + Fields fields = pr.Fields; + IEnumerator<string> fe = fields.GetEnumerator(); + + fe.MoveNext(); + string f = fe.Current; + Assert.AreEqual("field1", f); + CheckTerms(fields.GetTerms(f), liveDocs, "brown", "fox", "jumps", "quick", "the"); + + fe.MoveNext(); + f = fe.Current; + Assert.AreEqual("field2", f); + CheckTerms(fields.GetTerms(f), liveDocs, "brown", "fox", "jumps", "quick", "the"); + + fe.MoveNext(); + f = fe.Current; + Assert.AreEqual("field3", f); + CheckTerms(fields.GetTerms(f), liveDocs, "dog", "fox", "jumps", "lazy", "over", "the"); + + Assert.IsFalse(fe.MoveNext()); + } + } +} \ No newline at end of file
