http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestMultiFields.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestMultiFields.cs b/src/Lucene.Net.Tests/Index/TestMultiFields.cs new file mode 100644 index 0000000..0d70862 --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestMultiFields.cs @@ -0,0 +1,228 @@ +using Lucene.Net.Documents; +using Lucene.Net.Randomized.Generators; +using System; +using System.Collections.Generic; + +namespace Lucene.Net.Index +{ + using Lucene.Net.Analysis; + + using Lucene.Net.Store; + using Lucene.Net.Util; + using NUnit.Framework; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator; + + [TestFixture] + public class TestMultiFields : LuceneTestCase + { + [Test] + public virtual void TestRandom() + { + int num = AtLeast(2); + for (int iter = 0; iter < num; iter++) + { + if (VERBOSE) + { + Console.WriteLine("TEST: iter=" + iter); + } + + Directory dir = NewDirectory(); + + IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.COMPOUND_FILES)); + // we can do this because we use NoMergePolicy (and dont merge to "nothing") + w.KeepFullyDeletedSegments = true; + + IDictionary<BytesRef, IList<int?>> docs = new Dictionary<BytesRef, IList<int?>>(); + HashSet<int?> deleted = new HashSet<int?>(); + IList<BytesRef> terms = new List<BytesRef>(); + + int numDocs = TestUtil.NextInt(Random(), 1, 100 * RANDOM_MULTIPLIER); + Documents.Document doc = new Documents.Document(); + Field f = NewStringField("field", "", Field.Store.NO); + doc.Add(f); + Field id = NewStringField("id", "", Field.Store.NO); + doc.Add(id); + + bool onlyUniqueTerms = Random().NextBoolean(); + if (VERBOSE) + { + Console.WriteLine("TEST: onlyUniqueTerms=" + onlyUniqueTerms + " numDocs=" + numDocs); + } + HashSet<BytesRef> uniqueTerms = new HashSet<BytesRef>(); + for (int i = 0; i < numDocs; i++) + { + if (!onlyUniqueTerms && Random().NextBoolean() && terms.Count > 0) + { + // re-use existing term + BytesRef term = terms[Random().Next(terms.Count)]; + docs[term].Add(i); + f.SetStringValue(term.Utf8ToString()); + } + else + { + string s = TestUtil.RandomUnicodeString(Random(), 10); + BytesRef term = new BytesRef(s); + if (!docs.ContainsKey(term)) + { + docs[term] = new List<int?>(); + } + docs[term].Add(i); + terms.Add(term); + uniqueTerms.Add(term); + f.SetStringValue(s); + } + id.SetStringValue("" + i); + w.AddDocument(doc); + if (Random().Next(4) == 1) + { + w.Commit(); + } + if (i > 0 && Random().Next(20) == 1) + { + int delID = Random().Next(i); + deleted.Add(delID); + w.DeleteDocuments(new Term("id", "" + delID)); + if (VERBOSE) + { + Console.WriteLine("TEST: delete " + delID); + } + } + } + + if (VERBOSE) + { + List<BytesRef> termsList = new List<BytesRef>(uniqueTerms); +#pragma warning disable 612, 618 + termsList.Sort(BytesRef.UTF8SortedAsUTF16Comparer); +#pragma warning restore 612, 618 + Console.WriteLine("TEST: terms in UTF16 order:"); + foreach (BytesRef b in termsList) + { + Console.WriteLine(" " + UnicodeUtil.ToHexString(b.Utf8ToString()) + " " + b); + foreach (int docID in docs[b]) + { + if (deleted.Contains(docID)) + { + Console.WriteLine(" " + docID + " (deleted)"); + } + else + { + Console.WriteLine(" " + docID); + } + } + } + } + + IndexReader reader = w.Reader; + w.Dispose(); + if (VERBOSE) + { + Console.WriteLine("TEST: reader=" + reader); + } + + IBits liveDocs = MultiFields.GetLiveDocs(reader); + foreach (int delDoc in deleted) + { + Assert.IsFalse(liveDocs.Get(delDoc)); + } + + for (int i = 0; i < 100; i++) + { + BytesRef term = terms[Random().Next(terms.Count)]; + if (VERBOSE) + { + Console.WriteLine("TEST: seek term=" + UnicodeUtil.ToHexString(term.Utf8ToString()) + " " + term); + } + + DocsEnum docsEnum = TestUtil.Docs(Random(), reader, "field", term, liveDocs, null, DocsEnum.FLAG_NONE); + Assert.IsNotNull(docsEnum); + + foreach (int docID in docs[term]) + { + if (!deleted.Contains(docID)) + { + Assert.AreEqual(docID, docsEnum.NextDoc()); + } + } + Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, docsEnum.NextDoc()); + } + + reader.Dispose(); + dir.Dispose(); + } + } + + /* + private void verify(IndexReader r, String term, List<Integer> expected) throws Exception { + DocsEnum docs = TestUtil.Docs(random, r, + "field", + new BytesRef(term), + MultiFields.GetLiveDocs(r), + null, + false); + for(int docID : expected) { + Assert.AreEqual(docID, docs.NextDoc()); + } + Assert.AreEqual(docs.NO_MORE_DOCS, docs.NextDoc()); + } + */ + + [Test] + public virtual void TestSeparateEnums() + { + Directory dir = NewDirectory(); + IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + Documents.Document d = new Documents.Document(); + d.Add(NewStringField("f", "j", Field.Store.NO)); + w.AddDocument(d); + w.Commit(); + w.AddDocument(d); + IndexReader r = w.Reader; + w.Dispose(); + DocsEnum d1 = TestUtil.Docs(Random(), r, "f", new BytesRef("j"), null, null, DocsEnum.FLAG_NONE); + DocsEnum d2 = TestUtil.Docs(Random(), r, "f", new BytesRef("j"), null, null, DocsEnum.FLAG_NONE); + Assert.AreEqual(0, d1.NextDoc()); + Assert.AreEqual(0, d2.NextDoc()); + r.Dispose(); + dir.Dispose(); + } + + [Test] + public virtual void TestTermDocsEnum() + { + Directory dir = NewDirectory(); + IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + Documents.Document d = new Documents.Document(); + d.Add(NewStringField("f", "j", Field.Store.NO)); + w.AddDocument(d); + w.Commit(); + w.AddDocument(d); + IndexReader r = w.Reader; + w.Dispose(); + DocsEnum de = MultiFields.GetTermDocsEnum(r, null, "f", new BytesRef("j")); + Assert.AreEqual(0, de.NextDoc()); + Assert.AreEqual(1, de.NextDoc()); + Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, de.NextDoc()); + r.Dispose(); + dir.Dispose(); + } + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestMultiLevelSkipList.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestMultiLevelSkipList.cs b/src/Lucene.Net.Tests/Index/TestMultiLevelSkipList.cs new file mode 100644 index 0000000..49456ed --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestMultiLevelSkipList.cs @@ -0,0 +1,218 @@ +using Lucene.Net.Analysis.TokenAttributes; +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using Lucene.Net.Analysis; + using Lucene.Net.Support; + using NUnit.Framework; + using System.IO; + using BytesRef = Lucene.Net.Util.BytesRef; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using Field = Field; + using IndexInput = Lucene.Net.Store.IndexInput; + using IOContext = Lucene.Net.Store.IOContext; + using Lucene41PostingsFormat = Lucene.Net.Codecs.Lucene41.Lucene41PostingsFormat; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper; + using RAMDirectory = Lucene.Net.Store.RAMDirectory; + using TestUtil = Lucene.Net.Util.TestUtil; + + /// <summary> + /// this testcase tests whether multi-level skipping is being used + /// to reduce I/O while skipping through posting lists. + /// + /// Skipping in general is already covered by several other + /// testcases. + /// + /// </summary> + [TestFixture] + public class TestMultiLevelSkipList : LuceneTestCase + { + internal class CountingRAMDirectory : MockDirectoryWrapper + { + private readonly TestMultiLevelSkipList OuterInstance; + + public CountingRAMDirectory(TestMultiLevelSkipList outerInstance, Directory @delegate) + : base(Random(), @delegate) + { + this.OuterInstance = outerInstance; + } + + public override IndexInput OpenInput(string fileName, IOContext context) + { + IndexInput @in = base.OpenInput(fileName, context); + if (fileName.EndsWith(".frq")) + { + @in = new CountingStream(OuterInstance, @in); + } + return @in; + } + } + + [SetUp] + public override void SetUp() + { + base.SetUp(); + Counter = 0; + } + + [Test] + public virtual void TestSimpleSkip() + { + Directory dir = new CountingRAMDirectory(this, new RAMDirectory()); + IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer()).SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat())).SetMergePolicy(NewLogMergePolicy())); + Term term = new Term("test", "a"); + for (int i = 0; i < 5000; i++) + { + Document d1 = new Document(); + d1.Add(NewTextField(term.Field, term.Text(), Field.Store.NO)); + writer.AddDocument(d1); + } + writer.Commit(); + writer.ForceMerge(1); + writer.Dispose(); + + AtomicReader reader = GetOnlySegmentReader(DirectoryReader.Open(dir)); + + for (int i = 0; i < 2; i++) + { + Counter = 0; + DocsAndPositionsEnum tp = reader.TermPositionsEnum(term); + CheckSkipTo(tp, 14, 185); // no skips + CheckSkipTo(tp, 17, 190); // one skip on level 0 + CheckSkipTo(tp, 287, 200); // one skip on level 1, two on level 0 + + // this test would fail if we had only one skip level, + // because than more bytes would be read from the freqStream + CheckSkipTo(tp, 4800, 250); // one skip on level 2 + } + } + + public virtual void CheckSkipTo(DocsAndPositionsEnum tp, int target, int maxCounter) + { + tp.Advance(target); + if (maxCounter < Counter) + { + Assert.Fail("Too many bytes read: " + Counter + " vs " + maxCounter); + } + + Assert.AreEqual(target, tp.DocID, "Wrong document " + tp.DocID + " after skipTo target " + target); + Assert.AreEqual(1, tp.Freq, "Frequency is not 1: " + tp.Freq); + tp.NextPosition(); + BytesRef b = tp.GetPayload(); + Assert.AreEqual(1, b.Length); + Assert.AreEqual((sbyte)target, (sbyte)b.Bytes[b.Offset], "Wrong payload for the target " + target + ": " + (sbyte)b.Bytes[b.Offset]); + } + + private class PayloadAnalyzer : Analyzer + { + internal readonly AtomicInt32 PayloadCount = new AtomicInt32(-1); + + protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) + { + Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true); + return new TokenStreamComponents(tokenizer, new PayloadFilter(PayloadCount, tokenizer)); + } + } + + private class PayloadFilter : TokenFilter + { + internal IPayloadAttribute PayloadAtt; + internal AtomicInt32 PayloadCount; + + protected internal PayloadFilter(AtomicInt32 payloadCount, TokenStream input) + : base(input) + { + this.PayloadCount = payloadCount; + PayloadAtt = AddAttribute<IPayloadAttribute>(); + } + + public sealed override bool IncrementToken() + { + bool hasNext = m_input.IncrementToken(); + if (hasNext) + { + PayloadAtt.Payload = new BytesRef(new[] { (byte)PayloadCount.IncrementAndGet() }); + } + return hasNext; + } + } + + private int Counter = 0; + + // Simply extends IndexInput in a way that we are able to count the number + // of bytes read + internal class CountingStream : IndexInput + { + private readonly TestMultiLevelSkipList OuterInstance; + + internal IndexInput Input; + + internal CountingStream(TestMultiLevelSkipList outerInstance, IndexInput input) + : base("CountingStream(" + input + ")") + { + this.OuterInstance = outerInstance; + this.Input = input; + } + + public override byte ReadByte() + { + OuterInstance.Counter++; + return this.Input.ReadByte(); + } + + public override void ReadBytes(byte[] b, int offset, int len) + { + OuterInstance.Counter += len; + this.Input.ReadBytes(b, offset, len); + } + + public override void Dispose() + { + this.Input.Dispose(); + } + + public override long FilePointer + { + get + { + return this.Input.FilePointer; + } + } + + public override void Seek(long pos) + { + this.Input.Seek(pos); + } + + public override long Length + { + get { return this.Input.Length; } + } + + public override object Clone() + { + return new CountingStream(OuterInstance, (IndexInput)this.Input.Clone()); + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestNRTReaderWithThreads.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestNRTReaderWithThreads.cs b/src/Lucene.Net.Tests/Index/TestNRTReaderWithThreads.cs new file mode 100644 index 0000000..918b5b5 --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestNRTReaderWithThreads.cs @@ -0,0 +1,140 @@ +using System; +using System.Threading; + +namespace Lucene.Net.Index +{ + using Lucene.Net.Support; + using NUnit.Framework; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper; + + [TestFixture] + public class TestNRTReaderWithThreads : LuceneTestCase + { + internal AtomicInt32 Seq = new AtomicInt32(1); + [Test] + public virtual void TestIndexing() + { + Directory mainDir = NewDirectory(); + var wrapper = mainDir as MockDirectoryWrapper; + if (wrapper != null) + { + wrapper.AssertNoDeleteOpenFile = true; + } + var writer = new IndexWriter(mainDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(10).SetMergePolicy(NewLogMergePolicy(false, 2))); + IndexReader reader = writer.Reader; // start pooling readers + reader.Dispose(); + var indexThreads = new RunThread[4]; + for (int x = 0; x < indexThreads.Length; x++) + { + indexThreads[x] = new RunThread(this, x % 2, writer); + indexThreads[x].Name = "Thread " + x; + indexThreads[x].Start(); + } + long startTime = Environment.TickCount; + long duration = 1000; + while ((Environment.TickCount - startTime) < duration) + { + Thread.Sleep(100); + } + int delCount = 0; + int addCount = 0; + for (int x = 0; x < indexThreads.Length; x++) + { + indexThreads[x].Run_Renamed = false; + Assert.IsNull(indexThreads[x].Ex, "Exception thrown: " + indexThreads[x].Ex); + addCount += indexThreads[x].AddCount; + delCount += indexThreads[x].DelCount; + } + for (int x = 0; x < indexThreads.Length; x++) + { + indexThreads[x].Join(); + } + for (int x = 0; x < indexThreads.Length; x++) + { + Assert.IsNull(indexThreads[x].Ex, "Exception thrown: " + indexThreads[x].Ex); + } + //System.out.println("addCount:"+addCount); + //System.out.println("delCount:"+delCount); + writer.Dispose(); + mainDir.Dispose(); + } + + public class RunThread : ThreadClass + { + private readonly TestNRTReaderWithThreads OuterInstance; + + internal IndexWriter Writer; + internal volatile bool Run_Renamed = true; + internal volatile Exception Ex; + internal int DelCount = 0; + internal int AddCount = 0; + internal int Type; + internal readonly Random r = new Random(Random().Next()); + + public RunThread(TestNRTReaderWithThreads outerInstance, int type, IndexWriter writer) + { + this.OuterInstance = outerInstance; + this.Type = type; + this.Writer = writer; + } + + public override void Run() + { + try + { + while (Run_Renamed) + { + //int n = random.nextInt(2); + if (Type == 0) + { + int i = OuterInstance.Seq.AddAndGet(1); + Document doc = DocHelper.CreateDocument(i, "index1", 10); + Writer.AddDocument(doc); + AddCount++; + } + else if (Type == 1) + { + // we may or may not delete because the term may not exist, + // however we're opening and closing the reader rapidly + IndexReader reader = Writer.Reader; + int id = r.Next(OuterInstance.Seq.Get()); + Term term = new Term("id", Convert.ToString(id)); + int count = TestIndexWriterReader.Count(term, reader); + Writer.DeleteDocuments(term); + reader.Dispose(); + DelCount += count; + } + } + } + catch (Exception ex) + { + Console.WriteLine(ex.StackTrace); + this.Ex = ex; + Run_Renamed = false; + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestNRTThreads.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestNRTThreads.cs b/src/Lucene.Net.Tests/Index/TestNRTThreads.cs new file mode 100644 index 0000000..f6815cb --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestNRTThreads.cs @@ -0,0 +1,185 @@ +using System; +using System.Diagnostics; +using System.Threading.Tasks; + +namespace Lucene.Net.Index +{ + using Lucene.Net.Randomized.Generators; + using NUnit.Framework; + using System.Collections.Generic; + using Directory = Lucene.Net.Store.Directory; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using IndexSearcher = Lucene.Net.Search.IndexSearcher; + using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper; + + // TODO + // - mix in forceMerge, addIndexes + // - randomoly mix in non-congruent docs + [SuppressCodecs("SimpleText", "Memory", "Direct")] + [TestFixture] + public class TestNRTThreads : ThreadedIndexingAndSearchingTestCase + { + private bool UseNonNrtReaders = true; + + [SetUp] + public override void SetUp() + { + base.SetUp(); + UseNonNrtReaders = Random().NextBoolean(); + } + + protected internal override void DoSearching(TaskScheduler es, DateTime stopTime) + { + bool anyOpenDelFiles = false; + + DirectoryReader r = DirectoryReader.Open(Writer, true); + + while (DateTime.UtcNow < stopTime && !Failed.Get()) + { + if (Random().NextBoolean()) + { + if (VERBOSE) + { + Console.WriteLine("TEST: now reopen r=" + r); + } + DirectoryReader r2 = DirectoryReader.OpenIfChanged(r); + if (r2 != null) + { + r.Dispose(); + r = r2; + } + } + else + { + if (VERBOSE) + { + Console.WriteLine("TEST: now close reader=" + r); + } + r.Dispose(); + Writer.Commit(); + ISet<string> openDeletedFiles = ((MockDirectoryWrapper)Dir).OpenDeletedFiles; + if (openDeletedFiles.Count > 0) + { + Console.WriteLine("OBD files: " + openDeletedFiles); + } + anyOpenDelFiles |= openDeletedFiles.Count > 0; + //Assert.AreEqual("open but deleted: " + openDeletedFiles, 0, openDeletedFiles.Size()); + if (VERBOSE) + { + Console.WriteLine("TEST: now open"); + } + r = DirectoryReader.Open(Writer, true); + } + if (VERBOSE) + { + Console.WriteLine("TEST: got new reader=" + r); + } + //System.out.println("numDocs=" + r.NumDocs + " + //openDelFileCount=" + dir.openDeleteFileCount()); + + if (r.NumDocs > 0) + { + FixedSearcher = new IndexSearcher(r, es); + SmokeTestSearcher(FixedSearcher); + RunSearchThreads(DateTime.UtcNow.AddMilliseconds(500)); + } + } + r.Dispose(); + + //System.out.println("numDocs=" + r.NumDocs + " openDelFileCount=" + dir.openDeleteFileCount()); + ISet<string> openDeletedFiles_ = ((MockDirectoryWrapper)Dir).OpenDeletedFiles; + if (openDeletedFiles_.Count > 0) + { + Console.WriteLine("OBD files: " + openDeletedFiles_); + } + anyOpenDelFiles |= openDeletedFiles_.Count > 0; + + Assert.IsFalse(anyOpenDelFiles, "saw non-zero open-but-deleted count"); + } + + protected internal override Directory GetDirectory(Directory @in) + { + Debug.Assert(@in is MockDirectoryWrapper); + if (!UseNonNrtReaders) + { + ((MockDirectoryWrapper)@in).AssertNoDeleteOpenFile = true; + } + return @in; + } + + protected internal override void DoAfterWriter(TaskScheduler es) + { + // Force writer to do reader pooling, always, so that + // all merged segments, even for merges before + // doSearching is called, are warmed: + Writer.Reader.Dispose(); + } + + private IndexSearcher FixedSearcher; + + protected internal override IndexSearcher CurrentSearcher + { + get + { + return FixedSearcher; + } + } + + protected internal override void ReleaseSearcher(IndexSearcher s) + { + if (s != FixedSearcher) + { + // Final searcher: + s.IndexReader.Dispose(); + } + } + + protected internal override IndexSearcher FinalSearcher + { + get + { + IndexReader r2; + if (UseNonNrtReaders) + { + if (Random().NextBoolean()) + { + r2 = Writer.Reader; + } + else + { + Writer.Commit(); + r2 = DirectoryReader.Open(Dir); + } + } + else + { + r2 = Writer.Reader; + } + return NewSearcher(r2); + } + } + + [Test] + public virtual void TestNRTThreads_Mem() + { + RunTest("TestNRTThreads"); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestNeverDelete.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestNeverDelete.cs b/src/Lucene.Net.Tests/Index/TestNeverDelete.cs new file mode 100644 index 0000000..572182c --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestNeverDelete.cs @@ -0,0 +1,158 @@ +using Lucene.Net.Documents; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; + +namespace Lucene.Net.Index +{ + using Lucene.Net.Support; + using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper; + using Document = Documents.Document; + using Field = Field; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper; + using TestUtil = Lucene.Net.Util.TestUtil; + + // Make sure if you use NoDeletionPolicy that no file + // referenced by a commit point is ever deleted + + [TestFixture] + public class TestNeverDelete : LuceneTestCase + { + [Test] + public virtual void TestIndexing() + { + DirectoryInfo tmpDir = CreateTempDir("TestNeverDelete"); + BaseDirectoryWrapper d = NewFSDirectory(tmpDir); + + // We want to "see" files removed if Lucene removed + // them. this is still worth running on Windows since + // some files the IR opens and closes. + if (d is MockDirectoryWrapper) + { + ((MockDirectoryWrapper)d).NoDeleteOpenFile = false; + } + RandomIndexWriter w = new RandomIndexWriter(Random(), d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetIndexDeletionPolicy(NoDeletionPolicy.INSTANCE)); + w.w.Config.SetMaxBufferedDocs(TestUtil.NextInt(Random(), 5, 30)); + + w.Commit(); + ThreadClass[] indexThreads = new ThreadClass[Random().Next(4)]; + long stopTime = Environment.TickCount + AtLeast(1000); + for (int x = 0; x < indexThreads.Length; x++) + { + indexThreads[x] = new ThreadAnonymousInnerClassHelper(w, stopTime, NewStringField, NewTextField); + indexThreads[x].Name = "Thread " + x; + indexThreads[x].Start(); + } + + HashSet<string> allFiles = new HashSet<string>(); + + DirectoryReader r = DirectoryReader.Open(d); + while (Environment.TickCount < stopTime) + { + IndexCommit ic = r.IndexCommit; + if (VERBOSE) + { + Console.WriteLine("TEST: check files: " + ic.FileNames); + } + allFiles.AddAll(ic.FileNames); + // Make sure no old files were removed + foreach (string fileName in allFiles) + { + Assert.IsTrue(SlowFileExists(d, fileName), "file " + fileName + " does not exist"); + } + DirectoryReader r2 = DirectoryReader.OpenIfChanged(r); + if (r2 != null) + { + r.Dispose(); + r = r2; + } + Thread.Sleep(1); + } + r.Dispose(); + + foreach (ThreadClass t in indexThreads) + { + t.Join(); + } + w.Dispose(); + d.Dispose(); + + System.IO.Directory.Delete(tmpDir.FullName, true); + } + + private class ThreadAnonymousInnerClassHelper : ThreadClass + { + private readonly Func<string, string, Field.Store, Field> NewStringField; + private readonly Func<string, string, Field.Store, Field> NewTextField; + + private RandomIndexWriter w; + private long StopTime; + + /// <param name="newStringField"> + /// LUCENENET specific + /// Passed in because <see cref="LuceneTestCase.NewStringField(string, string, Field.Store)"/> + /// is no longer static + /// </param> + /// <param name="newTextField"> + /// LUCENENET specific + /// Passed in because <see cref="LuceneTestCase.NewTextField(string, string, Field.Store)"/> + /// is no longer static + /// </param> + public ThreadAnonymousInnerClassHelper(RandomIndexWriter w, long stopTime, + Func<string, string, Field.Store, Field> newStringField, Func<string, string, Field.Store, Field> newTextField) + { + this.w = w; + this.StopTime = stopTime; + NewStringField = newStringField; + NewTextField = newTextField; + } + + public override void Run() + { + try + { + int docCount = 0; + while (Environment.TickCount < StopTime) + { + Document doc = new Document(); + doc.Add(NewStringField("dc", "" + docCount, Field.Store.YES)); + doc.Add(NewTextField("field", "here is some text", Field.Store.YES)); + w.AddDocument(doc); + + if (docCount % 13 == 0) + { + w.Commit(); + } + docCount++; + } + } + catch (Exception e) + { + throw new Exception(e.Message, e); + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestNewestSegment.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestNewestSegment.cs b/src/Lucene.Net.Tests/Index/TestNewestSegment.cs new file mode 100644 index 0000000..fc76173 --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestNewestSegment.cs @@ -0,0 +1,39 @@ +namespace Lucene.Net.Index +{ + using NUnit.Framework; + using Directory = Lucene.Net.Store.Directory; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + + [TestFixture] + public class TestNewestSegment : LuceneTestCase + { + [Test] + public virtual void TestNewestSegment_Mem() + { + Directory directory = NewDirectory(); + IndexWriter writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + Assert.IsNull(writer.NewestSegment()); + writer.Dispose(); + directory.Dispose(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestNoDeletionPolicy.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestNoDeletionPolicy.cs b/src/Lucene.Net.Tests/Index/TestNoDeletionPolicy.cs new file mode 100644 index 0000000..a91b485 --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestNoDeletionPolicy.cs @@ -0,0 +1,95 @@ +using System.Reflection; +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using NUnit.Framework; + using Support; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using Field = Field; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + + [TestFixture] + public class TestNoDeletionPolicy : LuceneTestCase + { + [Test] + public virtual void TestNoDeletionPolicy_Mem() + { + IndexDeletionPolicy idp = NoDeletionPolicy.INSTANCE; + idp.OnInit<IndexCommit>(null); + idp.OnCommit<IndexCommit>(null); + } + + [Test] + public virtual void TestFinalSingleton() + { + assertTrue(typeof(NoDeletionPolicy).GetTypeInfo().IsSealed); + ConstructorInfo[] ctors = typeof(NoDeletionPolicy).GetConstructors(BindingFlags.Instance | + BindingFlags.NonPublic | + BindingFlags.Public | + BindingFlags.DeclaredOnly); // LUCENENET NOTE: It seems .NET automatically adds a private static constructor, so leaving off the static BindingFlag + assertEquals("expected 1 private ctor only: " + Arrays.ToString(ctors), 1, ctors.Length); + assertTrue("that 1 should be private: " + ctors[0], ctors[0].IsPrivate); + } + + [Test] + public virtual void TestMethodsOverridden() + { + // Ensures that all methods of IndexDeletionPolicy are + // overridden/implemented. That's important to ensure that NoDeletionPolicy + // overrides everything, so that no unexpected behavior/error occurs. + // NOTE: even though IndexDeletionPolicy is an interface today, and so all + // methods must be implemented by NoDeletionPolicy, this test is important + // in case one day IDP becomes an abstract class. + foreach (MethodInfo m in typeof(NoDeletionPolicy).GetMethods()) + { + // getDeclaredMethods() returns just those methods that are declared on + // NoDeletionPolicy. getMethods() returns those that are visible in that + // context, including ones from Object. So just filter out Object. If in + // the future IndexDeletionPolicy will become a class that extends a + // different class than Object, this will need to change. + if (m.DeclaringType != typeof(object) && !m.IsFinal && m.IsVirtual) + { + Assert.IsTrue(m.DeclaringType == typeof(NoDeletionPolicy), m + " is not overridden !"); + } + } + } + + [Test] + public virtual void TestAllCommitsRemain() + { + Directory dir = NewDirectory(); + IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetIndexDeletionPolicy(NoDeletionPolicy.INSTANCE)); + for (int i = 0; i < 10; i++) + { + Document doc = new Document(); + doc.Add(NewTextField("c", "a" + i, Field.Store.YES)); + writer.AddDocument(doc); + writer.Commit(); + Assert.AreEqual(i + 1, DirectoryReader.ListCommits(dir).Count, "wrong number of commits !"); + } + writer.Dispose(); + dir.Dispose(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestNoMergePolicy.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestNoMergePolicy.cs b/src/Lucene.Net.Tests/Index/TestNoMergePolicy.cs new file mode 100644 index 0000000..694126b --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestNoMergePolicy.cs @@ -0,0 +1,83 @@ +using Lucene.Net.Support; +using NUnit.Framework; +using System.Reflection; + +namespace Lucene.Net.Index +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + [TestFixture] + public class TestNoMergePolicy : LuceneTestCase + { + [Test] + public virtual void TestNoMergePolicy_Mem() + { + MergePolicy mp = NoMergePolicy.NO_COMPOUND_FILES; + Assert.IsNull(mp.FindMerges(null, (SegmentInfos)null)); + Assert.IsNull(mp.FindForcedMerges(null, 0, null)); + Assert.IsNull(mp.FindForcedDeletesMerges(null)); + Assert.IsFalse(mp.UseCompoundFile(null, null)); + mp.Dispose(); + } + + [Test] + public virtual void TestCompoundFiles() + { + Assert.IsFalse(NoMergePolicy.NO_COMPOUND_FILES.UseCompoundFile(null, null)); + Assert.IsTrue(NoMergePolicy.COMPOUND_FILES.UseCompoundFile(null, null)); + } + + [Test] + public virtual void TestFinalSingleton() + { + assertTrue(typeof(NoMergePolicy).GetTypeInfo().IsSealed); + ConstructorInfo[] ctors = typeof(NoMergePolicy).GetConstructors(BindingFlags.Instance | + BindingFlags.NonPublic | + BindingFlags.Public | + BindingFlags.DeclaredOnly); // LUCENENET NOTE: It seems .NET automatically adds a private static constructor, so leaving off the static BindingFlag + assertEquals("expected 1 private ctor only: " + Arrays.ToString(ctors), 1, ctors.Length); + assertTrue("that 1 should be private: " + ctors[0], ctors[0].IsPrivate); + } + + [Test] + public virtual void TestMethodsOverridden() + { + // Ensures that all methods of MergePolicy are overridden. That's important + // to ensure that NoMergePolicy overrides everything, so that no unexpected + // behavior/error occurs + foreach (MethodInfo m in typeof(NoMergePolicy).GetMethods()) + { + // getDeclaredMethods() returns just those methods that are declared on + // NoMergePolicy. getMethods() returns those that are visible in that + // context, including ones from Object. So just filter out Object. If in + // the future MergePolicy will extend a different class than Object, this + // will need to change. + if (m.Name.Equals("Clone")) + { + continue; + } + if (m.DeclaringType != typeof(object) && !m.IsFinal && m.IsVirtual) + { + Assert.IsTrue(m.DeclaringType == typeof(NoMergePolicy), m + " is not overridden ! Declaring Type: " + m.DeclaringType); + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestNoMergeScheduler.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestNoMergeScheduler.cs b/src/Lucene.Net.Tests/Index/TestNoMergeScheduler.cs new file mode 100644 index 0000000..9e20f64 --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestNoMergeScheduler.cs @@ -0,0 +1,72 @@ +using Lucene.Net.Randomized.Generators; +using Lucene.Net.Support; +using NUnit.Framework; +using System; +using System.Linq; +using System.Reflection; + +namespace Lucene.Net.Index +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + [TestFixture] + public class TestNoMergeScheduler : LuceneTestCase + { + [Test] + public virtual void TestNoMergeScheduler_Mem() + { + MergeScheduler ms = NoMergeScheduler.INSTANCE; + ms.Dispose(); + ms.Merge(null, RandomInts.RandomFrom(Random(), Enum.GetValues(typeof(MergeTrigger)).Cast<MergeTrigger>().ToArray()), Random().NextBoolean()); + } + + [Test] + public virtual void TestFinalSingleton() + { + assertTrue(typeof(NoMergeScheduler).GetTypeInfo().IsSealed); + ConstructorInfo[] ctors = typeof(NoMergeScheduler).GetConstructors(BindingFlags.Instance | + BindingFlags.NonPublic | + BindingFlags.Public | + BindingFlags.DeclaredOnly); // LUCENENET NOTE: It seems .NET automatically adds a private static constructor, so leaving off the static BindingFlag + assertEquals("expected 1 private ctor only: " + Arrays.ToString(ctors), 1, ctors.Length); + assertTrue("that 1 should be private: " + ctors[0], ctors[0].IsPrivate); + } + + [Test] + public virtual void TestMethodsOverridden() + { + // Ensures that all methods of MergeScheduler are overridden. That's + // important to ensure that NoMergeScheduler overrides everything, so that + // no unexpected behavior/error occurs + foreach (MethodInfo m in typeof(NoMergeScheduler).GetMethods()) + { + // getDeclaredMethods() returns just those methods that are declared on + // NoMergeScheduler. getMethods() returns those that are visible in that + // context, including ones from Object. So just filter out Object. If in + // the future MergeScheduler will extend a different class than Object, + // this will need to change. + if (m.DeclaringType != typeof(object) && !m.IsFinal && m.IsVirtual) + { + Assert.IsTrue(m.DeclaringType == typeof(NoMergeScheduler), m + " is not overridden !"); + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestNorms.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestNorms.cs b/src/Lucene.Net.Tests/Index/TestNorms.cs new file mode 100644 index 0000000..f6bcdcf --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestNorms.cs @@ -0,0 +1,252 @@ +using System; +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using NUnit.Framework; + using BytesRef = Lucene.Net.Util.BytesRef; + using CollectionStatistics = Lucene.Net.Search.CollectionStatistics; + using DefaultSimilarity = Lucene.Net.Search.Similarities.DefaultSimilarity; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using Field = Field; + using LineFileDocs = Lucene.Net.Util.LineFileDocs; + + //using Slow = Lucene.Net.Util.LuceneTestCase.Slow; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using PerFieldSimilarityWrapper = Lucene.Net.Search.Similarities.PerFieldSimilarityWrapper; + using Similarity = Lucene.Net.Search.Similarities.Similarity; + using TermStatistics = Lucene.Net.Search.TermStatistics; + using TestUtil = Lucene.Net.Util.TestUtil; + using TextField = TextField; + using TFIDFSimilarity = Lucene.Net.Search.Similarities.TFIDFSimilarity; + + /// <summary> + /// Test that norms info is preserved during index life - including + /// separate norms, addDocument, addIndexes, forceMerge. + /// </summary> + [SuppressCodecs("Memory", "Direct", "SimpleText")] + [TestFixture] + public class TestNorms : LuceneTestCase + { + internal readonly string ByteTestField = "normsTestByte"; + + internal class CustomNormEncodingSimilarity : TFIDFSimilarity + { + private readonly TestNorms OuterInstance; + + public CustomNormEncodingSimilarity(TestNorms outerInstance) + { + this.OuterInstance = outerInstance; + } + + public override long EncodeNormValue(float f) + { + return (long)f; + } + + public override float DecodeNormValue(long norm) + { + return norm; + } + + public override float LengthNorm(FieldInvertState state) + { + return state.Length; + } + + public override float Coord(int overlap, int maxOverlap) + { + return 0; + } + + public override float QueryNorm(float sumOfSquaredWeights) + { + return 0; + } + + public override float Tf(float freq) + { + return 0; + } + + public override float Idf(long docFreq, long numDocs) + { + return 0; + } + + public override float SloppyFreq(int distance) + { + return 0; + } + + public override float ScorePayload(int doc, int start, int end, BytesRef payload) + { + return 0; + } + } + + // LUCENE-1260 + [Test] + public virtual void TestCustomEncoder() + { + Directory dir = NewDirectory(); + MockAnalyzer analyzer = new MockAnalyzer(Random()); + + IndexWriterConfig config = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); + config.SetSimilarity(new CustomNormEncodingSimilarity(this)); + RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, config); + Document doc = new Document(); + Field foo = NewTextField("foo", "", Field.Store.NO); + Field bar = NewTextField("bar", "", Field.Store.NO); + doc.Add(foo); + doc.Add(bar); + + for (int i = 0; i < 100; i++) + { + bar.SetStringValue("singleton"); + writer.AddDocument(doc); + } + + IndexReader reader = writer.Reader; + writer.Dispose(); + + NumericDocValues fooNorms = MultiDocValues.GetNormValues(reader, "foo"); + for (int i = 0; i < reader.MaxDoc; i++) + { + Assert.AreEqual(0, fooNorms.Get(i)); + } + + NumericDocValues barNorms = MultiDocValues.GetNormValues(reader, "bar"); + for (int i = 0; i < reader.MaxDoc; i++) + { + Assert.AreEqual(1, barNorms.Get(i)); + } + + reader.Dispose(); + dir.Dispose(); + } + + [Test] + public virtual void TestMaxByteNorms() + { + Directory dir = NewFSDirectory(CreateTempDir("TestNorms.testMaxByteNorms")); + BuildIndex(dir); + AtomicReader open = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir)); + NumericDocValues normValues = open.GetNormValues(ByteTestField); + Assert.IsNotNull(normValues); + for (int i = 0; i < open.MaxDoc; i++) + { + Document document = open.Document(i); + int expected = Convert.ToInt32(document.Get(ByteTestField)); + Assert.AreEqual(expected, normValues.Get(i) & 0xff); + } + open.Dispose(); + dir.Dispose(); + } + + // TODO: create a testNormsNotPresent ourselves by adding/deleting/merging docs + + public virtual void BuildIndex(Directory dir) + { + Random random = Random(); + MockAnalyzer analyzer = new MockAnalyzer(Random()); + analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH); + IndexWriterConfig config = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); + Similarity provider = new MySimProvider(this); + config.SetSimilarity(provider); + RandomIndexWriter writer = new RandomIndexWriter(random, dir, config); + LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues()); + int num = AtLeast(100); + for (int i = 0; i < num; i++) + { + Document doc = docs.NextDoc(); + int boost = Random().Next(255); + Field f = new TextField(ByteTestField, "" + boost, Field.Store.YES); + f.Boost = boost; + doc.Add(f); + writer.AddDocument(doc); + doc.RemoveField(ByteTestField); + if (Rarely()) + { + writer.Commit(); + } + } + writer.Commit(); + writer.Dispose(); + docs.Dispose(); + } + + public class MySimProvider : PerFieldSimilarityWrapper + { + private readonly TestNorms OuterInstance; + + public MySimProvider(TestNorms outerInstance) + { + this.OuterInstance = outerInstance; + } + + internal Similarity @delegate = new DefaultSimilarity(); + + public override float QueryNorm(float sumOfSquaredWeights) + { + return @delegate.QueryNorm(sumOfSquaredWeights); + } + + public override Similarity Get(string field) + { + if (OuterInstance.ByteTestField.Equals(field)) + { + return new ByteEncodingBoostSimilarity(); + } + else + { + return @delegate; + } + } + + public override float Coord(int overlap, int maxOverlap) + { + return @delegate.Coord(overlap, maxOverlap); + } + } + + public class ByteEncodingBoostSimilarity : Similarity + { + public override long ComputeNorm(FieldInvertState state) + { + int boost = (int)state.Boost; + return (sbyte)boost; + } + + public override SimWeight ComputeWeight(float queryBoost, CollectionStatistics collectionStats, params TermStatistics[] termStats) + { + throw new System.NotSupportedException(); + } + + public override SimScorer GetSimScorer(SimWeight weight, AtomicReaderContext context) + { + throw new System.NotSupportedException(); + } + } + } +} \ No newline at end of file
