http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestDocumentsWriterDeleteQueue.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestDocumentsWriterDeleteQueue.cs b/src/Lucene.Net.Tests/Index/TestDocumentsWriterDeleteQueue.cs new file mode 100644 index 0000000..b8de01b --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestDocumentsWriterDeleteQueue.cs @@ -0,0 +1,302 @@ +using Lucene.Net.Search; +using Lucene.Net.Support; +using NUnit.Framework; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; + +namespace Lucene.Net.Index +{ + using BytesRef = Lucene.Net.Util.BytesRef; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + using DeleteSlice = Lucene.Net.Index.DocumentsWriterDeleteQueue.DeleteSlice; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + using TermQuery = Lucene.Net.Search.TermQuery; + + /// <summary> + /// Unit test for <seealso cref="DocumentsWriterDeleteQueue"/> + /// </summary> + [TestFixture] + public class TestDocumentsWriterDeleteQueue : LuceneTestCase + { + [Test] + public virtual void TestUpdateDelteSlices() + { + DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); + int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER; + int?[] ids = new int?[size]; + for (int i = 0; i < ids.Length; i++) + { + ids[i] = Random().Next(); + } + DeleteSlice slice1 = queue.NewSlice(); + DeleteSlice slice2 = queue.NewSlice(); + BufferedUpdates bd1 = new BufferedUpdates(); + BufferedUpdates bd2 = new BufferedUpdates(); + int last1 = 0; + int last2 = 0; + HashSet<Term> uniqueValues = new HashSet<Term>(); + for (int j = 0; j < ids.Length; j++) + { + int? i = ids[j]; + // create an array here since we compare identity below against tailItem + Term[] term = new Term[] { new Term("id", i.ToString()) }; + uniqueValues.Add(term[0]); + queue.AddDelete(term); + if (Random().Next(20) == 0 || j == ids.Length - 1) + { + queue.UpdateSlice(slice1); + Assert.IsTrue(slice1.IsTailItem(term)); + slice1.Apply(bd1, j); + AssertAllBetween(last1, j, bd1, ids); + last1 = j + 1; + } + if (Random().Next(10) == 5 || j == ids.Length - 1) + { + queue.UpdateSlice(slice2); + Assert.IsTrue(slice2.IsTailItem(term)); + slice2.Apply(bd2, j); + AssertAllBetween(last2, j, bd2, ids); + last2 = j + 1; + } + Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes); + } + assertEquals(uniqueValues, new HashSet<Term>(bd1.terms.Keys)); + assertEquals(uniqueValues, new HashSet<Term>(bd2.terms.Keys)); + var frozenSet = new HashSet<Term>(); + foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable()) + { + BytesRef bytesRef = new BytesRef(); + bytesRef.CopyBytes(t.Bytes); + frozenSet.Add(new Term(t.Field, bytesRef)); + } + assertEquals(uniqueValues, frozenSet); + Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze"); + } + + private void AssertAllBetween(int start, int end, BufferedUpdates deletes, int?[] ids) + { + for (int i = start; i <= end; i++) + { + Assert.AreEqual(Convert.ToInt32(end), deletes.terms[new Term("id", ids[i].ToString())]); + } + } + + [Test] + public virtual void TestClear() + { + DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); + Assert.IsFalse(queue.AnyChanges()); + queue.Clear(); + Assert.IsFalse(queue.AnyChanges()); + int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER; + int termsSinceFreeze = 0; + int queriesSinceFreeze = 0; + for (int i = 0; i < size; i++) + { + Term term = new Term("id", "" + i); + if (Random().Next(10) == 0) + { + queue.AddDelete(new TermQuery(term)); + queriesSinceFreeze++; + } + else + { + queue.AddDelete(term); + termsSinceFreeze++; + } + Assert.IsTrue(queue.AnyChanges()); + if (Random().Next(10) == 0) + { + queue.Clear(); + queue.TryApplyGlobalSlice(); + Assert.IsFalse(queue.AnyChanges()); + } + } + } + + [Test] + public virtual void TestAnyChanges() + { + DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); + int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER; + int termsSinceFreeze = 0; + int queriesSinceFreeze = 0; + for (int i = 0; i < size; i++) + { + Term term = new Term("id", "" + i); + if (Random().Next(10) == 0) + { + queue.AddDelete(new TermQuery(term)); + queriesSinceFreeze++; + } + else + { + queue.AddDelete(term); + termsSinceFreeze++; + } + Assert.IsTrue(queue.AnyChanges()); + if (Random().Next(5) == 0) + { + FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null); + Assert.AreEqual(termsSinceFreeze, freezeGlobalBuffer.termCount); + Assert.AreEqual(queriesSinceFreeze, ((Query[])freezeGlobalBuffer.queries.Clone()).Length); + queriesSinceFreeze = 0; + termsSinceFreeze = 0; + Assert.IsFalse(queue.AnyChanges()); + } + } + } + + [Test] + public virtual void TestPartiallyAppliedGlobalSlice() + { + DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); + System.Reflection.FieldInfo field = typeof(DocumentsWriterDeleteQueue).GetField("globalBufferLock", + BindingFlags.NonPublic | BindingFlags.GetField | BindingFlags.Instance); + ReentrantLock @lock = (ReentrantLock)field.GetValue(queue); + @lock.Lock(); + ThreadClass t = new ThreadAnonymousInnerClassHelper(this, queue); + t.Start(); + t.Join(); + @lock.Unlock(); + Assert.IsTrue(queue.AnyChanges(), "changes in del queue but not in slice yet"); + queue.TryApplyGlobalSlice(); + Assert.IsTrue(queue.AnyChanges(), "changes in global buffer"); + FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null); + Assert.IsTrue(freezeGlobalBuffer.Any()); + Assert.AreEqual(1, freezeGlobalBuffer.termCount); + Assert.IsFalse(queue.AnyChanges(), "all changes applied"); + } + + private class ThreadAnonymousInnerClassHelper : ThreadClass + { + private readonly TestDocumentsWriterDeleteQueue OuterInstance; + + private DocumentsWriterDeleteQueue Queue; + + public ThreadAnonymousInnerClassHelper(TestDocumentsWriterDeleteQueue outerInstance, DocumentsWriterDeleteQueue queue) + { + this.OuterInstance = outerInstance; + this.Queue = queue; + } + + public override void Run() + { + Queue.AddDelete(new Term("foo", "bar")); + } + } + + [Test] + public virtual void TestStressDeleteQueue() + { + DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); + HashSet<Term> uniqueValues = new HashSet<Term>(); + int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER; + int?[] ids = new int?[size]; + for (int i = 0; i < ids.Length; i++) + { + ids[i] = Random().Next(); + uniqueValues.Add(new Term("id", ids[i].ToString())); + } + CountdownEvent latch = new CountdownEvent(1); + AtomicInt32 index = new AtomicInt32(0); + int numThreads = 2 + Random().Next(5); + UpdateThread[] threads = new UpdateThread[numThreads]; + for (int i = 0; i < threads.Length; i++) + { + threads[i] = new UpdateThread(queue, index, ids, latch); + threads[i].Start(); + } + latch.Signal(); + for (int i = 0; i < threads.Length; i++) + { + threads[i].Join(); + } + + foreach (UpdateThread updateThread in threads) + { + DeleteSlice slice = updateThread.Slice; + queue.UpdateSlice(slice); + BufferedUpdates deletes = updateThread.Deletes; + slice.Apply(deletes, BufferedUpdates.MAX_INT32); + assertEquals(uniqueValues, new HashSet<Term>(deletes.terms.Keys)); + } + queue.TryApplyGlobalSlice(); + HashSet<Term> frozenSet = new HashSet<Term>(); + foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable()) + { + BytesRef bytesRef = new BytesRef(); + bytesRef.CopyBytes(t.Bytes); + frozenSet.Add(new Term(t.Field, bytesRef)); + } + Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze"); + Assert.AreEqual(uniqueValues.Count, frozenSet.Count); + assertEquals(uniqueValues, frozenSet); + } + + private class UpdateThread : ThreadClass + { + internal readonly DocumentsWriterDeleteQueue Queue; + internal readonly AtomicInt32 Index; + internal readonly int?[] Ids; + internal readonly DeleteSlice Slice; + internal readonly BufferedUpdates Deletes; + internal readonly CountdownEvent Latch; + + protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInt32 index, int?[] ids, CountdownEvent latch) + { + this.Queue = queue; + this.Index = index; + this.Ids = ids; + this.Slice = queue.NewSlice(); + Deletes = new BufferedUpdates(); + this.Latch = latch; + } + + public override void Run() + { +#if !NETSTANDARD + try + { +#endif + Latch.Wait(); +#if !NETSTANDARD + } + catch (ThreadInterruptedException e) + { + throw new ThreadInterruptedException("Thread Interrupted Exception", e); + } +#endif + + int i = 0; + while ((i = Index.GetAndIncrement()) < Ids.Length) + { + Term term = new Term("id", Ids[i].ToString()); + Queue.Add(term, Slice); + Assert.IsTrue(Slice.IsTailItem(term)); + Slice.Apply(Deletes, BufferedUpdates.MAX_INT32); + } + } + } + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestDocumentsWriterStallControl.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestDocumentsWriterStallControl.cs b/src/Lucene.Net.Tests/Index/TestDocumentsWriterStallControl.cs new file mode 100644 index 0000000..9459fca --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestDocumentsWriterStallControl.cs @@ -0,0 +1,473 @@ +using System; +using System.Collections.Generic; +using System.Threading; + +namespace Lucene.Net.Index +{ + using Lucene.Net.Randomized.Generators; + using Lucene.Net.Support; + using NUnit.Framework; + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /// <summary> + /// Tests for <seealso cref="DocumentsWriterStallControl"/> + /// </summary> + [TestFixture] + public class TestDocumentsWriterStallControl : LuceneTestCase + { + [Test] + public virtual void TestSimpleStall() + { + DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl(); + + ctrl.UpdateStalled(false); + ThreadClass[] waitThreads = WaitThreads(AtLeast(1), ctrl); + Start(waitThreads); + Assert.IsFalse(ctrl.HasBlocked); + Assert.IsFalse(ctrl.AnyStalledThreads()); + Join(waitThreads); + + // now stall threads and wake them up again + ctrl.UpdateStalled(true); + waitThreads = WaitThreads(AtLeast(1), ctrl); + Start(waitThreads); + AwaitState(ThreadState.WaitSleepJoin, waitThreads); + Assert.IsTrue(ctrl.HasBlocked); + Assert.IsTrue(ctrl.AnyStalledThreads()); + ctrl.UpdateStalled(false); + Assert.IsFalse(ctrl.AnyStalledThreads()); + Join(waitThreads); + } + + [Test] + public virtual void TestRandom() + { + DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl(); + ctrl.UpdateStalled(false); + + ThreadClass[] stallThreads = new ThreadClass[AtLeast(3)]; + for (int i = 0; i < stallThreads.Length; i++) + { + int stallProbability = 1 + Random().Next(10); + stallThreads[i] = new ThreadAnonymousInnerClassHelper(ctrl, stallProbability); + } + Start(stallThreads); + long time = Environment.TickCount; + /* + * use a 100 sec timeout to make sure we not hang forever. join will fail in + * that case + */ + while ((Environment.TickCount - time) < 100 * 1000 && !Terminated(stallThreads)) + { + ctrl.UpdateStalled(false); + if (Random().NextBoolean()) + { + Thread.Sleep(0); + } + else + { + Thread.Sleep(1); + } + } + Join(stallThreads); + } + + private class ThreadAnonymousInnerClassHelper : ThreadClass + { + private DocumentsWriterStallControl Ctrl; + private int StallProbability; + + public ThreadAnonymousInnerClassHelper(DocumentsWriterStallControl ctrl, int stallProbability) + { + this.Ctrl = ctrl; + this.StallProbability = stallProbability; + } + + public override void Run() + { + int iters = AtLeast(1000); + for (int j = 0; j < iters; j++) + { + Ctrl.UpdateStalled(Random().Next(StallProbability) == 0); + if (Random().Next(5) == 0) // thread 0 only updates + { + Ctrl.WaitIfStalled(); + } + } + } + } + + [Test] + public virtual void TestAccquireReleaseRace() + { + DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl(); + ctrl.UpdateStalled(false); + AtomicBoolean stop = new AtomicBoolean(false); + AtomicBoolean checkPoint = new AtomicBoolean(true); + + int numStallers = AtLeast(1); + int numReleasers = AtLeast(1); + int numWaiters = AtLeast(1); + var sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers + numWaiters); + var threads = new ThreadClass[numReleasers + numStallers + numWaiters]; + IList<Exception> exceptions = new SynchronizedList<Exception>(); + for (int i = 0; i < numReleasers; i++) + { + threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions); + } + for (int i = numReleasers; i < numReleasers + numStallers; i++) + { + threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions); + } + for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++) + { + threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions); + } + + Start(threads); + int iters = AtLeast(10000); + float checkPointProbability = TEST_NIGHTLY ? 0.5f : 0.1f; + for (int i = 0; i < iters; i++) + { + if (checkPoint.Get()) + { + Assert.IsTrue(sync.UpdateJoin.Wait(new TimeSpan(0, 0, 0, 10)), "timed out waiting for update threads - deadlock?"); + if (exceptions.Count > 0) + { + foreach (Exception throwable in exceptions) + { + Console.WriteLine(throwable.ToString()); + Console.Write(throwable.StackTrace); + } + Assert.Fail("got exceptions in threads"); + } + + if (ctrl.HasBlocked && ctrl.IsHealthy) + { + AssertState(numReleasers, numStallers, numWaiters, threads, ctrl); + } + + checkPoint.Set(false); + sync.Waiter.Signal(); + sync.LeftCheckpoint.Wait(); + } + Assert.IsFalse(checkPoint.Get()); + Assert.AreEqual(0, sync.Waiter.CurrentCount); + if (checkPointProbability >= (float)Random().NextDouble()) + { + sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters); + checkPoint.Set(true); + } + } + if (!checkPoint.Get()) + { + sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters); + checkPoint.Set(true); + } + + Assert.IsTrue(sync.UpdateJoin.Wait(new TimeSpan(0, 0, 0, 10))); + AssertState(numReleasers, numStallers, numWaiters, threads, ctrl); + checkPoint.Set(false); + stop.Set(true); + sync.Waiter.Signal(); + sync.LeftCheckpoint.Wait(); + + for (int i = 0; i < threads.Length; i++) + { + ctrl.UpdateStalled(false); + threads[i].Join(2000); + if (threads[i].IsAlive && threads[i] is Waiter) + { + if (threads[i].State == ThreadState.WaitSleepJoin) + { + Assert.Fail("waiter is not released - anyThreadsStalled: " + ctrl.AnyStalledThreads()); + } + } + } + } + + private void AssertState(int numReleasers, int numStallers, int numWaiters, ThreadClass[] threads, DocumentsWriterStallControl ctrl) + { + int millisToSleep = 100; + while (true) + { + if (ctrl.HasBlocked && ctrl.IsHealthy) + { + for (int n = numReleasers + numStallers; n < numReleasers + numStallers + numWaiters; n++) + { + if (ctrl.IsThreadQueued(threads[n])) + { + if (millisToSleep < 60000) + { + Thread.Sleep(millisToSleep); + millisToSleep *= 2; + break; + } + else + { + Assert.Fail("control claims no stalled threads but waiter seems to be blocked "); + } + } + } + break; + } + else + { + break; + } + } + } + + internal class Waiter : ThreadClass + { + internal Synchronizer Sync; + internal DocumentsWriterStallControl Ctrl; + internal AtomicBoolean CheckPoint; + internal AtomicBoolean Stop; + internal IList<Exception> Exceptions; + + public Waiter(AtomicBoolean stop, AtomicBoolean checkPoint, DocumentsWriterStallControl ctrl, Synchronizer sync, IList<Exception> exceptions) + : base("waiter") + { + this.Stop = stop; + this.CheckPoint = checkPoint; + this.Ctrl = ctrl; + this.Sync = sync; + this.Exceptions = exceptions; + } + + public override void Run() + { + try + { + while (!Stop.Get()) + { + Ctrl.WaitIfStalled(); + if (CheckPoint.Get()) + { +#if !NETSTANDARD + try + { +#endif + Assert.IsTrue(Sync.await()); +#if !NETSTANDARD + } + catch (ThreadInterruptedException e) + { + Console.WriteLine("[Waiter] got interrupted - wait count: " + Sync.Waiter.CurrentCount); + throw new ThreadInterruptedException("Thread Interrupted Exception", e); + } +#endif + } + } + } + catch (Exception e) + { + Console.WriteLine(e.ToString()); + Console.Write(e.StackTrace); + Exceptions.Add(e); + } + } + } + + internal class Updater : ThreadClass + { + internal Synchronizer Sync; + internal DocumentsWriterStallControl Ctrl; + internal AtomicBoolean CheckPoint; + internal AtomicBoolean Stop; + internal bool Release; + internal IList<Exception> Exceptions; + + public Updater(AtomicBoolean stop, AtomicBoolean checkPoint, DocumentsWriterStallControl ctrl, Synchronizer sync, bool release, IList<Exception> exceptions) + : base("updater") + { + this.Stop = stop; + this.CheckPoint = checkPoint; + this.Ctrl = ctrl; + this.Sync = sync; + this.Release = release; + this.Exceptions = exceptions; + } + + public override void Run() + { + try + { + while (!Stop.Get()) + { + int internalIters = Release && Random().NextBoolean() ? AtLeast(5) : 1; + for (int i = 0; i < internalIters; i++) + { + Ctrl.UpdateStalled(Random().NextBoolean()); + } + if (CheckPoint.Get()) + { + Sync.UpdateJoin.Signal(); + try + { + Assert.IsTrue(Sync.await()); + } +#if !NETSTANDARD + catch (ThreadInterruptedException e) + { + Console.WriteLine("[Updater] got interrupted - wait count: " + Sync.Waiter.CurrentCount); + throw new ThreadInterruptedException("Thread Interrupted Exception", e); + } +#endif + catch (Exception e) + { + Console.Write("signal failed with : " + e); + throw e; + } + + Sync.LeftCheckpoint.Signal(); + } + if (Random().NextBoolean()) + { + Thread.Sleep(0); + } + } + } + catch (Exception e) + { + Console.WriteLine(e.ToString()); + Console.Write(e.StackTrace); + Exceptions.Add(e); + } + + if (!Sync.UpdateJoin.IsSet) + { + Sync.UpdateJoin.Signal(); + } + } + } + + public static bool Terminated(ThreadClass[] threads) + { + foreach (ThreadClass thread in threads) + { + if (ThreadState.Stopped != thread.State) + { + return false; + } + } + return true; + } + + public static void Start(ThreadClass[] tostart) + { + foreach (ThreadClass thread in tostart) + { + thread.Start(); + } + Thread.Sleep(1); // let them start + } + + public static void Join(ThreadClass[] toJoin) + { + foreach (ThreadClass thread in toJoin) + { + thread.Join(); + } + } + + internal static ThreadClass[] WaitThreads(int num, DocumentsWriterStallControl ctrl) + { + ThreadClass[] array = new ThreadClass[num]; + for (int i = 0; i < array.Length; i++) + { + array[i] = new ThreadAnonymousInnerClassHelper2(ctrl); + } + return array; + } + + private class ThreadAnonymousInnerClassHelper2 : ThreadClass + { + private DocumentsWriterStallControl Ctrl; + + public ThreadAnonymousInnerClassHelper2(DocumentsWriterStallControl ctrl) + { + this.Ctrl = ctrl; + } + + public override void Run() + { + Ctrl.WaitIfStalled(); + } + } + + /// <summary> + /// Waits for all incoming threads to be in wait() + /// methods. + /// </summary> + public static void AwaitState(ThreadState state, params ThreadClass[] threads) + { + while (true) + { + bool done = true; + foreach (ThreadClass thread in threads) + { + if (thread.State != state) + { + done = false; + break; + } + } + if (done) + { + return; + } + if (Random().NextBoolean()) + { + Thread.Sleep(0); + } + else + { + Thread.Sleep(1); + } + } + } + + public sealed class Synchronizer + { + internal volatile CountdownEvent Waiter; + internal volatile CountdownEvent UpdateJoin; + internal volatile CountdownEvent LeftCheckpoint; + + public Synchronizer(int numUpdater, int numThreads) + { + Reset(numUpdater, numThreads); + } + + public void Reset(int numUpdaters, int numThreads) + { + this.Waiter = new CountdownEvent(1); + this.UpdateJoin = new CountdownEvent(numUpdaters); + this.LeftCheckpoint = new CountdownEvent(numUpdaters); + } + + public bool @await() + { + return Waiter.Wait(new TimeSpan(0, 0, 0, 10)); + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestDuelingCodecs.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestDuelingCodecs.cs b/src/Lucene.Net.Tests/Index/TestDuelingCodecs.cs new file mode 100644 index 0000000..4ac4e65 --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestDuelingCodecs.cs @@ -0,0 +1,183 @@ +using Lucene.Net.Documents; +using NUnit.Framework; +using System; + +namespace Lucene.Net.Index +{ + using System.Text.RegularExpressions; + using Attributes; + using BytesRef = Lucene.Net.Util.BytesRef; + using Codec = Lucene.Net.Codecs.Codec; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using LineFileDocs = Lucene.Net.Util.LineFileDocs; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using NumericDocValuesField = NumericDocValuesField; + using SortedSetDocValuesField = SortedSetDocValuesField; + using TestUtil = Lucene.Net.Util.TestUtil; + + /// <summary> + /// Compares one codec against another + /// </summary> + [TestFixture] + public class TestDuelingCodecs : LuceneTestCase + { + private Directory LeftDir; + private IndexReader LeftReader; + private Codec LeftCodec; + + private Directory RightDir; + private IndexReader RightReader; + private Codec RightCodec; + + private string Info; // for debugging + + [SetUp] + public override void SetUp() + { + base.SetUp(); + + // for now its SimpleText vs Lucene46(random postings format) + // as this gives the best overall coverage. when we have more + // codecs we should probably pick 2 from Codec.availableCodecs() + + LeftCodec = Codec.ForName("SimpleText"); + RightCodec = new RandomCodec(Random()); + + LeftDir = NewDirectory(); + RightDir = NewDirectory(); + + long seed = Random().Next(); + + // must use same seed because of random payloads, etc + int maxTermLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH); + MockAnalyzer leftAnalyzer = new MockAnalyzer(new Random((int)seed)); + leftAnalyzer.MaxTokenLength = maxTermLength; + MockAnalyzer rightAnalyzer = new MockAnalyzer(new Random((int)seed)); + rightAnalyzer.MaxTokenLength = maxTermLength; + + // but these can be different + // TODO: this turns this into a really big test of Multi*, is that what we want? + IndexWriterConfig leftConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, leftAnalyzer); + leftConfig.SetCodec(LeftCodec); + // preserve docids + leftConfig.SetMergePolicy(NewLogMergePolicy()); + + IndexWriterConfig rightConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, rightAnalyzer); + rightConfig.SetCodec(RightCodec); + // preserve docids + rightConfig.SetMergePolicy(NewLogMergePolicy()); + + // must use same seed because of random docvalues fields, etc + RandomIndexWriter leftWriter = new RandomIndexWriter(new Random((int)seed), LeftDir, leftConfig); + RandomIndexWriter rightWriter = new RandomIndexWriter(new Random((int)seed), RightDir, rightConfig); + + int numdocs = AtLeast(100); + CreateRandomIndex(numdocs, leftWriter, seed); + CreateRandomIndex(numdocs, rightWriter, seed); + + LeftReader = MaybeWrapReader(leftWriter.Reader); + leftWriter.Dispose(); + RightReader = MaybeWrapReader(rightWriter.Reader); + rightWriter.Dispose(); + + // check that our readers are valid + TestUtil.CheckReader(LeftReader); + TestUtil.CheckReader(RightReader); + + Info = "left: " + LeftCodec.ToString() + " / right: " + RightCodec.ToString(); + } + + [TearDown] + public override void TearDown() + { + if (LeftReader != null) + { + LeftReader.Dispose(); + } + if (RightReader != null) + { + RightReader.Dispose(); + } + + if (LeftDir != null) + { + LeftDir.Dispose(); + } + if (RightDir != null) + { + RightDir.Dispose(); + } + + base.TearDown(); + } + + /// <summary> + /// populates a writer with random stuff. this must be fully reproducable with the seed! + /// </summary> + public static void CreateRandomIndex(int numdocs, RandomIndexWriter writer, long seed) + { + Random random = new Random((int)seed); + // primary source for our data is from linefiledocs, its realistic. + LineFileDocs lineFileDocs = new LineFileDocs(random); + + // LUCENENET: compile a regex so we don't have to do it in each loop (for regex.split()) + Regex whiteSpace = new Regex("\\s+", RegexOptions.Compiled); + + // TODO: we should add other fields that use things like docs&freqs but omit positions, + // because linefiledocs doesn't cover all the possibilities. + for (int i = 0; i < numdocs; i++) + { + Document document = lineFileDocs.NextDoc(); + // grab the title and add some SortedSet instances for fun + string title = document.Get("titleTokenized"); + string[] split = whiteSpace.Split(title); + foreach (string trash in split) + { + document.Add(new SortedSetDocValuesField("sortedset", new BytesRef(trash))); + } + // add a numeric dv field sometimes + document.RemoveFields("sparsenumeric"); + if (random.Next(4) == 2) + { + document.Add(new NumericDocValuesField("sparsenumeric", random.Next())); + } + writer.AddDocument(document); + } + + lineFileDocs.Dispose(); + } + + /// <summary> + /// checks the two indexes are equivalent + /// </summary> +#if !NETSTANDARD + // LUCENENET: There is no Timeout on NUnit for .NET Core. + [Timeout(120000)] +#endif + [Test, HasTimeout] + public virtual void TestEquals() + { + AssertReaderEquals(Info, LeftReader, RightReader); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestExceedMaxTermLength.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestExceedMaxTermLength.cs b/src/Lucene.Net.Tests/Index/TestExceedMaxTermLength.cs new file mode 100644 index 0000000..aa3470d --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestExceedMaxTermLength.cs @@ -0,0 +1,108 @@ +using System; +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using Lucene.Net.Randomized.Generators; + using NUnit.Framework; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using Field = Field; + using FieldType = FieldType; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using TestUtil = Lucene.Net.Util.TestUtil; + + /// <summary> + /// Tests that a useful exception is thrown when attempting to index a term that is + /// too large + /// </summary> + /// <seealso cref= IndexWriter#MAX_TERM_LENGTH </seealso> + [TestFixture] + public class TestExceedMaxTermLength : LuceneTestCase + { + private static readonly int MinTestTermLength = IndexWriter.MAX_TERM_LENGTH + 1; + private static readonly int MaxTestTermLegnth = IndexWriter.MAX_TERM_LENGTH * 2; + + internal Directory Dir = null; + + [SetUp] + public virtual void CreateDir() + { + Dir = NewDirectory(); + } + + [TearDown] + public virtual void DestroyDir() + { + Dir.Dispose(); + Dir = null; + } + + [Test] + public virtual void Test() + { + IndexWriter w = new IndexWriter(Dir, NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + try + { + FieldType ft = new FieldType(); + ft.IsIndexed = true; + ft.IsStored = Random().NextBoolean(); + ft.Freeze(); + + Document doc = new Document(); + if (Random().NextBoolean()) + { + // totally ok short field value + doc.Add(new Field(TestUtil.RandomSimpleString(Random(), 1, 10), TestUtil.RandomSimpleString(Random(), 1, 10), ft)); + } + // problematic field + string name = TestUtil.RandomSimpleString(Random(), 1, 50); + string value = TestUtil.RandomSimpleString(Random(), MinTestTermLength, MaxTestTermLegnth); + Field f = new Field(name, value, ft); + if (Random().NextBoolean()) + { + // totally ok short field value + doc.Add(new Field(TestUtil.RandomSimpleString(Random(), 1, 10), TestUtil.RandomSimpleString(Random(), 1, 10), ft)); + } + doc.Add(f); + + try + { + w.AddDocument(doc); + Assert.Fail("Did not get an exception from adding a monster term"); + } + catch (System.ArgumentException e) + { + string maxLengthMsg = Convert.ToString(IndexWriter.MAX_TERM_LENGTH); + string msg = e.Message; + Assert.IsTrue(msg.Contains("immense term"), "IllegalArgumentException didn't mention 'immense term': " + msg); + Assert.IsTrue(msg.Contains(maxLengthMsg), "IllegalArgumentException didn't mention max length (" + maxLengthMsg + "): " + msg); + Assert.IsTrue(msg.Contains(name), "IllegalArgumentException didn't mention field name (" + name + "): " + msg); + } + } + finally + { + w.Dispose(); + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestFieldInfos.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestFieldInfos.cs b/src/Lucene.Net.Tests/Index/TestFieldInfos.cs new file mode 100644 index 0000000..35f8d8a --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestFieldInfos.cs @@ -0,0 +1,126 @@ +namespace Lucene.Net.Index +{ + using NUnit.Framework; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using Codec = Lucene.Net.Codecs.Codec; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using FieldInfosReader = Lucene.Net.Codecs.FieldInfosReader; + using FieldInfosWriter = Lucene.Net.Codecs.FieldInfosWriter; + using IndexOutput = Lucene.Net.Store.IndexOutput; + using IOContext = Lucene.Net.Store.IOContext; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + //import org.cnlp.utils.properties.ResourceBundleHelper; + + [TestFixture] + public class TestFieldInfos : LuceneTestCase + { + private Document TestDoc = new Document(); + + [SetUp] + public override void SetUp() + { + base.SetUp(); + DocHelper.SetupDoc(TestDoc); + } + + public virtual FieldInfos CreateAndWriteFieldInfos(Directory dir, string filename) + { + //Positive test of FieldInfos + Assert.IsTrue(TestDoc != null); + FieldInfos.Builder builder = new FieldInfos.Builder(); + foreach (IIndexableField field in TestDoc) + { + builder.AddOrUpdate(field.Name, field.FieldType); + } + FieldInfos fieldInfos = builder.Finish(); + //Since the complement is stored as well in the fields map + Assert.IsTrue(fieldInfos.Count == DocHelper.All.Count); //this is all b/c we are using the no-arg constructor + + IndexOutput output = dir.CreateOutput(filename, NewIOContext(Random())); + Assert.IsTrue(output != null); + //Use a RAMOutputStream + + FieldInfosWriter writer = Codec.Default.FieldInfosFormat.FieldInfosWriter; + writer.Write(dir, filename, "", fieldInfos, IOContext.DEFAULT); + output.Dispose(); + return fieldInfos; + } + + public virtual FieldInfos ReadFieldInfos(Directory dir, string filename) + { + FieldInfosReader reader = Codec.Default.FieldInfosFormat.FieldInfosReader; + return reader.Read(dir, filename, "", IOContext.DEFAULT); + } + + [Test] + public virtual void Test() + { + string name = "testFile"; + Directory dir = NewDirectory(); + FieldInfos fieldInfos = CreateAndWriteFieldInfos(dir, name); + + FieldInfos readIn = ReadFieldInfos(dir, name); + Assert.IsTrue(fieldInfos.Count == readIn.Count); + FieldInfo info = readIn.FieldInfo("textField1"); + Assert.IsTrue(info != null); + Assert.IsTrue(info.HasVectors == false); + Assert.IsTrue(info.OmitsNorms == false); + + info = readIn.FieldInfo("textField2"); + Assert.IsTrue(info != null); + Assert.IsTrue(info.OmitsNorms == false); + + info = readIn.FieldInfo("textField3"); + Assert.IsTrue(info != null); + Assert.IsTrue(info.HasVectors == false); + Assert.IsTrue(info.OmitsNorms == true); + + info = readIn.FieldInfo("omitNorms"); + Assert.IsTrue(info != null); + Assert.IsTrue(info.HasVectors == false); + Assert.IsTrue(info.OmitsNorms == true); + + dir.Dispose(); + } + + [Test] + public virtual void TestReadOnly() + { + string name = "testFile"; + Directory dir = NewDirectory(); + FieldInfos fieldInfos = CreateAndWriteFieldInfos(dir, name); + FieldInfos readOnly = ReadFieldInfos(dir, name); + AssertReadOnly(readOnly, fieldInfos); + dir.Dispose(); + } + + private void AssertReadOnly(FieldInfos readOnly, FieldInfos modifiable) + { + Assert.AreEqual(modifiable.Count, readOnly.Count); + // assert we can iterate + foreach (FieldInfo fi in readOnly) + { + Assert.AreEqual(fi.Name, modifiable.FieldInfo(fi.Number).Name); + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestFieldsReader.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestFieldsReader.cs b/src/Lucene.Net.Tests/Index/TestFieldsReader.cs new file mode 100644 index 0000000..e539561 --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestFieldsReader.cs @@ -0,0 +1,286 @@ +using System; +using System.Collections.Generic; +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using NUnit.Framework; + using System.IO; + using BaseDirectory = Lucene.Net.Store.BaseDirectory; + using BufferedIndexInput = Lucene.Net.Store.BufferedIndexInput; + using Directory = Lucene.Net.Store.Directory; + using Document = Documents.Document; + using DocumentStoredFieldVisitor = DocumentStoredFieldVisitor; + using Field = Field; + using IndexInput = Lucene.Net.Store.IndexInput; + using IndexOutput = Lucene.Net.Store.IndexOutput; + using IOContext = Lucene.Net.Store.IOContext; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + using TestUtil = Lucene.Net.Util.TestUtil; + + [TestFixture] + public class TestFieldsReader : LuceneTestCase + { + private static Directory Dir; + private static Document TestDoc; + private static FieldInfos.Builder FieldInfos = null; + + /// <summary> + /// LUCENENET specific + /// Is non-static because NewIndexWriterConfig is no longer static. + /// </summary> + [OneTimeSetUp] + public void BeforeClass() + { + TestDoc = new Document(); + FieldInfos = new FieldInfos.Builder(); + DocHelper.SetupDoc(TestDoc); + foreach (IIndexableField field in TestDoc) + { + FieldInfos.AddOrUpdate(field.Name, field.FieldType); + } + Dir = NewDirectory(); + IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()); + conf.MergePolicy.NoCFSRatio = 0.0; + IndexWriter writer = new IndexWriter(Dir, conf); + writer.AddDocument(TestDoc); + writer.Dispose(); + FaultyIndexInput.DoFail = false; + } + + [OneTimeTearDown] + public static void AfterClass() + { + Dir.Dispose(); + Dir = null; + FieldInfos = null; + TestDoc = null; + } + + [Test] + public virtual void Test() + { + Assert.IsTrue(Dir != null); + Assert.IsTrue(FieldInfos != null); + IndexReader reader = DirectoryReader.Open(Dir); + Document doc = reader.Document(0); + Assert.IsTrue(doc != null); + Assert.IsTrue(doc.GetField(DocHelper.TEXT_FIELD_1_KEY) != null); + + Field field = (Field)doc.GetField(DocHelper.TEXT_FIELD_2_KEY); + Assert.IsTrue(field != null); + Assert.IsTrue(field.FieldType.StoreTermVectors); + + Assert.IsFalse(field.FieldType.OmitNorms); + Assert.IsTrue(field.FieldType.IndexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + + field = (Field)doc.GetField(DocHelper.TEXT_FIELD_3_KEY); + Assert.IsTrue(field != null); + Assert.IsFalse(field.FieldType.StoreTermVectors); + Assert.IsTrue(field.FieldType.OmitNorms); + Assert.IsTrue(field.FieldType.IndexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + + field = (Field)doc.GetField(DocHelper.NO_TF_KEY); + Assert.IsTrue(field != null); + Assert.IsFalse(field.FieldType.StoreTermVectors); + Assert.IsFalse(field.FieldType.OmitNorms); + Assert.IsTrue(field.FieldType.IndexOptions == IndexOptions.DOCS_ONLY); + + DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(DocHelper.TEXT_FIELD_3_KEY); + reader.Document(0, visitor); + IList<IIndexableField> fields = visitor.Document.Fields; + Assert.AreEqual(1, fields.Count); + Assert.AreEqual(DocHelper.TEXT_FIELD_3_KEY, fields[0].Name); + reader.Dispose(); + } + + public class FaultyFSDirectory : BaseDirectory + { + internal Directory FsDir; + + public FaultyFSDirectory(DirectoryInfo dir) + { + FsDir = NewFSDirectory(dir); + m_lockFactory = FsDir.LockFactory; + } + + public override IndexInput OpenInput(string name, IOContext context) + { + return new FaultyIndexInput(FsDir.OpenInput(name, context)); + } + + public override string[] ListAll() + { + return FsDir.ListAll(); + } + + [Obsolete("this method will be removed in 5.0")] + public override bool FileExists(string name) + { + return FsDir.FileExists(name); + } + + public override void DeleteFile(string name) + { + FsDir.DeleteFile(name); + } + + public override long FileLength(string name) + { + return FsDir.FileLength(name); + } + + public override IndexOutput CreateOutput(string name, IOContext context) + { + return FsDir.CreateOutput(name, context); + } + + public override void Sync(ICollection<string> names) + { + FsDir.Sync(names); + } + + public override void Dispose() + { + FsDir.Dispose(); + } + } + + private class FaultyIndexInput : BufferedIndexInput + { + internal IndexInput @delegate; + internal static bool DoFail; + internal int Count; + + internal FaultyIndexInput(IndexInput @delegate) + : base("FaultyIndexInput(" + @delegate + ")", BufferedIndexInput.BUFFER_SIZE) + { + this.@delegate = @delegate; + } + + internal virtual void SimOutage() + { + if (DoFail && Count++ % 2 == 1) + { + throw new IOException("Simulated network outage"); + } + } + + protected override void ReadInternal(byte[] b, int offset, int length) + { + SimOutage(); + @delegate.Seek(FilePointer); + @delegate.ReadBytes(b, offset, length); + } + + protected override void SeekInternal(long pos) + { + } + + public override long Length + { + get { return @delegate.Length; } + } + + public override void Dispose() + { + @delegate.Dispose(); + } + + public override object Clone() + { + FaultyIndexInput i = new FaultyIndexInput((IndexInput)@delegate.Clone()); + // seek the clone to our current position + try + { + i.Seek(FilePointer); + } +#pragma warning disable 168 + catch (IOException e) +#pragma warning restore 168 + { + throw new Exception(); + } + return i; + } + } + + // LUCENE-1262 + [Test] + public virtual void TestExceptions() + { + DirectoryInfo indexDir = CreateTempDir("testfieldswriterexceptions"); + + try + { + Directory dir = new FaultyFSDirectory(indexDir); + IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.CREATE); + IndexWriter writer = new IndexWriter(dir, iwc); + for (int i = 0; i < 2; i++) + { + writer.AddDocument(TestDoc); + } + writer.ForceMerge(1); + writer.Dispose(); + + IndexReader reader = DirectoryReader.Open(dir); + + FaultyIndexInput.DoFail = true; + + bool exc = false; + + for (int i = 0; i < 2; i++) + { + try + { + reader.Document(i); + } +#pragma warning disable 168 + catch (IOException ioe) +#pragma warning restore 168 + { + // expected + exc = true; + } + try + { + reader.Document(i); + } +#pragma warning disable 168 + catch (IOException ioe) +#pragma warning restore 168 + { + // expected + exc = true; + } + } + Assert.IsTrue(exc); + reader.Dispose(); + dir.Dispose(); + } + finally + { + System.IO.Directory.Delete(indexDir.FullName, true); + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestFilterAtomicReader.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestFilterAtomicReader.cs b/src/Lucene.Net.Tests/Index/TestFilterAtomicReader.cs new file mode 100644 index 0000000..f22c0ee --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestFilterAtomicReader.cs @@ -0,0 +1,224 @@ +using System; +using System.Linq; +using System.Reflection; +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using NUnit.Framework; + using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper; + using IBits = Lucene.Net.Util.IBits; + using BytesRef = Lucene.Net.Util.BytesRef; + using Directory = Lucene.Net.Store.Directory; + using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator; + using Document = Documents.Document; + using Field = Field; + using LuceneTestCase = Lucene.Net.Util.LuceneTestCase; + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer; + + [TestFixture] + public class TestFilterAtomicReader : LuceneTestCase + { + private class TestReader : FilterAtomicReader + { + /// <summary> + /// Filter that only permits terms containing 'e'. </summary> + private class TestFields : FilterFields + { + internal TestFields(Fields @in) + : base(@in) + { + } + + public override Terms GetTerms(string field) + { + return new TestTerms(base.GetTerms(field)); + } + } + + private class TestTerms : FilterTerms + { + internal TestTerms(Terms @in) + : base(@in) + { + } + + public override TermsEnum GetIterator(TermsEnum reuse) + { + return new TestTermsEnum(base.GetIterator(reuse)); + } + } + + private class TestTermsEnum : FilterTermsEnum + { + public TestTermsEnum(TermsEnum @in) + : base(@in) + { + } + + /// <summary> + /// Scan for terms containing the letter 'e'. </summary> + public override BytesRef Next() + { + BytesRef text; + while ((text = m_input.Next()) != null) + { + if (text.Utf8ToString().IndexOf('e') != -1) + { + return text; + } + } + return null; + } + + public override DocsAndPositionsEnum DocsAndPositions(IBits liveDocs, DocsAndPositionsEnum reuse, int flags) + { + return new TestPositions(base.DocsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsAndPositionsEnum)reuse).m_input, flags)); + } + } + + /// <summary> + /// Filter that only returns odd numbered documents. </summary> + private class TestPositions : FilterDocsAndPositionsEnum + { + public TestPositions(DocsAndPositionsEnum input) + : base(input) + { + } + + /// <summary> + /// Scan for odd numbered documents. </summary> + public override int NextDoc() + { + int doc; + while ((doc = m_input.NextDoc()) != NO_MORE_DOCS) + { + if ((doc % 2) == 1) + { + return doc; + } + } + return NO_MORE_DOCS; + } + } + + public TestReader(IndexReader reader) + : base(SlowCompositeReaderWrapper.Wrap(reader)) + { + } + + public override Fields Fields + { + get { return new TestFields(base.Fields); } + } + } + + /// <summary> + /// Tests the IndexReader.getFieldNames implementation </summary> + /// <exception cref="Exception"> on error </exception> + [Test] + public virtual void TestFilterIndexReader() + { + Directory directory = NewDirectory(); + + IndexWriter writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + + Document d1 = new Document(); + d1.Add(NewTextField("default", "one two", Field.Store.YES)); + writer.AddDocument(d1); + + Document d2 = new Document(); + d2.Add(NewTextField("default", "one three", Field.Store.YES)); + writer.AddDocument(d2); + + Document d3 = new Document(); + d3.Add(NewTextField("default", "two four", Field.Store.YES)); + writer.AddDocument(d3); + + writer.Dispose(); + + Directory target = NewDirectory(); + + // We mess with the postings so this can fail: + ((BaseDirectoryWrapper)target).CrossCheckTermVectorsOnClose = false; + + writer = new IndexWriter(target, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); + IndexReader reader = new TestReader(DirectoryReader.Open(directory)); + writer.AddIndexes(reader); + writer.Dispose(); + reader.Dispose(); + reader = DirectoryReader.Open(target); + + TermsEnum terms = MultiFields.GetTerms(reader, "default").GetIterator(null); + while (terms.Next() != null) + { + Assert.IsTrue(terms.Term.Utf8ToString().IndexOf('e') != -1); + } + + Assert.AreEqual(TermsEnum.SeekStatus.FOUND, terms.SeekCeil(new BytesRef("one"))); + + DocsAndPositionsEnum positions = terms.DocsAndPositions(MultiFields.GetLiveDocs(reader), null); + while (positions.NextDoc() != DocIdSetIterator.NO_MORE_DOCS) + { + Assert.IsTrue((positions.DocID % 2) == 1); + } + + reader.Dispose(); + directory.Dispose(); + target.Dispose(); + } + + private static void CheckOverrideMethods(Type clazz) + { + Type superClazz = clazz.GetTypeInfo().BaseType; + foreach (MethodInfo m in superClazz.GetMethods()) + { + // LUCENENET specific - since we changed to using a property for Attributes rather than a method, + // we need to reflect that as get_Attributes here. + if (m.IsStatic || m.IsAbstract || m.IsFinal || /*m.Synthetic ||*/ m.Name.Equals("get_Attributes")) + { + continue; + } + // The point of these checks is to ensure that methods that have a default + // impl through other methods are not overridden. this makes the number of + // methods to override to have a working impl minimal and prevents from some + // traps: for example, think about having getCoreCacheKey delegate to the + // filtered impl by default + MethodInfo subM = clazz.GetMethod(m.Name, m.GetParameters().Select(p => p.ParameterType).ToArray()); + if (subM.DeclaringType == clazz && m.DeclaringType != typeof(object) && m.DeclaringType != subM.DeclaringType) + { + Assert.Fail(clazz + " overrides " + m + " although it has a default impl"); + } + } + } + + [Test] + public virtual void TestOverrideMethods() + { + CheckOverrideMethods(typeof(FilterAtomicReader)); + CheckOverrideMethods(typeof(FilterAtomicReader.FilterFields)); + CheckOverrideMethods(typeof(FilterAtomicReader.FilterTerms)); + CheckOverrideMethods(typeof(FilterAtomicReader.FilterTermsEnum)); + CheckOverrideMethods(typeof(FilterAtomicReader.FilterDocsEnum)); + CheckOverrideMethods(typeof(FilterAtomicReader.FilterDocsAndPositionsEnum)); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestFlex.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests/Index/TestFlex.cs b/src/Lucene.Net.Tests/Index/TestFlex.cs new file mode 100644 index 0000000..01cd762 --- /dev/null +++ b/src/Lucene.Net.Tests/Index/TestFlex.cs @@ -0,0 +1,100 @@ +using Lucene.Net.Documents; + +namespace Lucene.Net.Index +{ + using Lucene.Net.Analysis; + + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using Lucene.Net.Store; + using Lucene.Net.Util; + using NUnit.Framework; + using Lucene41PostingsFormat = Lucene.Net.Codecs.Lucene41.Lucene41PostingsFormat; + + [TestFixture] + public class TestFlex : LuceneTestCase + { + // Test non-flex API emulated on flex index + [Test] + public virtual void TestNonFlex() + { + Directory d = NewDirectory(); + + const int DOC_COUNT = 177; + + IndexWriter w = new IndexWriter(d, (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMaxBufferedDocs(7).SetMergePolicy(NewLogMergePolicy())); + + for (int iter = 0; iter < 2; iter++) + { + if (iter == 0) + { + Documents.Document doc = new Documents.Document(); + doc.Add(NewTextField("field1", "this is field1", Field.Store.NO)); + doc.Add(NewTextField("field2", "this is field2", Field.Store.NO)); + doc.Add(NewTextField("field3", "aaa", Field.Store.NO)); + doc.Add(NewTextField("field4", "bbb", Field.Store.NO)); + for (int i = 0; i < DOC_COUNT; i++) + { + w.AddDocument(doc); + } + } + else + { + w.ForceMerge(1); + } + + IndexReader r = w.Reader; + + TermsEnum terms = MultiFields.GetTerms(r, "field3").GetIterator(null); + Assert.AreEqual(TermsEnum.SeekStatus.END, terms.SeekCeil(new BytesRef("abc"))); + r.Dispose(); + } + + w.Dispose(); + d.Dispose(); + } + + [Test] + public virtual void TestTermOrd() + { + Directory d = NewDirectory(); + IndexWriter w = new IndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat()))); + Documents.Document doc = new Documents.Document(); + doc.Add(NewTextField("f", "a b c", Field.Store.NO)); + w.AddDocument(doc); + w.ForceMerge(1); + DirectoryReader r = w.Reader; + TermsEnum terms = GetOnlySegmentReader(r).Fields.GetTerms("f").GetIterator(null); + Assert.IsTrue(terms.Next() != null); + try + { + Assert.AreEqual(0, terms.Ord); + } +#pragma warning disable 168 + catch (System.NotSupportedException uoe) +#pragma warning restore 168 + { + // ok -- codec is not required to support this op + } + r.Dispose(); + w.Dispose(); + d.Dispose(); + } + } +} \ No newline at end of file
