http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Stats/Points.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Stats/Points.cs b/src/Lucene.Net.Benchmark/ByTask/Stats/Points.cs new file mode 100644 index 0000000..ed54d92 --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Stats/Points.cs @@ -0,0 +1,108 @@ +using Lucene.Net.Benchmarks.ByTask.Tasks; +using Lucene.Net.Benchmarks.ByTask.Utils; +using System.Collections.Generic; + +namespace Lucene.Net.Benchmarks.ByTask.Stats +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Test run data points collected as the test proceeds. + /// </summary> + public class Points + { + // stat points ordered by their start time. + // for now we collect points as TaskStats objects. + // later might optimize to collect only native data. + private List<TaskStats> points = new List<TaskStats>(); + + private int nextTaskRunNum = 0; + + private TaskStats currentStats; + + /// <summary> + /// Create a Points statistics object. + /// </summary> + public Points(Config config) + { + } + + /// <summary> + /// Gets the current task stats. + /// The actual task stats are returned, so caller should not modify this task stats. + /// </summary> + public virtual IList<TaskStats> TaskStats + { + get { return points; } + } + + /// <summary> + /// Mark that a task is starting. + /// Create a task stats for it and store it as a point. + /// </summary> + /// <param name="task">The starting task.</param> + /// <param name="round">The new task stats created for the starting task.</param> + /// <returns></returns> + public virtual TaskStats MarkTaskStart(PerfTask task, int round) + { + lock (this) + { + TaskStats stats = new TaskStats(task, NextTaskRunNum(), round); + this.currentStats = stats; + points.Add(stats); + return stats; + } + } + + public virtual TaskStats CurrentStats + { + get { return currentStats; } + } + + // return next task num + private int NextTaskRunNum() + { + lock (this) + { + return nextTaskRunNum++; + } + } + + /// <summary> + /// mark the end of a task + /// </summary> + public virtual void MarkTaskEnd(TaskStats stats, int count) + { + lock (this) + { + int numParallelTasks = nextTaskRunNum - 1 - stats.TaskRunNum; + // note: if the stats were cleared, might be that this stats object is + // no longer in points, but this is just ok. + stats.MarkEnd(numParallelTasks, count); + } + } + + /// <summary> + /// Clear all data, prepare for more tests. + /// </summary> + public virtual void ClearData() + { + points.Clear(); + } + } +}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Stats/Report.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Stats/Report.cs b/src/Lucene.Net.Benchmark/ByTask/Stats/Report.cs new file mode 100644 index 0000000..59fd725 --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Stats/Report.cs @@ -0,0 +1,70 @@ +namespace Lucene.Net.Benchmarks.ByTask.Stats +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Textual report of current statistics. + /// </summary> + public class Report + { + private string text; + private int size; + private int outOf; + private int reported; + + public Report(string text, int size, int reported, int outOf) + { + this.text = text; + this.size = size; + this.reported = reported; + this.outOf = outOf; + } + + /// <summary> + /// Gets total number of stats points when this report was created. + /// </summary> + public virtual int OutOf + { + get { return outOf; } + } + + /// <summary> + /// Gets number of lines in the report. + /// </summary> + public virtual int Count + { + get { return size; } + } + + /// <summary> + /// Gets the report text. + /// </summary> + public virtual string Text + { + get { return text; } + } + + /// <summary> + /// Gets number of stats points represented in this report. + /// </summary> + public virtual int Reported + { + get { return reported; } + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Stats/TaskStats.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Stats/TaskStats.cs b/src/Lucene.Net.Benchmark/ByTask/Stats/TaskStats.cs new file mode 100644 index 0000000..4d32c7b --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Stats/TaskStats.cs @@ -0,0 +1,237 @@ +using Lucene.Net.Benchmarks.ByTask.Tasks; +using Lucene.Net.Support; +using System; +using System.Diagnostics; +using System.Text; + +namespace Lucene.Net.Benchmarks.ByTask.Stats +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Statistics for a task run. + /// <para/> + /// The same task can run more than once, but, if that task records statistics, + /// each run would create its own TaskStats. + /// </summary> + public class TaskStats + { + /// <summary>Task for which data was collected.</summary> + private PerfTask task; + + /// <summary>Round in which task run started.</summary> + private int round; + + /// <summary>Task start time.</summary> + private long start; + + /// <summary>Task elapsed time. elapsed >= 0 indicates run completion!</summary> + private long elapsed = -1; + + /// <summary>Max tot mem during task.</summary> + private long maxTotMem; + + /// <summary>Max used mem during task.</summary> + private long maxUsedMem; + + /// <summary>Serial run number of this task run in the perf run.</summary> + private int taskRunNum; + + /// <summary>Number of other tasks that started to run while this task was still running.</summary> + private int numParallelTasks; + + /// <summary> + /// Number of work items done by this task. + /// For indexing that can be number of docs added. + /// For warming that can be number of scanned items, etc. + /// For repeating tasks, this is a sum over repetitions. + /// </summary> + private int count; + + /// <summary> + /// Number of similar tasks aggregated into this record. + /// Used when summing up on few runs/instances of similar tasks. + /// </summary> + private int numRuns = 1; + + /// <summary> + /// Create a run data for a task that is starting now. + /// To be called from Points. + /// </summary> + internal TaskStats(PerfTask task, int taskRunNum, int round) + { + this.task = task; + this.taskRunNum = taskRunNum; + this.round = round; + maxTotMem = GC.GetTotalMemory(false); //Runtime.getRuntime().totalMemory(); + maxUsedMem = maxTotMem; // - Runtime.getRuntime().freeMemory(); // LUCENENET TODO: available RAM + start = Stopwatch.GetTimestamp(); + } + + /// <summary> + /// Mark the end of a task. + /// </summary> + internal void MarkEnd(int numParallelTasks, int count) + { + elapsed = Support.Time.CurrentTimeMilliseconds(); + long totMem = GC.GetTotalMemory(false); //Runtime.getRuntime().totalMemory(); + if (totMem > maxTotMem) + { + maxTotMem = totMem; + } + long usedMem = totMem; //- Runtime.getRuntime().freeMemory(); // LUCENENET TODO: available RAM + if (usedMem > maxUsedMem) + { + maxUsedMem = usedMem; + } + this.numParallelTasks = numParallelTasks; + this.count = count; + } + + private int[] countsByTime; + private long countsByTimeStepMSec; + + public virtual void SetCountsByTime(int[] counts, long msecStep) + { + countsByTime = counts; + countsByTimeStepMSec = msecStep; + } + + [WritableArray] + public virtual int[] GetCountsByTime() + { + return countsByTime; + } + + public virtual long CountsByTimeStepMSec + { + get { return countsByTimeStepMSec; } + } + + /// <summary>Gets the taskRunNum.</summary> + public virtual int TaskRunNum + { + get { return taskRunNum; } + } + + /// <seealso cref="object.ToString()"/> + public override string ToString() + { + StringBuilder res = new StringBuilder(task.GetName()); + res.Append(" "); + res.Append(count); + res.Append(" "); + res.Append(elapsed); + return res.ToString(); + } + + /// <summary>Gets the count.</summary> + public virtual int Count + { + get { return count; } + } + + /// <summary>Gets elapsed time.</summary> + public virtual long Elapsed + { + get { return elapsed; } + } + + /// <summary>Gets the maxTotMem.</summary> + public virtual long MaxTotMem + { + get { return maxTotMem; } + } + + /// <summary>Gets the maxUsedMem.</summary> + public virtual long MaxUsedMem + { + get { return maxUsedMem; } + } + + /// <summary>Gets the numParallelTasks.</summary> + public virtual int NumParallelTasks + { + get { return numParallelTasks; } + } + + /// <summary>Gets the task.</summary> + public virtual PerfTask Task + { + get { return task; } + } + + /// <summary>Gets the numRuns.</summary> + public virtual int NumRuns + { + get { return numRuns; } + } + + /// <summary> + /// Add data from another stat, for aggregation. + /// </summary> + /// <param name="stat2">The added stat data.</param> + public virtual void Add(TaskStats stat2) + { + numRuns += stat2.NumRuns; + elapsed += stat2.Elapsed; + maxTotMem += stat2.MaxTotMem; + maxUsedMem += stat2.MaxUsedMem; + count += stat2.Count; + if (round != stat2.round) + { + round = -1; // no meaning if aggregating tasks of different round. + } + + if (countsByTime != null && stat2.countsByTime != null) + { + if (countsByTimeStepMSec != stat2.countsByTimeStepMSec) + { + throw new InvalidOperationException("different by-time msec step"); + } + if (countsByTime.Length != stat2.countsByTime.Length) + { + throw new InvalidOperationException("different by-time msec count"); + } + for (int i = 0; i < stat2.countsByTime.Length; i++) + { + countsByTime[i] += stat2.countsByTime[i]; + } + } + } + +#if FEATURE_CLONEABLE + /// <seealso cref="ICloneable.Clone()"/> +#endif + public virtual object Clone() + { + TaskStats c = (TaskStats)base.MemberwiseClone(); + if (c.countsByTime != null) + { + c.countsByTime = (int[])c.countsByTime.Clone(); + } + return c; + } + + /// <summary>Gets the round number.</summary> + public virtual int Round + { + get { return round; } + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/AddDocTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/AddDocTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddDocTask.cs new file mode 100644 index 0000000..8e92f48 --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddDocTask.cs @@ -0,0 +1,93 @@ +using Lucene.Net.Benchmarks.ByTask.Feeds; +using Lucene.Net.Documents; +using System.Globalization; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Add a document, optionally of a certain size. + /// <para/> + /// Other side effects: none. + /// <para/> + /// Takes optional param: document size. + /// </summary> + public class AddDocTask : PerfTask + { + public AddDocTask(PerfRunData runData) + : base(runData) + { + } + + private int docSize = 0; + + /// <summary> + /// Volatile data passed between <see cref="Setup()"/>, <see cref="DoLogic()"/>, <see cref="TearDown()"/>. + /// The doc is created at <see cref="Setup()"/> and added at <see cref="DoLogic()"/>. + /// </summary> + protected Document m_doc = null; + + public override void Setup() + { + base.Setup(); + DocMaker docMaker = RunData.DocMaker; + if (docSize > 0) + { + m_doc = docMaker.MakeDocument(docSize); + } + else + { + m_doc = docMaker.MakeDocument(); + } + } + + public override void TearDown() + { + m_doc = null; + base.TearDown(); + } + + protected override string GetLogMessage(int recsCount) + { + return string.Format(CultureInfo.InvariantCulture, "added {0:N9} docs", recsCount); + } + + public override int DoLogic() + { + RunData.IndexWriter.AddDocument(m_doc); + return 1; + } + + /// <summary> + /// Set the params (docSize only). + /// </summary> + /// <param name="params">docSize, or 0 for no limit.</param> + public override void SetParams(string @params) + { + base.SetParams(@params); + docSize = (int)float.Parse(@params, CultureInfo.InvariantCulture); + } + + /// <seealso cref="PerfTask.SupportsParams"/> + public override bool SupportsParams + { + get { return true; } + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/AddFacetedDocTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/AddFacetedDocTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddFacetedDocTask.cs new file mode 100644 index 0000000..6ae761c --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddFacetedDocTask.cs @@ -0,0 +1,95 @@ +using Lucene.Net.Benchmarks.ByTask.Feeds; +using Lucene.Net.Facet; +using System.Collections.Generic; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Add a faceted document. + /// </summary> + /// <remarks> + /// Config properties: + /// <list type="bullet"> + /// <item> + /// <term>with.facets</term> + /// <description> + /// <tells whether to actually add any facets to the document| Default: true> + /// <para/> + /// This config property allows to easily compare the performance of adding docs + /// with and without facets. Note that facets are created even when this is + /// <c>false</c>, just that they are not added to the document (nor to the taxonomy). + /// </description> + /// </item> + /// </list> + /// <para/> + /// See <see cref="AddDocTask"/> for general document parameters and configuration. + /// <para/> + /// Makes use of the <see cref="FacetSource"/> in effect - see <see cref="PerfRunData"/> for + /// facet source settings. + /// </remarks> + public class AddFacetedDocTask : AddDocTask + { + private FacetsConfig config; + + public AddFacetedDocTask(PerfRunData runData) + : base(runData) + { + } + + public override void Setup() + { + base.Setup(); + if (config == null) + { + bool withFacets = RunData.Config.Get("with.facets", true); + if (withFacets) + { + FacetSource facetsSource = RunData.FacetSource; + config = new FacetsConfig(); + facetsSource.Configure(config); + } + } + } + + protected override string GetLogMessage(int recsCount) + { + if (config == null) + { + return base.GetLogMessage(recsCount); + } + return base.GetLogMessage(recsCount) + " with facets"; + } + + public override int DoLogic() + { + if (config != null) + { + List<FacetField> facets = new List<FacetField>(); + RunData.FacetSource.GetNextFacets(facets); + foreach (FacetField ff in facets) + { + m_doc.Add(ff); + } + m_doc = config.Build(RunData.TaxonomyWriter, m_doc); + } + return base.DoLogic(); + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/AddIndexesTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/AddIndexesTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddIndexesTask.cs new file mode 100644 index 0000000..f05ca32 --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/AddIndexesTask.cs @@ -0,0 +1,104 @@ +using Lucene.Net.Index; +using Lucene.Net.Store; +using System; +using System.IO; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Adds an input index to an existing index, using + /// <see cref="IndexWriter.AddIndexes(Store.Directory[])"/> or + /// <see cref="IndexWriter.AddIndexes(IndexReader[])"/>. The location of the input + /// index is specified by the parameter <see cref="ADDINDEXES_INPUT_DIR"/> and is + /// assumed to be a directory on the file system. + /// <para/> + /// Takes optional parameter <see cref="useAddIndexesDir"/> which specifies which + /// AddIndexes variant to use (defaults to <c>true</c>, to use <c>AddIndexes(Directory)</c>). + /// </summary> + public class AddIndexesTask : PerfTask + { + public static readonly string ADDINDEXES_INPUT_DIR = "addindexes.input.dir"; + + public AddIndexesTask(PerfRunData runData) + : base(runData) + { + } + + private bool useAddIndexesDir = true; + private FSDirectory inputDir; + + public override void Setup() + { + base.Setup(); + string inputDirProp = RunData.Config.Get(ADDINDEXES_INPUT_DIR, null); + if (inputDirProp == null) + { + throw new ArgumentException("config parameter " + ADDINDEXES_INPUT_DIR + " not specified in configuration"); + } + inputDir = FSDirectory.Open(new DirectoryInfo(inputDirProp)); + } + + public override int DoLogic() + { + IndexWriter writer = RunData.IndexWriter; + if (useAddIndexesDir) + { + writer.AddIndexes(inputDir); + } + else + { + IndexReader r = DirectoryReader.Open(inputDir); + try + { + writer.AddIndexes(r); + } + finally + { + r.Dispose(); + } + } + return 1; + } + + /// <summary> + /// Set the params (useAddIndexesDir only) + /// </summary> + /// <param name="params"> + /// <c>useAddIndexesDir=true</c> for using <see cref="IndexWriter.AddIndexes(Store.Directory[])"/> or <c>false</c> for + /// using <see cref="IndexWriter.AddIndexes(IndexReader[])"/>. Defaults to <c>true</c>. + /// </param> + public override void SetParams(string @params) + { + base.SetParams(@params); + useAddIndexesDir = bool.Parse(@params); + } + + public override bool SupportsParams + { + get { return true; } + } + + public override void TearDown() + { + inputDir.Dispose(); + base.TearDown(); + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs new file mode 100644 index 0000000..56b0114 --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/AnalyzerFactoryTask.cs @@ -0,0 +1,580 @@ +using Lucene.Net.Analysis.Util; +using Lucene.Net.Benchmarks.ByTask.Utils; +using Lucene.Net.Support.IO; +using Lucene.Net.Util; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text.RegularExpressions; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Analyzer factory construction task. The name given to the constructed factory may + /// be given to <see cref="NewAnalyzerTask"/>, which will call <see cref="AnalyzerFactory.Create()"/>. + /// </summary> + /// <remarks> + /// Params are in the form argname:argvalue or argname:"argvalue" or argname:'argvalue'; + /// use backslashes to escape '"' or "'" inside a quoted value when it's used as the enclosing + /// quotation mark, + /// <para/> + /// Specify params in a comma separated list of the following, in order: + /// <list type="number"> + /// <item><description> + /// <list type="bullet"> + /// <item><description><b>Required</b>: <c>name:<i>analyzer-factory-name</i></c></description></item> + /// <item><description>Optional: <c>positionIncrementGap:<i>int value</i></c> (default: 0)</description></item> + /// <item><description>Optional: <c>offsetGap:<i>int value</i></c> (default: 1)</description></item> + /// </list> + /// </description></item> + /// <item><description>zero or more CharFilterFactory's, followed by</description></item> + /// <item><description>exactly one TokenizerFactory, followed by</description></item> + /// <item><description>zero or more TokenFilterFactory's</description></item> + /// </list> + /// <para/> + /// Each component analysis factory map specify <tt>luceneMatchVersion</tt> (defaults to + /// <see cref="LuceneVersion.LUCENE_CURRENT"/>) and any of the args understood by the specified + /// *Factory class, in the above-describe param format. + /// <para/> + /// Example: + /// <code> + /// -AnalyzerFactory(name:'strip html, fold to ascii, whitespace tokenize, max 10k tokens', + /// positionIncrementGap:100, + /// HTMLStripCharFilter, + /// MappingCharFilter(mapping:'mapping-FoldToASCII.txt'), + /// WhitespaceTokenizer(luceneMatchVersion:LUCENE_43), + /// TokenLimitFilter(maxTokenCount:10000, consumeAllTokens:false)) + /// [...] + /// -NewAnalyzer('strip html, fold to ascii, whitespace tokenize, max 10k tokens') + /// </code> + /// <para/> + /// <see cref="AnalyzerFactory"/> will direct analysis component factories to look for resources + /// under the directory specified in the "work.dir" property. + /// </remarks> + public class AnalyzerFactoryTask : PerfTask + { + private static readonly string LUCENE_ANALYSIS_PACKAGE_PREFIX = "Lucene.Net.Analysis."; + private static readonly Regex ANALYSIS_COMPONENT_SUFFIX_PATTERN + = new Regex("(?s:(?:(?:Token|Char)?Filter|Tokenizer)(?:Factory)?)$", RegexOptions.Compiled); + private static readonly Regex TRAILING_DOT_ZERO_PATTERN = new Regex(@"\.0$", RegexOptions.Compiled); + + private enum ArgType { ANALYZER_ARG, ANALYZER_ARG_OR_CHARFILTER_OR_TOKENIZER, TOKENFILTER } + + string factoryName = null; + int? positionIncrementGap = null; + int? offsetGap = null; + private IList<CharFilterFactory> charFilterFactories = new List<CharFilterFactory>(); + private TokenizerFactory tokenizerFactory = null; + private IList<TokenFilterFactory> tokenFilterFactories = new List<TokenFilterFactory>(); + + public AnalyzerFactoryTask(PerfRunData runData) + : base(runData) + { + } + + public override int DoLogic() + { + return 1; + } + + /// <summary> + /// Sets the params. + /// Analysis component factory names may optionally include the "Factory" suffix. + /// </summary> + /// <param name="params"> + /// analysis pipeline specification: name, (optional) positionIncrementGap, + /// (optional) offsetGap, 0+ CharFilterFactory's, 1 TokenizerFactory, + /// and 0+ TokenFilterFactory's + /// </param> + public override void SetParams(string @params) + { + base.SetParams(@params); + ArgType expectedArgType = ArgType.ANALYZER_ARG; + + StreamTokenizer stok = new StreamTokenizer(new StringReader(@params)); + stok.CommentChar('#'); + stok.QuoteChar('"'); + stok.QuoteChar('\''); + stok.IsEOLSignificant = false; + stok.OrdinaryChar('('); + stok.OrdinaryChar(')'); + stok.OrdinaryChar(':'); + stok.OrdinaryChar(','); + try + { + while (stok.NextToken() != StreamTokenizer.TT_EOF) + { + switch (stok.TokenType) + { + case ',': + { + // Do nothing + break; + } + case StreamTokenizer.TT_WORD: + { + if (expectedArgType.Equals(ArgType.ANALYZER_ARG)) + { + string argName = stok.StringValue; + if (!argName.Equals("name", StringComparison.OrdinalIgnoreCase) + && !argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase) + && !argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase)) + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Missing 'name' param to AnalyzerFactory: '" + @params + "'"); + } + stok.NextToken(); + if (stok.TokenType != ':') + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Missing ':' after '" + argName + "' param to AnalyzerFactory"); + } + + stok.NextToken(); + string argValue = stok.StringValue; + switch (stok.TokenType) + { + case StreamTokenizer.TT_NUMBER: + { + argValue = stok.NumberValue.ToString(CultureInfo.InvariantCulture); + // Drop the ".0" from numbers, for integer arguments + argValue = TRAILING_DOT_ZERO_PATTERN.Replace(argValue, "", 1); + // Intentional fallthrough + + if (argName.Equals("name", StringComparison.OrdinalIgnoreCase)) + { + factoryName = argValue; + expectedArgType = ArgType.ANALYZER_ARG_OR_CHARFILTER_OR_TOKENIZER; + } + else + { + int intArgValue = 0; + try + { + intArgValue = int.Parse(argValue, CultureInfo.InvariantCulture); + } + catch (FormatException e) + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Exception parsing " + argName + " value '" + argValue + "'", e); + } + if (argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase)) + { + positionIncrementGap = intArgValue; + } + else if (argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase)) + { + offsetGap = intArgValue; + } + } + break; + } + case '"': + case '\'': + case StreamTokenizer.TT_WORD: + { + if (argName.Equals("name", StringComparison.OrdinalIgnoreCase)) + { + factoryName = argValue; + expectedArgType = ArgType.ANALYZER_ARG_OR_CHARFILTER_OR_TOKENIZER; + } + else + { + int intArgValue = 0; + try + { + intArgValue = int.Parse(argValue, CultureInfo.InvariantCulture); + } + catch (FormatException e) + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Exception parsing " + argName + " value '" + argValue + "'", e); + } + if (argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase)) + { + positionIncrementGap = intArgValue; + } + else if (argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase)) + { + offsetGap = intArgValue; + } + } + break; + } + case StreamTokenizer.TT_EOF: + { + throw new Exception("Unexpected EOF: " + stok.ToString()); + } + default: + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Unexpected token: " + stok.ToString()); + } + } + } + else if (expectedArgType.Equals(ArgType.ANALYZER_ARG_OR_CHARFILTER_OR_TOKENIZER)) + { + string argName = stok.StringValue; + + if (argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase) + || argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase)) + { + stok.NextToken(); + if (stok.TokenType != ':') + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Missing ':' after '" + argName + "' param to AnalyzerFactory"); + } + stok.NextToken(); + int intArgValue = (int)stok.NumberValue; + switch (stok.TokenType) + { + case '"': + case '\'': + case StreamTokenizer.TT_WORD: + { + intArgValue = 0; + try + { + intArgValue = int.Parse(stok.StringValue.Trim(), CultureInfo.InvariantCulture); + } + catch (FormatException e) + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Exception parsing " + argName + " value '" + stok.StringValue + "'", e); + } + // Intentional fall-through + + if (argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase)) + { + positionIncrementGap = intArgValue; + } + else if (argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase)) + { + offsetGap = intArgValue; + } + break; + } + case StreamTokenizer.TT_NUMBER: + { + if (argName.Equals("positionIncrementGap", StringComparison.OrdinalIgnoreCase)) + { + positionIncrementGap = intArgValue; + } + else if (argName.Equals("offsetGap", StringComparison.OrdinalIgnoreCase)) + { + offsetGap = intArgValue; + } + break; + } + case StreamTokenizer.TT_EOF: + { + throw new Exception("Unexpected EOF: " + stok.ToString()); + } + default: + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Unexpected token: " + stok.ToString()); + } + } + break; + } + try + { + Type clazz; + clazz = LookupAnalysisClass(argName, typeof(CharFilterFactory)); + CreateAnalysisPipelineComponent(stok, clazz); + } + catch (ArgumentException /*e*/) + { + try + { + Type clazz; + clazz = LookupAnalysisClass(argName, typeof(TokenizerFactory)); + CreateAnalysisPipelineComponent(stok, clazz); + expectedArgType = ArgType.TOKENFILTER; + } + catch (ArgumentException e2) + { + throw new Exception("Line #" + GetLineNumber(stok) + ": Can't find class '" + + argName + "' as CharFilterFactory or TokenizerFactory", e2); + } + } + } + else + { // expectedArgType = ArgType.TOKENFILTER + string className = stok.StringValue; + Type clazz; + try + { + clazz = LookupAnalysisClass(className, typeof(TokenFilterFactory)); + } + catch (ArgumentException e) + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Can't find class '" + className + "' as TokenFilterFactory", e); + } + CreateAnalysisPipelineComponent(stok, clazz); + } + break; + } + default: + { + throw new Exception("Line #" + GetLineNumber(stok) + ": Unexpected token: " + stok.ToString()); + } + } + } + } + catch (Exception e) + { + if (e.Message.StartsWith("Line #", StringComparison.Ordinal)) + { + throw e; + } + else + { + throw new Exception("Line #" + GetLineNumber(stok) + ": ", e); + } + } + + AnalyzerFactory analyzerFactory = new AnalyzerFactory + (charFilterFactories, tokenizerFactory, tokenFilterFactories); + analyzerFactory.SetPositionIncrementGap(positionIncrementGap.GetValueOrDefault()); + analyzerFactory.SetOffsetGap(offsetGap.GetValueOrDefault()); + RunData.AnalyzerFactories[factoryName] = analyzerFactory; + } + + /// <summary> + /// Instantiates the given analysis factory class after pulling params from + /// the given stream tokenizer, then stores the result in the appropriate + /// pipeline component list. + /// </summary> + /// <param name="stok">Stream tokenizer from which to draw analysis factory params.</param> + /// <param name="clazz">Analysis factory class to instantiate.</param> + private void CreateAnalysisPipelineComponent(StreamTokenizer stok, Type clazz) + { + IDictionary<string, string> argMap = new Dictionary<string, string>(); + bool parenthetical = false; + try + { + while (stok.NextToken() != StreamTokenizer.TT_EOF) + { + switch (stok.TokenType) + { + case ',': + { + if (parenthetical) + { + // Do nothing + break; + } + else + { + // Finished reading this analysis factory configuration + goto WHILE_LOOP_BREAK; + } + } + case '(': + { + if (parenthetical) + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Unexpected opening parenthesis."); + } + parenthetical = true; + break; + } + case ')': + { + if (parenthetical) + { + parenthetical = false; + } + else + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Unexpected closing parenthesis."); + } + break; + } + case StreamTokenizer.TT_WORD: + { + if (!parenthetical) + { + throw new Exception("Line #" + GetLineNumber(stok) + ": Unexpected token '" + stok.StringValue + "'"); + } + string argName = stok.StringValue; + stok.NextToken(); + if (stok.TokenType != ':') + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Missing ':' after '" + argName + "' param to " + clazz.Name); + } + stok.NextToken(); + string argValue = stok.StringValue; + switch (stok.TokenType) + { + case StreamTokenizer.TT_NUMBER: + { + argValue = stok.NumberValue.ToString(CultureInfo.InvariantCulture); + // Drop the ".0" from numbers, for integer arguments + argValue = TRAILING_DOT_ZERO_PATTERN.Replace(argValue, "", 1); + // Intentional fall-through + argMap[argName] = argValue; + break; + } + case '"': + case '\'': + case StreamTokenizer.TT_WORD: + { + argMap[argName] = argValue; + break; + } + case StreamTokenizer.TT_EOF: + { + throw new Exception("Unexpected EOF: " + stok.ToString()); + } + default: + { + throw new Exception + ("Line #" + GetLineNumber(stok) + ": Unexpected token: " + stok.ToString()); + } + } + break; + } + } + } + WHILE_LOOP_BREAK: { } + + if (!argMap.ContainsKey("luceneMatchVersion")) + { +#pragma warning disable 612, 618 + argMap["luceneMatchVersion"] = LuceneVersion.LUCENE_CURRENT.ToString(); +#pragma warning restore 612, 618 + } + AbstractAnalysisFactory instance; + try + { + instance = (AbstractAnalysisFactory)Activator.CreateInstance(clazz, argMap); + } + catch (Exception e) + { + throw new Exception("Line #" + GetLineNumber(stok) + ": ", e); + } + if (instance is IResourceLoaderAware) + { + DirectoryInfo baseDir = new DirectoryInfo(RunData.Config.Get("work.dir", "work")); + ((IResourceLoaderAware)instance).Inform(new FilesystemResourceLoader(baseDir)); + } + if (typeof(CharFilterFactory).IsAssignableFrom(clazz)) + { + charFilterFactories.Add((CharFilterFactory)instance); + } + else if (typeof(TokenizerFactory).IsAssignableFrom(clazz)) + { + tokenizerFactory = (TokenizerFactory)instance; + } + else if (typeof(TokenFilterFactory).IsAssignableFrom(clazz)) + { + tokenFilterFactories.Add((TokenFilterFactory)instance); + } + } + catch (Exception e) + { + if (e.Message.StartsWith("Line #", StringComparison.Ordinal)) + { + throw e; + } + else + { + throw new Exception("Line #" + GetLineNumber(stok) + ": ", e); + } + } + } + + /// <summary> + /// This method looks up a class with its fully qualified name (FQN), or a short-name + /// class-simplename, or with a package suffix, assuming "Lucene.Net.Analysis." + /// as the namespace prefix (e.g. "standard.ClassicTokenizerFactory" -> + /// "Lucene.Net.Analysis.Standard.ClassicTokenizerFactory"). + /// </summary> + /// <remarks> + /// If <paramref name="className"/> contains a period, the class is first looked up as-is, assuming that it + /// is an FQN. If this fails, lookup is retried after prepending the Lucene analysis + /// package prefix to the class name. + /// <para/> + /// If <paramref name="className"/> does not contain a period, the analysis SPI *Factory.LookupClass() + /// methods are used to find the class. + /// </remarks> + /// <param name="className">The namespace qualified name or the short name of the class.</param> + /// <param name="expectedType">The superclass <paramref name="className"/> is expected to extend. </param> + /// <returns>The loaded type.</returns> + /// <exception cref="TypeLoadException">If lookup fails.</exception> + public virtual Type LookupAnalysisClass(string className, Type expectedType) + { + if (className.Contains(".")) + { + // First, try className == FQN + Type result = Type.GetType(className); + if (result == null) + { + // Second, retry lookup after prepending the Lucene analysis package prefix + result = Type.GetType(LUCENE_ANALYSIS_PACKAGE_PREFIX + className); + + if (result == null) + { + throw new TypeLoadException("Can't find class '" + className + + "' or '" + LUCENE_ANALYSIS_PACKAGE_PREFIX + className + "'"); + } + } + return result; + } + // No dot - use analysis SPI lookup + string analysisComponentName = ANALYSIS_COMPONENT_SUFFIX_PATTERN.Replace(className, "", 1); + if (typeof(CharFilterFactory).IsAssignableFrom(expectedType)) + { + return CharFilterFactory.LookupClass(analysisComponentName); + } + else if (typeof(TokenizerFactory).IsAssignableFrom(expectedType)) + { + return TokenizerFactory.LookupClass(analysisComponentName); + } + else if (typeof(TokenFilterFactory).IsAssignableFrom(expectedType)) + { + return TokenFilterFactory.LookupClass(analysisComponentName); + } + + throw new TypeLoadException("Can't find class '" + className + "'"); + } + + /// <seealso cref="PerfTask.SupportsParams"/> + public override bool SupportsParams + { + get { return true; } + } + + /// <summary>Returns the current line in the algorithm file</summary> + public virtual int GetLineNumber(StreamTokenizer stok) + { + return AlgLineNum + stok.LineNumber; + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/BenchmarkHighlighter.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/BenchmarkHighlighter.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/BenchmarkHighlighter.cs new file mode 100644 index 0000000..a20160c --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/BenchmarkHighlighter.cs @@ -0,0 +1,32 @@ +using Lucene.Net.Analysis; +using Lucene.Net.Documents; +using Lucene.Net.Index; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Abstract class for benchmarking highlighting performance + /// </summary> + public abstract class BenchmarkHighlighter + { + public abstract int DoHighlight(IndexReader reader, int doc, string field, + Document document, Analyzer analyzer, string text); + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ClearStatsTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ClearStatsTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ClearStatsTask.cs new file mode 100644 index 0000000..16a1859 --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ClearStatsTask.cs @@ -0,0 +1,44 @@ +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Clear statistics data. + /// <para/> + /// Other side effects: None. + /// </summary> + public class ClearStatsTask : PerfTask + { + public ClearStatsTask(PerfRunData runData) + : base(runData) + { + } + + public override int DoLogic() + { + RunData.Points.ClearData(); + return 0; + } + + /// <seealso cref="PerfTask.ShouldNotRecordStats"/> + protected override bool ShouldNotRecordStats + { + get { return true; } + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseIndexTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseIndexTask.cs new file mode 100644 index 0000000..30e31d9 --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseIndexTask.cs @@ -0,0 +1,67 @@ +using Lucene.Net.Index; +using Lucene.Net.Util; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Close index writer. + /// <para/> + /// Other side effects: index writer object in perfRunData is nullified. + /// <para/> + /// Takes optional param "doWait": if false, then close(false) is called. + /// </summary> + public class CloseIndexTask : PerfTask + { + public CloseIndexTask(PerfRunData runData) + : base(runData) + { + } + + bool doWait = true; + + public override int DoLogic() + { + IndexWriter iw = RunData.IndexWriter; + if (iw != null) + { + // If infoStream was set to output to a file, close it. + InfoStream infoStream = iw.Config.InfoStream; + if (infoStream != null) + { + infoStream.Dispose(); + } + iw.Dispose(doWait); + RunData.IndexWriter = null; + } + return 1; + } + + public override void SetParams(string @params) + { + base.SetParams(@params); + doWait = bool.Parse(@params); + } + + public override bool SupportsParams + { + get { return true; } + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseReaderTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseReaderTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseReaderTask.cs new file mode 100644 index 0000000..7a8c61c --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseReaderTask.cs @@ -0,0 +1,49 @@ +using Lucene.Net.Index; +using Lucene.Net.Support; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Close index reader. + /// <para/> + /// Other side effects: index reader in perfRunData is nullified. + /// <para/> + /// This would cause read related tasks to reopen their own reader. + /// </summary> + public class CloseReaderTask : PerfTask + { + public CloseReaderTask(PerfRunData runData) + : base(runData) + { + } + + public override int DoLogic() + { + IndexReader reader = RunData.GetIndexReader(); + RunData.SetIndexReader(null); + if (reader.RefCount != 1) + { + SystemConsole.WriteLine("WARNING: CloseReader: reference count is currently " + reader.RefCount); + } + reader.DecRef(); + return 1; + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyIndexTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyIndexTask.cs new file mode 100644 index 0000000..7d94a9a --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyIndexTask.cs @@ -0,0 +1,42 @@ +using Lucene.Net.Util; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Close taxonomy index. + /// <para/> + /// Other side effects: taxonomy writer object in perfRunData is nullified. + /// </summary> + public class CloseTaxonomyIndexTask : PerfTask + { + public CloseTaxonomyIndexTask(PerfRunData runData) + : base(runData) + { + } + + public override int DoLogic() + { + IOUtils.Dispose(RunData.TaxonomyWriter); + RunData.TaxonomyWriter = null; + + return 1; + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyReaderTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyReaderTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyReaderTask.cs new file mode 100644 index 0000000..fc1ab27 --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CloseTaxonomyReaderTask.cs @@ -0,0 +1,47 @@ +using Lucene.Net.Facet.Taxonomy; +using Lucene.Net.Support; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Close taxonomy reader. + /// <para/> + /// Other side effects: taxonomy reader in perfRunData is nullified. + /// </summary> + public class CloseTaxonomyReaderTask : PerfTask + { + public CloseTaxonomyReaderTask(PerfRunData runData) + : base(runData) + { + } + + public override int DoLogic() + { + TaxonomyReader taxoReader = RunData.GetTaxonomyReader(); + RunData.SetTaxonomyReader(null); + if (taxoReader.RefCount != 1) + { + SystemConsole.WriteLine("WARNING: CloseTaxonomyReader: reference count is currently " + taxoReader.RefCount); + } + taxoReader.Dispose(); + return 1; + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitIndexTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitIndexTask.cs new file mode 100644 index 0000000..b914371 --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitIndexTask.cs @@ -0,0 +1,62 @@ +using Lucene.Net.Index; +using System.Collections.Generic; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Commits the <see cref="IndexWriter"/>. + /// </summary> + public class CommitIndexTask : PerfTask + { + IDictionary<string, string> commitUserData; + + public CommitIndexTask(PerfRunData runData) + : base(runData) + { + } + + public override bool SupportsParams + { + get { return true; } + } + + public override void SetParams(string @params) + { + base.SetParams(@params); + commitUserData = new Dictionary<string, string>(); + commitUserData[OpenReaderTask.USER_DATA] = @params; + } + + public override int DoLogic() + { + IndexWriter iw = RunData.IndexWriter; + if (iw != null) + { + if (commitUserData != null) + { + iw.SetCommitData(commitUserData); + } + iw.Commit(); + } + + return 1; + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitTaxonomyIndexTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitTaxonomyIndexTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitTaxonomyIndexTask.cs new file mode 100644 index 0000000..4b8cc2c --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/CommitTaxonomyIndexTask.cs @@ -0,0 +1,48 @@ +using Lucene.Net.Facet.Taxonomy; +using System; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Commits the Taxonomy Index. + /// </summary> + public class CommitTaxonomyIndexTask : PerfTask + { + public CommitTaxonomyIndexTask(PerfRunData runData) + : base(runData) + { + } + + public override int DoLogic() + { + ITaxonomyWriter taxonomyWriter = RunData.TaxonomyWriter; + if (taxonomyWriter != null) + { + taxonomyWriter.Commit(); + } + else + { + throw new InvalidOperationException("TaxonomyWriter is not currently open"); + } + + return 1; + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/ConsumeContentSourceTask.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/ConsumeContentSourceTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/ConsumeContentSourceTask.cs new file mode 100644 index 0000000..36a5a14 --- /dev/null +++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/ConsumeContentSourceTask.cs @@ -0,0 +1,48 @@ +using Lucene.Net.Benchmarks.ByTask.Feeds; +using System.Threading; + +namespace Lucene.Net.Benchmarks.ByTask.Tasks +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary> + /// Consumes a <see cref="Feeds.ContentSource"/>. + /// </summary> + public class ConsumeContentSourceTask : PerfTask + { + private readonly ContentSource source; + private ThreadLocal<DocData> dd = new ThreadLocal<DocData>(); + + public ConsumeContentSourceTask(PerfRunData runData) + : base(runData) + { + source = runData.ContentSource; + } + + protected override string GetLogMessage(int recsCount) + { + return "read " + recsCount + " documents from the content source"; + } + + public override int DoLogic() + { + dd.Value = source.GetNextDocData(dd.Value); + return 1; + } + } +}
