This is an automated email from the ASF dual-hosted git repository.
nightowl888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucenenet.git
The following commit(s) were added to refs/heads/master by this push:
new a9f7d306e Lucene.Net.TestFramework: Improved error handling and test
reporting (#1092)
a9f7d306e is described below
commit a9f7d306e5aa4c77da71176ecea56ec40c81fd8b
Author: Shad Storhaug <[email protected]>
AuthorDate: Wed Jan 8 23:41:03 2025 +0700
Lucene.Net.TestFramework: Improved error handling and test reporting (#1092)
* Lucene.Net.Util.LuceneTestCase.SetUpFixture::OneTimeSetUpWrapper():
Removed debug code.
* Lucene.Net.Util.LuceneTestCase (OneTimeSetUp() + OneTimeTearDown()):
Fixed error reporting, since NUnit swallows exceptions in these events. This
will write them to stderr by default and allows enabling failing the tests if
an exception occurs in LuceneTestCase.OneTimeSetUp() which will make them
appear in the Test Output window in Visual Studio. The failures can be enabled
by setting the tests:failontestfixtureonetimesetuperror system property to true.
* azure-pipelines.yml: Added FailOnTestFixtureOneTimeSetUpError environment
variable (defaulting to 'true') to set the system property of the same name.
* Lucene.Net.Search.RandomSimilarityProvider: Use
J2N.Collections.Generic.Dictionary<TKey, TValue>, which will format itself.
Marked fields private instead of internal.
* PERFORMANCE: Moved logging random test settings and system properties
from SetUp() to TearDown() and only include them in the test message upon
failure.
* Lucene.Net.Support.ExceptionHandling.TestExceptionExtensions: Removed
messages from calls to Assert.Pass(), since this writes to the output and
bloats the size of the test logs with no benefit.
* publish-test-results.yml + run-tests-on-os.yml: Changed build to fail
when maximumAllowedFailures is 0 and there are errors in the stderr file.
* publish-test-results.yml + run-tests-on-os.yml: Changed build to fail
when maximumAllowedFailures is 0 and there are errors in the stdout in the
TestResults.trx file.
* publish-test-results.yml: Check for [ERROR] in RunInfo rather than StdOut
---
.build/azure-templates/publish-test-results.yml | 35 +++++++
.build/azure-templates/run-tests-on-os.yml | 10 ++
azure-pipelines.yml | 5 +-
.../Search/RandomSimilarityProvider.cs | 16 +--
.../Support/Util/LuceneTestCase.SetUpFixture.cs | 1 -
.../Support/Util/TestExtensions.cs | 90 ++++++++++++++++
.../Util/LuceneTestCase.cs | 116 +++++++++++----------
.../ExceptionHandling/TestExceptionExtensions.cs | 7 +-
8 files changed, 211 insertions(+), 69 deletions(-)
diff --git a/.build/azure-templates/publish-test-results.yml
b/.build/azure-templates/publish-test-results.yml
index 1a71be0ee..b615f47e8 100644
--- a/.build/azure-templates/publish-test-results.yml
+++ b/.build/azure-templates/publish-test-results.yml
@@ -87,6 +87,13 @@ steps:
Write-Host "##vso[task.setvariable
variable=CrashedRuns;]$crashedRuns"
$crashed = $true
}
+ if ($innerXml -and ($innerXml.Contains('[ERROR]'))) {
+ Write-Host "##vso[task.setvariable
variable=StdOutFailure;]true"
+ # Report all of the test projects that had stdout
failures
+ $stdOutFailureRuns =
"$env:STDOUTFAILURERUNS,$testProjectName".TrimStart(',')
+ Write-Host "##vso[task.setvariable
variable=StdOutFailureRuns;]$stdOutFailureRuns"
+ $crashed = $true
+ }
}
}
if ($reader.NodeType -eq [System.Xml.XmlNodeType]::EndElement
-and $reader.Name -eq 'RunInfos') {
@@ -102,6 +109,34 @@ steps:
Write-Host "##vso[task.setvariable
variable=TestResultsFileExists;]$testResultsFileExists"
displayName: 'Parse Test Results File'
+- pwsh: |
+ $testProjectName = "${{ parameters.testProjectName }}"
+ $testStdErrFileName = "$(Build.ArtifactStagingDirectory)/${{
parameters.testResultsArtifactName }}/${{ parameters.osName }}/${{
parameters.framework }}/${{ parameters.vsTestPlatform
}}/$testProjectName/dotnet-test-error.log"
+ $testStdErrFileExists = Test-Path $testStdErrFileName
+ if ($testStdErrFileExists) {
+ $fileLength = (Get-Item $testStdErrFileName).Length
+ if ($fileLength -gt 0) {
+ $stream = [System.IO.StreamReader]::new($testStdErrFileName)
+ try {
+ while (-not $stream.EndOfStream) {
+ $line = $stream.ReadLine()
+ if ($line -match "Test Run Failed" -or $line -match
"\[ERROR\]") {
+ Write-Host "##vso[task.setvariable
variable=StdErrFailure;]true"
+ # Report all of the test projects that had stderr
failures
+ $stdErrFailureRuns =
"$env:STDERRFAILURERUNS,$testProjectName".TrimStart(',')
+ Write-Host "##vso[task.setvariable
variable=StdErrFailureRuns;]$stdErrFailureRuns"
+ break # No need to continue reading after detecting a
failure
+ }
+ }
+ } finally {
+ $stream.Dispose()
+ }
+ }
+ } else {
+ Write-Host "WARNING: File not found: $testStdErrFileName"
+ }
+ displayName: 'Parse StdErr File'
+
- task: PublishTestResults@2
displayName: 'Publish Test Results ${{ parameters.testProjectName }},${{
parameters.framework }},${{ parameters.vsTestPlatform }}'
inputs:
diff --git a/.build/azure-templates/run-tests-on-os.yml
b/.build/azure-templates/run-tests-on-os.yml
index b535aa394..18df1945f 100644
--- a/.build/azure-templates/run-tests-on-os.yml
+++ b/.build/azure-templates/run-tests-on-os.yml
@@ -310,6 +310,16 @@ steps:
Write-Host "##vso[task.logissue type=error;]Test run failed due to too
many failed tests. Maximum failures allowed: $maximumAllowedFailures, total
failures: $($env:TOTALFAILURES)."
$failed = $true
}
+ if ([int]$maximumAllowedFailures -eq 0 -and $env:STDERRFAILURE -eq 'true')
{
+ $runsExpanded = "$($env:STDERRFAILURERUNS)" -replace ',',"`r`n"
+ Write-Host "##vso[task.logissue type=error;]StdErr file(s) indicate
test failures. Review the testresults artifacts for details. (click here to
view the projects that failed)`r`n$runsExpanded"
+ $failed = $true
+ }
+ if ([int]$maximumAllowedFailures -eq 0 -and $env:STDOUTFAILURE -eq 'true')
{
+ $runsExpanded = "$($env:STDOUTFAILURERUNS)" -replace ',',"`r`n"
+ Write-Host "##vso[task.logissue type=error;]StdOut file(s) indicate
test failures. Review the testresults artifacts for details. (click here to
view the projects that failed)`r`n$runsExpanded"
+ $failed = $true
+ }
if ($failed) {
Write-Host "##vso[task.complete result=Failed;]"
}
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 84a798402..2db143413 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -49,6 +49,7 @@ name: 'vNext$(rev:.r)' # Format for build number (will be
overridden)
# Verbose: 'false' (Optional - set to true for verbose logging output)
# Multiplier: '1' (Optional - the number of iterations to multiply applicable
tests by)
# DisplayFullName: 'true' (Optional - set to 'false' to display only the test
name instead of the full name with class and method)
+# FailOnTestFixtureOneTimeSetUpError: 'true' (Optional - set to 'false' to
allow tests to pass if the test fixture (class) has a OneTimeSetUp failure.)
# RunX86Tests: 'false' (Optional - set to 'true' to enable x86 tests)
@@ -162,6 +163,7 @@ stages:
$directory = if ($env:Directory -eq $null) { 'random' } else {
$env:Directory }
$verbose = if ($env:Verbose -eq 'true') { 'true' } else { 'false' }
$multiplier = if ($env:Multiplier -eq $null) { '1' } else {
$env:Multiplier }
+ $failOnTestFixtureOneTimeSetUpError = if
($env.FailOnTestFixtureOneTimeSetUpError -eq 'false') { 'false' } else { 'true'
}
$fileText = "{`n`t" +
"""assert"": ""$assert"",`n`t" +
"""tests"": {`n`t`t" +
@@ -174,7 +176,8 @@ stages:
"""postingsformat"": ""$postingsFormat"",`n`t`t" +
"""directory"": ""$directory"",`n`t`t" +
"""verbose"": ""$verbose"",`n`t`t" +
- """multiplier"": ""$multiplier""`n`t" +
+ """multiplier"": ""$multiplier"",`n`t`t" +
+ """failontestfixtureonetimesetuperror"":
""$failOnTestFixtureOneTimeSetUpError""`n`t" +
"}`n" +
"}"
Out-File -filePath
"$(Build.ArtifactStagingDirectory)/$(TestSettingsFileName)" -encoding UTF8
-inputObject $fileText
diff --git a/src/Lucene.Net.TestFramework/Search/RandomSimilarityProvider.cs
b/src/Lucene.Net.TestFramework/Search/RandomSimilarityProvider.cs
index e681de99a..6bebdcfc7 100644
--- a/src/Lucene.Net.TestFramework/Search/RandomSimilarityProvider.cs
+++ b/src/Lucene.Net.TestFramework/Search/RandomSimilarityProvider.cs
@@ -36,12 +36,12 @@ namespace Lucene.Net.Search
/// </summary>
public class RandomSimilarityProvider : PerFieldSimilarityWrapper
{
- internal readonly DefaultSimilarity defaultSim = new
DefaultSimilarity();
- internal readonly IList<Similarity> knownSims;
- internal IDictionary<string, Similarity> previousMappings = new
Dictionary<string, Similarity>();
- internal readonly int perFieldSeed;
- internal readonly int coordType; // 0 = no coord, 1 = coord, 2 = crazy
coord
- internal readonly bool shouldQueryNorm;
+ private readonly DefaultSimilarity defaultSim = new
DefaultSimilarity();
+ private readonly IList<Similarity> knownSims;
+ private readonly JCG.Dictionary<string, Similarity> previousMappings =
new JCG.Dictionary<string, Similarity>();
+ private readonly int perFieldSeed;
+ private readonly int coordType; // 0 = no coord, 1 = coord, 2 = crazy
coord
+ private readonly bool shouldQueryNorm;
public RandomSimilarityProvider(Random random)
{
@@ -162,7 +162,7 @@ namespace Lucene.Net.Search
else
sb.Append("crazy");
sb.Append("): ");
- sb.AppendFormat(J2N.Text.StringFormatter.InvariantCulture,
"{0}", previousMappings);
+ sb.Append(previousMappings);
return sb.ToString();
}
finally
@@ -171,4 +171,4 @@ namespace Lucene.Net.Search
}
}
}
-}
\ No newline at end of file
+}
diff --git
a/src/Lucene.Net.TestFramework/Support/Util/LuceneTestCase.SetUpFixture.cs
b/src/Lucene.Net.TestFramework/Support/Util/LuceneTestCase.SetUpFixture.cs
index 1f57513fc..620ba9b2d 100644
--- a/src/Lucene.Net.TestFramework/Support/Util/LuceneTestCase.SetUpFixture.cs
+++ b/src/Lucene.Net.TestFramework/Support/Util/LuceneTestCase.SetUpFixture.cs
@@ -70,7 +70,6 @@ namespace Lucene.Net.Util
[OneTimeSetUp]
public void OneTimeSetUpWrapper()
{
- //var fixture =
TestExecutionContext.CurrentContext.CurrentTest;
if (stackCount.GetAndIncrement() == 0)
{
// Set up for assembly
diff --git a/src/Lucene.Net.TestFramework/Support/Util/TestExtensions.cs
b/src/Lucene.Net.TestFramework/Support/Util/TestExtensions.cs
new file mode 100644
index 000000000..26f2d0439
--- /dev/null
+++ b/src/Lucene.Net.TestFramework/Support/Util/TestExtensions.cs
@@ -0,0 +1,90 @@
+using NUnit.Framework.Internal;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+#nullable enable
+
+namespace Lucene.Net.Util
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Extensions to <see cref="Test"/>.
+ /// </summary>
+ internal static class TestExtensions
+ {
+ /// <summary>
+ /// Mark the test and all descendents as Invalid (not runnable)
specifying a reason and an exception.
+ /// </summary>
+ /// <param name="test">This <see cref="Test"/>.</param>
+ /// <param name="reason">The reason the test is not runnable</param>
+ /// <exception cref="ArgumentNullException"><paramref name="test"/> or
<paramref name="reason"/> is <c>null</c>.</exception>
+ public static void MakeAllInvalid(this Test test, string reason)
+ => MakeAllInvalidInternal(test, null, reason);
+
+ /// <summary>
+ /// Mark the test and all descendents as Invalid (not runnable)
specifying a reason and an exception.
+ /// </summary>
+ /// <param name="test">This <see cref="Test"/>.</param>
+ /// <param name="exception">The exception that was the cause.</param>
+ /// <param name="reason">The reason the test is not runnable</param>
+ /// <exception cref="ArgumentNullException"><paramref name="test"/>,
<paramref name="exception"/> or <paramref name="reason"/> is
<c>null</c>.</exception>
+ public static void MakeAllInvalid(this Test test, Exception exception,
string reason)
+ {
+ if (exception is null)
+ throw new ArgumentNullException(nameof(exception));
+ MakeAllInvalidInternal(test, exception, reason);
+ }
+
+ private static void MakeAllInvalidInternal(this Test test, Exception?
exception, string reason)
+ {
+ if (test is null)
+ throw new ArgumentNullException(nameof(test));
+ if (reason is null)
+ throw new ArgumentNullException(nameof(reason));
+
+ if (exception is null)
+ test.MakeInvalid(reason);
+ else
+ test.MakeInvalid(exception, reason);
+
+ if (test.HasChildren)
+ {
+ var stack = new Stack<Test>(test.Tests.OfType<Test>());
+
+ while (stack.Count > 0)
+ {
+ var currentTest = stack.Pop();
+ if (exception is null)
+ currentTest.MakeInvalid(reason);
+ else
+ currentTest.MakeInvalid(exception, reason);
+
+ // Add children to the stack if they exist
+ if (currentTest.HasChildren)
+ {
+ foreach (var child in currentTest.Tests.OfType<Test>())
+ {
+ stack.Push(child);
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
b/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
index cf2d398d1..3d8b53dc0 100644
--- a/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
+++ b/src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs
@@ -505,6 +505,7 @@ namespace Lucene.Net.Util
TestSlow = SystemProperties.GetPropertyAsBoolean(SYSPROP_SLOW,
true); // LUCENENET specific - made default true, as per the docs
TestThrottling = TestNightly ? Throttling.SOMETIMES :
Throttling.NEVER;
LeaveTemporary = LoadLeaveTemorary();
+ FailOnTestFixtureOneTimeSetUpError =
SystemProperties.GetPropertyAsBoolean("tests:failontestfixtureonetimesetuperror",
true);
}
@@ -583,6 +584,16 @@ namespace Lucene.Net.Util
}
return defaultValue;
}
+
+ /// <summary>
+ /// Fail when <see cref="OneTimeSetUp()"/> throws an exception for
a given test fixture (usually a class).
+ /// By default, NUnit swallows exceptions when they happen here.
We explicitly report these errors
+ /// to standard error, but enabling this feature will also cause
all tests in the fixture and nested
+ /// tests to fail. Defaults to <c>false</c>.
+ /// <para/>
+ /// LUCENENET specific.
+ /// </summary>
+ public bool FailOnTestFixtureOneTimeSetUpError { get; }
}
@@ -658,6 +669,15 @@ namespace Lucene.Net.Util
/// Leave temporary files on disk, even on successful runs. </summary>
public static bool LeaveTemporary => TestProperties.LeaveTemporary;
+ /// <summary>
+ /// Fail when <see cref="OneTimeSetUp()"/> throws an exception for a
given test fixture (usually a class).
+ /// By default, NUnit swallows exceptions when they happen here. We
explicitly report these errors
+ /// to standard error, but enabling this feature will also cause all
tests in the fixture and nested
+ /// tests to fail. Defaults to <c>false</c>.
+ /// <para/>
+ /// LUCENENET specific.
+ /// </summary>
+ public static bool FailOnTestFixtureOneTimeSetUpError =>
TestProperties.FailOnTestFixtureOneTimeSetUpError;
// LUCENENET: Not Implemented
/////// <summary>
@@ -866,12 +886,6 @@ namespace Lucene.Net.Util
// Suite and test case setup/ cleanup.
// -----------------------------------------------------------------
- // LUCENENET specific: Temporary storage for random selections so they
- // can be set once per OneTimeSetUp and reused multiple times in SetUp
- // where they are written to the output.
- private string codecType;
- private string similarityName;
-
/// <summary>
/// For subclasses to override. Overrides must call
<c>base.SetUp()</c>.
/// </summary>
@@ -880,48 +894,6 @@ namespace Lucene.Net.Util
{
// LUCENENET TODO: Not sure how to convert these
//ParentChainCallRule.SetupCalled = true;
-
- // LUCENENET: Printing out randomized context regardless
- // of whether verbose is enabled (since we need it for debugging,
- // but the verbose output can crash tests).
- Console.Write("RandomSeed: ");
-
Console.WriteLine(RandomizedContext.CurrentContext.RandomSeedAsHex);
-
- Console.Write("Culture: ");
- Console.WriteLine(ClassEnvRule.locale.Name);
-
- Console.Write("Time Zone: ");
- Console.WriteLine(ClassEnvRule.timeZone.DisplayName);
-
- Console.Write("Default Codec: ");
- Console.Write(ClassEnvRule.codec.Name);
- Console.Write(" (");
- Console.Write(codecType);
- Console.WriteLine(")");
-
- Console.Write("Default Similarity: ");
- Console.WriteLine(similarityName);
-
- Console.Write("Nightly: ");
- Console.WriteLine(TestNightly);
-
- Console.Write("Weekly: ");
- Console.WriteLine(TestWeekly);
-
- Console.Write("Slow: ");
- Console.WriteLine(TestSlow);
-
- Console.Write("Awaits Fix: ");
- Console.WriteLine(TestAwaitsFix);
-
- Console.Write("Directory: ");
- Console.WriteLine(TestDirectory);
-
- Console.Write("Verbose: ");
- Console.WriteLine(Verbose);
-
- Console.Write("Random Multiplier: ");
- Console.WriteLine(RandomMultiplier);
}
/// <summary>
@@ -974,6 +946,26 @@ namespace Lucene.Net.Util
}
}
+ Fixture Test Values
+ =================
+
+ Random Seed:
{{RandomizedContext.CurrentContext.RandomSeedAsHex}}
+ Culture: {{ClassEnvRule.locale.Name}}
+ Time Zone:
{{ClassEnvRule.timeZone.DisplayName}}
+ Default Codec: {{ClassEnvRule.codec.Name}}
({{ClassEnvRule.codec.GetType().Name}})
+ Default Similarity: {{ClassEnvRule.similarity}}
+
+ System Properties
+ =================
+
+ Nightly: {{TestNightly}}
+ Weekly: {{TestWeekly}}
+ Slow: {{TestSlow}}
+ Awaits Fix: {{TestAwaitsFix}}
+ Directory: {{TestDirectory}}
+ Verbose: {{Verbose}}
+ Random Multiplier: {{RandomMultiplier}}
+
""";
result.SetResult(result.ResultState, message,
result.StackTrace);
@@ -994,15 +986,19 @@ namespace Lucene.Net.Util
try
{
ClassEnvRule.Before();
-
- // LUCENENET: Generate the info once so it can be printed out
for each test
- codecType = ClassEnvRule.codec.GetType().Name;
- similarityName = ClassEnvRule.similarity.ToString();
}
catch (Exception ex)
{
- // Write the stack trace so we have something to go on if an
error occurs here.
- throw new Exception($"An exception occurred during
OneTimeSetUp:\n{ex}", ex);
+ // This is a bug in the test framework that should be fixed
and/or reported if it occurs.
+ if (FailOnTestFixtureOneTimeSetUpError)
+ {
+ // LUCENENET: Patch NUnit so it will report all of the
tests in the class as a failure if we got an exception.
+
TestExecutionContext.CurrentContext.CurrentTest.MakeAllInvalid(ex, $"An
exception occurred during OneTimeSetUp:\n{ex}");
+ }
+ else
+ {
+ NUnit.Framework.TestContext.Error.WriteLine($"[ERROR] An
exception occurred during OneTimeSetUp:\n{ex}");
+ }
}
}
@@ -1019,12 +1015,20 @@ namespace Lucene.Net.Util
try
{
ClassEnvRule.After();
+ }
+ catch (Exception ex)
+ {
+ // LUCENENET: Patch NUnit so it will report a failure in
stderr if there was an exception during teardown.
+ NUnit.Framework.TestContext.Error.WriteLine($"[ERROR]
OneTimeTearDown: An exception occurred during ClassEnvRule.After():\n{ex}");
+ }
+ try
+ {
CleanupTemporaryFiles();
}
catch (Exception ex)
{
- // Write the stack trace so we have something to go on if an
error occurs here.
- throw new Exception($"An exception occurred during
OneTimeTearDown:\n{ex}", ex);
+ // LUCENENET: Patch NUnit so it will report a failure in
stderr if there was an exception during teardown.
+ NUnit.Framework.TestContext.Error.WriteLine($"[ERROR]
OneTimeTearDown: An exception occurred during CleanupTemporaryFiles():\n{ex}");
}
}
diff --git
a/src/Lucene.Net.Tests.AllProjects/Support/ExceptionHandling/TestExceptionExtensions.cs
b/src/Lucene.Net.Tests.AllProjects/Support/ExceptionHandling/TestExceptionExtensions.cs
index 1960c482b..8525684ac 100644
---
a/src/Lucene.Net.Tests.AllProjects/Support/ExceptionHandling/TestExceptionExtensions.cs
+++
b/src/Lucene.Net.Tests.AllProjects/Support/ExceptionHandling/TestExceptionExtensions.cs
@@ -387,7 +387,7 @@ namespace Lucene.Net.Support.ExceptionHandling
catch (Exception e) when (extensionMethodExpression(e))
{
// expected
- Assert.Pass($"Expected: Caught exception
{e.GetType().FullName}");
+ Assert.Pass();
}
}
catch (Exception e) when (e is not
NUnit.Framework.SuccessException)
@@ -410,7 +410,8 @@ namespace Lucene.Net.Support.ExceptionHandling
// Special case - need to suppress this from being thrown
to the outer catch
// or we will get a false failure
Assert.IsFalse(extensionMethodExpression(e));
- Assert.Pass($"Expected: Did not catch exception
{e.GetType().FullName}");
+ // expected
+ Assert.Pass();
}
catch (Exception e) when (extensionMethodExpression(e))
{
@@ -421,7 +422,7 @@ namespace Lucene.Net.Support.ExceptionHandling
catch (Exception e) when (e is not
NUnit.Framework.AssertionException)
{
// expected
- Assert.Pass($"Expected: Did not catch exception
{e.GetType().FullName}");
+ Assert.Pass();
}
}
}
