This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "CMake".
The branch, next has been updated
via 13af2b0fdf0695e57434a1280a3516639923d19c (commit)
via 2bf0dec52a41c7eca81e5202c8036832c13ee8ba (commit)
via 83620f005466912adc850cc84abb38e62ea988e0 (commit)
from 53cbf3a06bca2772947ccfa960523d09caa59285 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=13af2b0fdf0695e57434a1280a3516639923d19c
commit 13af2b0fdf0695e57434a1280a3516639923d19c
Merge: 53cbf3a 2bf0dec
Author: Brad King <brad.k...@kitware.com>
AuthorDate: Tue Mar 17 11:43:24 2015 -0400
Commit: CMake Topic Stage <kwro...@kitware.com>
CommitDate: Tue Mar 17 11:43:24 2015 -0400
Merge topic 'ctest-repeat-until-fail' into next
2bf0dec5 Help: Add notes for topic 'ctest-repeat-until-fail'
83620f00 ctest: Add a new --repeat-until-fail option
http://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=2bf0dec52a41c7eca81e5202c8036832c13ee8ba
commit 2bf0dec52a41c7eca81e5202c8036832c13ee8ba
Author: Brad King <brad.k...@kitware.com>
AuthorDate: Tue Mar 17 11:39:43 2015 -0400
Commit: Brad King <brad.k...@kitware.com>
CommitDate: Tue Mar 17 11:41:24 2015 -0400
Help: Add notes for topic 'ctest-repeat-until-fail'
diff --git a/Help/release/dev/ctest-repeat-until-fail.rst
b/Help/release/dev/ctest-repeat-until-fail.rst
new file mode 100644
index 0000000..8a679c6
--- /dev/null
+++ b/Help/release/dev/ctest-repeat-until-fail.rst
@@ -0,0 +1,5 @@
+ctest-repeat-until-fail
+-----------------------
+
+* The :manual:`ctest(1)` tool learned a new ``--repeat-until-fail <n>``
+ option to help find sporadic test failures.
http://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=83620f005466912adc850cc84abb38e62ea988e0
commit 83620f005466912adc850cc84abb38e62ea988e0
Author: Bill Hoffman <bill.hoff...@kitware.com>
AuthorDate: Thu Mar 5 16:51:10 2015 -0500
Commit: Brad King <brad.k...@kitware.com>
CommitDate: Tue Mar 17 11:41:10 2015 -0400
ctest: Add a new --repeat-until-fail option
This option tells ctest to run each test N times until the test fails or
the N times have run. This is useful for finding random failing tests.
diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst
index cc132c2..dd3bcfb 100644
--- a/Help/manual/ctest.1.rst
+++ b/Help/manual/ctest.1.rst
@@ -194,6 +194,11 @@ Options
subsequent calls to ctest with the --rerun-failed option will run
the set of tests that most recently failed (if any).
+``--repeat-until-fail <n>``
+ Require each test to run ``<n>`` times without failing in order to pass.
+
+ This is useful in finding sporadic failures in test cases.
+
``--max-width <width>``
Set the max width for a test name to output
diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx
b/Source/CTest/cmCTestMultiProcessHandler.cxx
index eb33d8e..bd090db 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -121,6 +121,11 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test)
this->RunningCount += GetProcessorsUsed(test);
cmCTestRunTest* testRun = new cmCTestRunTest(this->TestHandler);
+ if(this->CTest->GetRepeatUntilFail())
+ {
+ testRun->SetRunUntilFailOn();
+ testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
+ }
testRun->SetIndex(test);
testRun->SetTestProperties(this->Properties[test]);
@@ -289,7 +294,13 @@ bool cmCTestMultiProcessHandler::CheckOutput()
cmCTestRunTest* p = *i;
int test = p->GetIndex();
- if(p->EndTest(this->Completed, this->Total, true))
+ bool testResult = p->EndTest(this->Completed, this->Total, true);
+ if(p->StartAgain())
+ {
+ this->Completed--; // remove the completed test because run again
+ continue;
+ }
+ if(testResult)
{
this->Passed->push_back(p->GetTestProperties()->Name);
}
diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx
index 03131fd..6f72684 100644
--- a/Source/CTest/cmCTestRunTest.cxx
+++ b/Source/CTest/cmCTestRunTest.cxx
@@ -33,6 +33,9 @@ cmCTestRunTest::cmCTestRunTest(cmCTestTestHandler* handler)
this->CompressedOutput = "";
this->CompressionRatio = 2;
this->StopTimePassed = false;
+ this->NumberOfRunsLeft = 1; // default to 1 run of the test
+ this->RunUntilFail = false; // default to run the test once
+ this->RunAgain = false; // default to not having to run again
}
cmCTestRunTest::~cmCTestRunTest()
@@ -357,13 +360,50 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t
total, bool started)
this->MemCheckPostProcess();
this->ComputeWeightedCost();
}
- // Always push the current TestResult onto the
+ // If the test does not need to rerun push the current TestResult onto the
// TestHandler vector
- this->TestHandler->TestResults.push_back(this->TestResult);
+ if(!this->NeedsToRerun())
+ {
+ this->TestHandler->TestResults.push_back(this->TestResult);
+ }
delete this->TestProcess;
return passed;
}
+bool cmCTestRunTest::StartAgain()
+{
+ if(!this->RunAgain)
+ {
+ return false;
+ }
+ this->RunAgain = false; // reset
+ // change to tests directory
+ std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
+ cmSystemTools::ChangeDirectory(this->TestProperties->Directory);
+ this->StartTest(this->TotalNumberOfTests);
+ // change back
+ cmSystemTools::ChangeDirectory(current_dir);
+ return true;
+}
+
+bool cmCTestRunTest::NeedsToRerun()
+{
+ this->NumberOfRunsLeft--;
+ if(this->NumberOfRunsLeft == 0)
+ {
+ return false;
+ }
+ // if number of runs left is not 0, and we are running until
+ // we find a failed test, then return true so the test can be
+ // restarted
+ if(this->RunUntilFail
+ && this->TestResult.Status == cmCTestTestHandler::COMPLETED)
+ {
+ this->RunAgain = true;
+ return true;
+ }
+ return false;
+}
//----------------------------------------------------------------------
void cmCTestRunTest::ComputeWeightedCost()
{
@@ -400,6 +440,7 @@ void cmCTestRunTest::MemCheckPostProcess()
// Starts the execution of a test. Returns once it has started
bool cmCTestRunTest::StartTest(size_t total)
{
+ this->TotalNumberOfTests = total; // save for rerun case
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(2*getNumWidth(total) + 8)
<< "Start "
<< std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
@@ -494,10 +535,10 @@ bool cmCTestRunTest::StartTest(size_t total)
//----------------------------------------------------------------------
void cmCTestRunTest::ComputeArguments()
{
+ this->Arguments.clear(); // reset becaue this might be a rerun
std::vector<std::string>::const_iterator j =
this->TestProperties->Args.begin();
++j; // skip test name
-
// find the test executable
if(this->TestHandler->MemCheck)
{
@@ -682,10 +723,28 @@ bool cmCTestRunTest::ForkProcess(double testTimeOut, bool
explicitTimeout,
void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
{
- cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
- << completed << "/");
- cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
- << total << " ");
+ // if this is the last or only run of this test
+ // then print out completed / total
+ // Only issue is if a test fails and we are running until fail
+ // then it will never print out the completed / total, same would
+ // got for run until pass. Trick is when this is called we don't
+ // yet know if we are passing or failing.
+ if(this->NumberOfRunsLeft == 1)
+ {
+ cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+ << completed << "/");
+ cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+ << total << " ");
+ }
+ // if this is one of several runs of a test just print blank space
+ // to keep things neat
+ else
+ {
+ cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+ << " " << " ");
+ cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+ << " " << " ");
+ }
if ( this->TestHandler->MemCheck )
{
diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h
index 476f3e1..3b5c831 100644
--- a/Source/CTest/cmCTestRunTest.h
+++ b/Source/CTest/cmCTestRunTest.h
@@ -27,6 +27,8 @@ public:
cmCTestRunTest(cmCTestTestHandler* handler);
~cmCTestRunTest();
+ void SetNumberOfRuns(int n) {this->NumberOfRunsLeft = n;}
+ void SetRunUntilFailOn() { this->RunUntilFail = true;}
void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties * prop)
{ this->TestProperties = prop; }
@@ -58,7 +60,10 @@ public:
void ComputeArguments();
void ComputeWeightedCost();
+
+ bool StartAgain();
private:
+ bool NeedsToRerun();
void DartProcessing();
void ExeNotFound(std::string exe);
// Figures out a final timeout which is min(STOP_TIME, NOW+TIMEOUT)
@@ -92,6 +97,10 @@ private:
std::string ActualCommand;
std::vector<std::string> Arguments;
bool StopTimePassed;
+ bool RunUntilFail;
+ int NumberOfRunsLeft;
+ bool RunAgain;
+ size_t TotalNumberOfTests;
};
inline int getNumWidth(size_t n)
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index e6d3960..6946a20 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -328,6 +328,8 @@ cmCTest::cmCTest()
this->OutputTestOutputOnTestFailure = false;
this->ComputedCompressTestOutput = false;
this->ComputedCompressMemCheckOutput = false;
+ this->RepeatTests = 1; // default to run each test once
+ this->RepeatUntilFail = false;
if(cmSystemTools::GetEnv("CTEST_OUTPUT_ON_FAILURE"))
{
this->OutputTestOutputOnTestFailure = true;
@@ -1983,11 +1985,11 @@ bool cmCTest::CheckArgument(const std::string& arg,
const char* varg1,
//----------------------------------------------------------------------
// Processes one command line argument (and its arguments if any)
// for many simple options and then returns
-void cmCTest::HandleCommandLineArguments(size_t &i,
- std::vector<std::string> &args)
+bool cmCTest::HandleCommandLineArguments(size_t &i,
+ std::vector<std::string> &args,
+ std::string& errormsg)
{
std::string arg = args[i];
-
if(this->CheckArgument(arg, "-F"))
{
this->Failover = true;
@@ -2005,6 +2007,27 @@ void cmCTest::HandleCommandLineArguments(size_t &i,
this->SetParallelLevel(plevel);
this->ParallelLevelSetInCli = true;
}
+ if(this->CheckArgument(arg, "--repeat-until-fail"))
+ {
+ if( i >= args.size() - 1)
+ {
+ errormsg = "'--repeat-until-fail' requires an argument";
+ return false;
+ }
+ i++;
+ long repeat = 1;
+ if(!cmSystemTools::StringToLong(args[i].c_str(), &repeat))
+ {
+ errormsg = "'--repeat-until-fail' given non-integer value '"
+ + args[i] + "'";
+ return false;
+ }
+ this->RepeatTests = repeat;
+ if(repeat > 1)
+ {
+ this->RepeatUntilFail = true;
+ }
+ }
if(this->CheckArgument(arg, "--no-compress-output"))
{
@@ -2190,6 +2213,7 @@ void cmCTest::HandleCommandLineArguments(size_t &i,
this->GetHandler("test")->SetPersistentOption("RerunFailed", "true");
this->GetHandler("memcheck")->SetPersistentOption("RerunFailed", "true");
}
+ return true;
}
//----------------------------------------------------------------------
@@ -2272,7 +2296,12 @@ int cmCTest::Run(std::vector<std::string> &args,
std::string* output)
for(size_t i=1; i < args.size(); ++i)
{
// handle the simple commandline arguments
- this->HandleCommandLineArguments(i,args);
+ std::string errormsg;
+ if(!this->HandleCommandLineArguments(i,args, errormsg))
+ {
+ cmSystemTools::Error(errormsg.c_str());
+ return 1;
+ }
// handle the script arguments -S -SR -SP
this->HandleScriptArguments(i,args,SRArgumentSpecified);
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index 88191c4..3f033d9 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -429,8 +429,13 @@ public:
{
return this->Definitions;
}
-
+ // return the number of times a test should be run
+ int GetTestRepeat() { return this->RepeatTests;}
+ // return true if test should run until fail
+ bool GetRepeatUntilFail() { return this->RepeatUntilFail;}
private:
+ int RepeatTests;
+ bool RepeatUntilFail;
std::string ConfigType;
std::string ScheduleType;
std::string StopTime;
@@ -535,8 +540,9 @@ private:
bool AddVariableDefinition(const std::string &arg);
//! parse and process most common command line arguments
- void HandleCommandLineArguments(size_t &i,
- std::vector<std::string> &args);
+ bool HandleCommandLineArguments(size_t &i,
+ std::vector<std::string> &args,
+ std::string& errormsg);
//! hande the -S -SP and -SR arguments
void HandleScriptArguments(size_t &i,
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index c0eb8ac..0fc47b7 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -75,6 +75,8 @@ static const char * cmDocumentationOptions[][2] =
"Run a specific number of tests by number."},
{"-U, --union", "Take the Union of -I and -R"},
{"--rerun-failed", "Run only the tests that failed previously"},
+ {"--repeat-until-fail <n>", "Require each test to run <n> "
+ "times without failing in order to pass"},
{"--max-width <width>", "Set the max width for a test name to output"},
{"--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1."},
{"--no-label-summary", "Disable timing summary information for labels."},
diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt
index 08765de..4300c7e 100644
--- a/Tests/CMakeLists.txt
+++ b/Tests/CMakeLists.txt
@@ -2583,6 +2583,13 @@ ${CMake_BINARY_DIR}/bin/cmake -DDIR=dev -P
${CMake_SOURCE_DIR}/Utilities/Release
ADD_TEST_MACRO(CTestTestSerialOrder ${CMAKE_CTEST_COMMAND}
--output-on-failure -C "\${CTestTest_CONFIG}")
+ add_test_macro(CTestTestRunUntilFail ${CMAKE_CTEST_COMMAND}
+ --repeat-until-fail 4 -C "\${CTestTest_CONFIG}")
+ set_tests_properties(CTestTestRunUntilFail PROPERTIES
+ PASS_REGULAR_EXPRESSION
+ "1/2.*Test #1:.* initialization.*Passed.*Test #2.*test1.*Failed"
+ )
+
if(NOT BORLAND)
set(CTestLimitDashJ_CTEST_OPTIONS --force-new-ctest-process)
add_test_macro(CTestLimitDashJ ${CMAKE_CTEST_COMMAND} -j 4
diff --git a/Tests/CTestTestRunUntilFail/CMakeLists.txt
b/Tests/CTestTestRunUntilFail/CMakeLists.txt
new file mode 100644
index 0000000..62f4814
--- /dev/null
+++ b/Tests/CTestTestRunUntilFail/CMakeLists.txt
@@ -0,0 +1,13 @@
+cmake_minimum_required(VERSION 2.8.12)
+project(CTestTestRunUntilFail)
+enable_testing()
+set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
+add_test(NAME initialization
+ COMMAND ${CMAKE_COMMAND}
+ "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+ -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
+add_test(NAME test1
+ COMMAND ${CMAKE_COMMAND}
+ "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+ -P "${CMAKE_CURRENT_SOURCE_DIR}/test1.cmake")
+set_tests_properties(test1 PROPERTIES DEPENDS "initialization")
diff --git a/Tests/CTestTestRunUntilFail/init.cmake
b/Tests/CTestTestRunUntilFail/init.cmake
new file mode 100644
index 0000000..560fb5e
--- /dev/null
+++ b/Tests/CTestTestRunUntilFail/init.cmake
@@ -0,0 +1 @@
+file(WRITE "${TEST_OUTPUT_FILE}" "0")
diff --git a/Tests/CTestTestRunUntilFail/test1.cmake
b/Tests/CTestTestRunUntilFail/test1.cmake
new file mode 100644
index 0000000..d7fafb8
--- /dev/null
+++ b/Tests/CTestTestRunUntilFail/test1.cmake
@@ -0,0 +1,8 @@
+message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
+file(READ "${TEST_OUTPUT_FILE}" COUNT)
+message("COUNT= ${COUNT}")
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
+if(${COUNT} EQUAL 2)
+ message(FATAL_ERROR "this test fails on the 2nd run")
+endif()
diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt
index eb42057..ffda31f 100644
--- a/Tests/RunCMake/CMakeLists.txt
+++ b/Tests/RunCMake/CMakeLists.txt
@@ -198,6 +198,7 @@ add_RunCMake_test(CommandLine)
add_RunCMake_test(install)
add_RunCMake_test(CPackInstallProperties)
add_RunCMake_test(ExternalProject)
+add_RunCMake_test(CTestCommandLine)
set(IfacePaths_INCLUDE_DIRECTORIES_ARGS -DTEST_PROP=INCLUDE_DIRECTORIES)
add_RunCMake_test(IfacePaths_INCLUDE_DIRECTORIES TEST_DIR IfacePaths)
diff --git a/Tests/RunCMake/CTestCommandLine/CMakeLists.txt
b/Tests/RunCMake/CTestCommandLine/CMakeLists.txt
new file mode 100644
index 0000000..2843a51
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/CMakeLists.txt
@@ -0,0 +1,5 @@
+cmake_minimum_required(VERSION 3.0)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)
+add_test(hello ${CMAKE_COMMAND} -E echo hello)
+add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)
diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
new file mode 100644
index 0000000..7be07c1
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -0,0 +1,8 @@
+include(RunCMake)
+
+run_cmake_command(repeat-until-fail-bad1 ${CMAKE_CTEST_COMMAND}
+ --repeat-until-fail)
+run_cmake_command(repeat-until-fail-bad2 ${CMAKE_CTEST_COMMAND}
+ --repeat-until-fail foo)
+run_cmake_command(repeat-until-fail-good ${CMAKE_CTEST_COMMAND}
+ --repeat-until-fail 2)
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-result.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-stderr.txt
new file mode 100644
index 0000000..5ea8816
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad1-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--repeat-until-fail' requires an argument$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-result.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-stderr.txt
new file mode 100644
index 0000000..a79faae
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-bad2-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--repeat-until-fail' given non-integer value 'foo'$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-good-stderr.txt
b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-good-stderr.txt
new file mode 100644
index 0000000..a7c4b11
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-good-stderr.txt
@@ -0,0 +1 @@
+^No tests were found!!!$
-----------------------------------------------------------------------
Summary of changes:
hooks/post-receive
--
CMake
_______________________________________________
Cmake-commits mailing list
Cmake-commits@cmake.org
http://public.kitware.com/mailman/listinfo/cmake-commits
