Revision: 22143
Author: [email protected]
Date: Wed Jul 2 08:15:44 2014 UTC
Log: Let test runner rerun failures to test for flakes.
When enabled, this dynamically adds jobs that had failures back to the
pool. Special json output for flakes will be handled in a separate CL.
BUG=374134
LOG=n
[email protected]
Review URL: https://codereview.chromium.org/360113003
http://code.google.com/p/v8/source/detail?r=22143
Modified:
/branches/bleeding_edge/tools/run-deopt-fuzzer.py
/branches/bleeding_edge/tools/run-tests.py
/branches/bleeding_edge/tools/testrunner/local/execution.py
/branches/bleeding_edge/tools/testrunner/objects/context.py
/branches/bleeding_edge/tools/testrunner/objects/testcase.py
=======================================
--- /branches/bleeding_edge/tools/run-deopt-fuzzer.py Wed May 28 13:00:47
2014 UTC
+++ /branches/bleeding_edge/tools/run-deopt-fuzzer.py Wed Jul 2 08:15:44
2014 UTC
@@ -369,9 +369,11 @@
timeout, options.isolates,
options.command_prefix,
options.extra_flags,
- False,
+ False, # Keep i18n on by default.
options.random_seed,
- True)
+ True, # No sorting of test cases.
+ 0, # Don't rerun failing tests.
+ 0) # No use of a rerun-failing-tests maximum.
# Find available test suites and read test cases from them.
variables = {
=======================================
--- /branches/bleeding_edge/tools/run-tests.py Tue Jun 24 05:27:44 2014 UTC
+++ /branches/bleeding_edge/tools/run-tests.py Wed Jul 2 08:15:44 2014 UTC
@@ -181,6 +181,13 @@
default=False, action="store_true")
result.add_option("--json-test-results",
help="Path to a file for storing json results.")
+ result.add_option("--rerun-failures-count",
+ help=("Number of times to rerun each failing test
case. "
+ "Very slow tests will be rerun only once."),
+ default=0, type="int")
+ result.add_option("--rerun-failures-max",
+ help="Maximum number of failing test cases to rerun.",
+ default=100, type="int")
result.add_option("--shard-count",
help="Split testsuites into this number of shards",
default=1, type="int")
@@ -416,7 +423,9 @@
options.extra_flags,
options.no_i18n,
options.random_seed,
- options.no_sorting)
+ options.no_sorting,
+ options.rerun_failures_count,
+ options.rerun_failures_max)
# TODO(all): Combine "simulator" and "simulator_run".
simulator_run = not options.dont_skip_simulator_slow_tests and \
=======================================
--- /branches/bleeding_edge/tools/testrunner/local/execution.py Fri Jun 27
09:51:08 2014 UTC
+++ /branches/bleeding_edge/tools/testrunner/local/execution.py Wed Jul 2
08:15:44 2014 UTC
@@ -81,6 +81,7 @@
self.remaining = num_tests
self.failed = []
self.crashed = 0
+ self.reran_tests = 0
def _RunPerfSafe(self, fun):
try:
@@ -89,6 +90,42 @@
print("PerfData exception: %s" % e)
self.perf_failures = True
+ def _GetJob(self, test):
+ command = self.GetCommand(test)
+ timeout = self.context.timeout
+ if ("--stress-opt" in test.flags or
+ "--stress-opt" in self.context.mode_flags or
+ "--stress-opt" in self.context.extra_flags):
+ timeout *= 4
+ if test.dependency is not None:
+ dep_command = [ c.replace(test.path, test.dependency) for c in
command ]
+ else:
+ dep_command = None
+ return Job(command, dep_command, test.id, timeout,
self.context.verbose)
+
+ def _MaybeRerun(self, pool, test):
+ if test.run <= self.context.rerun_failures_count:
+ # Possibly rerun this test if its run count is below the maximum per
+ # test.
+ if test.run == 1:
+ # Count the overall number of reran tests on the first rerun.
+ if self.reran_tests < self.context.rerun_failures_max:
+ self.reran_tests += 1
+ else:
+ # Don't rerun this if the overall number of rerun tests has been
+ # reached.
+ return
+ if test.run >= 2 and test.duration > self.context.timeout / 20:
+ # Rerun slow tests at most once.
+ return
+
+ # Rerun this test.
+ test.duration = None
+ test.output = None
+ test.run += 1
+ pool.add([self._GetJob(test)])
+ self.remaining += 1
+
def Run(self, jobs):
self.indicator.Starting()
self._RunInternal(jobs)
@@ -109,23 +146,12 @@
assert test.id >= 0
test_map[test.id] = test
try:
- command = self.GetCommand(test)
+ queue.append([self._GetJob(test)])
except Exception, e:
# If this failed, save the exception and re-raise it later (after
# all other tests have had a chance to run).
queued_exception = e
continue
- timeout = self.context.timeout
- if ("--stress-opt" in test.flags or
- "--stress-opt" in self.context.mode_flags or
- "--stress-opt" in self.context.extra_flags):
- timeout *= 4
- if test.dependency is not None:
- dep_command = [ c.replace(test.path, test.dependency) for c in
command ]
- else:
- dep_command = None
- job = Job(command, dep_command, test.id, timeout,
self.context.verbose)
- queue.append([job])
try:
it = pool.imap_unordered(RunTest, queue)
for result in it:
@@ -143,6 +169,9 @@
self.succeeded += 1
self.remaining -= 1
self.indicator.HasRun(test, has_unexpected_output)
+ if has_unexpected_output:
+ # Rerun test failures after the indicator has processed the
results.
+ self._MaybeRerun(pool, test)
finally:
pool.terminate()
self._RunPerfSafe(lambda: self.perf_data_manager.close())
=======================================
--- /branches/bleeding_edge/tools/testrunner/objects/context.py Wed May 28
10:49:10 2014 UTC
+++ /branches/bleeding_edge/tools/testrunner/objects/context.py Wed Jul 2
08:15:44 2014 UTC
@@ -29,7 +29,7 @@
class Context():
def __init__(self, arch, mode, shell_dir, mode_flags, verbose, timeout,
isolates, command_prefix, extra_flags, noi18n, random_seed,
- no_sorting):
+ no_sorting, rerun_failures_count, rerun_failures_max):
self.arch = arch
self.mode = mode
self.shell_dir = shell_dir
@@ -42,15 +42,18 @@
self.noi18n = noi18n
self.random_seed = random_seed
self.no_sorting = no_sorting
+ self.rerun_failures_count = rerun_failures_count
+ self.rerun_failures_max = rerun_failures_max
def Pack(self):
return [self.arch, self.mode, self.mode_flags, self.timeout,
self.isolates,
self.command_prefix, self.extra_flags, self.noi18n,
- self.random_seed, self.no_sorting]
+ self.random_seed, self.no_sorting, self.rerun_failures_count,
+ self.rerun_failures_max]
@staticmethod
def Unpack(packed):
# For the order of the fields, refer to Pack() above.
return Context(packed[0], packed[1], None, packed[2], False,
packed[3], packed[4], packed[5], packed[6], packed[7],
- packed[8], packed[9])
+ packed[8], packed[9], packed[10], packed[11])
=======================================
--- /branches/bleeding_edge/tools/testrunner/objects/testcase.py Mon Sep 24
09:38:46 2012 UTC
+++ /branches/bleeding_edge/tools/testrunner/objects/testcase.py Wed Jul 2
08:15:44 2014 UTC
@@ -38,6 +38,7 @@
self.output = None
self.id = None # int, used to map result back to TestCase instance
self.duration = None # assigned during execution
+ self.run = 1 # The nth time this test is executed.
def CopyAddingFlags(self, flags):
copy = TestCase(self.suite, self.path, self.flags + flags,
self.dependency)
@@ -60,6 +61,7 @@
test = TestCase(str(task[0]), task[1], task[2], task[3])
test.outcomes = set(task[4])
test.id = task[5]
+ test.run = 1
return test
def SetSuiteObject(self, suites):
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
