Diff
Modified: trunk/Tools/ChangeLog (124500 => 124501)
--- trunk/Tools/ChangeLog 2012-08-02 21:34:51 UTC (rev 124500)
+++ trunk/Tools/ChangeLog 2012-08-02 21:35:34 UTC (rev 124501)
@@ -1,3 +1,47 @@
+2012-08-02 Dirk Pranke <[email protected]>
+
+ test-webkitpy: clean up runner in preparation for running tests serially as necessary
+ https://bugs.webkit.org/show_bug.cgi?id=92922
+
+ Reviewed by Ojan Vafai.
+
+ In order to run some tests by themselves (serially, rather than
+ in parallel with other tests), we will need to be able to run
+ multiple test suites; this causes us to move loading the logic
+ for finding test method names out of the runner and into main.
+
+ I'm taking advantage of this to simplify some other stuff from
+ the runner as well; it is now very simple and doesn't expose its
+ dependency on unittest.TestResult at all (nor will the Printer
+ use TestResult).
+
+ Subsequent patches will move the custom loader from
+ port_testcase so that we can properly choose whether to run
+ integration tests and/or serial tests, and then update the
+ appropriate tests to run only serially.
+
+ * Scripts/webkitpy/test/main.py:
+ (Tester._run_tests):
+ (Tester):
+ (Tester._check_imports):
+ (Tester._test_names):
+ (Tester._all_test_names):
+ * Scripts/webkitpy/test/printer.py:
+ (Printer.__init__):
+ (Printer.write_update):
+ (Printer):
+ (Printer.print_finished_test):
+ (Printer.print_result):
+ * Scripts/webkitpy/test/runner.py:
+ (unit_test_name):
+ (Runner.__init__):
+ (Runner.run):
+ (Runner.handle):
+ (_Worker.handle):
+ * Scripts/webkitpy/test/runner_unittest.py:
+ (FakeLoader.loadTestsFromName):
+ (RunnerTest.test_run):
+
2012-08-02 Adam Barth <[email protected]>
Turn on tests for the mac-ews, for realz this time.
Modified: trunk/Tools/Scripts/webkitpy/test/main.py (124500 => 124501)
--- trunk/Tools/Scripts/webkitpy/test/main.py 2012-08-02 21:34:51 UTC (rev 124500)
+++ trunk/Tools/Scripts/webkitpy/test/main.py 2012-08-02 21:35:34 UTC (rev 124501)
@@ -29,13 +29,14 @@
import os
import StringIO
import sys
+import time
import traceback
import unittest
from webkitpy.common.system.filesystem import FileSystem
from webkitpy.test.finder import Finder
from webkitpy.test.printer import Printer
-from webkitpy.test.runner import Runner
+from webkitpy.test.runner import Runner, unit_test_name
_log = logging.getLogger(__name__)
@@ -128,10 +129,30 @@
# Make sure PYTHONPATH is set up properly.
sys.path = self.finder.additional_paths(sys.path) + sys.path
- _log.debug("Loading the tests...")
+ self.printer.write_update("Checking imports ...")
+ if not self._check_imports(names):
+ return False
+ self.printer.write_update("Finding the individual test methods ...")
loader = unittest.defaultTestLoader
- suites = []
+ test_names = self._test_names(loader, names)
+
+ self.printer.write_update("Running the tests ...")
+ self.printer.num_tests = len(test_names)
+ start = time.time()
+ test_runner = Runner(self.printer, loader)
+ test_runner.run(test_names, self._options.child_processes)
+
+ self.printer.print_result(time.time() - start)
+
+ if self._options.coverage:
+ cov.stop()
+ cov.save()
+ cov.report(show_missing=False)
+
+ return not self.printer.num_errors and not self.printer.num_failures
+
+ def _check_imports(self, names):
for name in names:
if self.finder.is_module(name):
# if we failed to load a name and it looks like a module,
@@ -143,19 +164,23 @@
_log.fatal('Failed to import %s:' % name)
self._log_exception()
return False
+ return True
- suites.append(loader.loadTestsFromName(name, None))
+ def _test_names(self, loader, names):
+ test_names = []
+ for name in names:
+ test_names.extend(self._all_test_names(loader.loadTestsFromName(name, None)))
- test_suite = unittest.TestSuite(suites)
- test_runner = Runner(self.printer, self._options, loader)
+ return test_names
- _log.debug("Running the tests.")
- result = test_runner.run(test_suite)
- if self._options.coverage:
- cov.stop()
- cov.save()
- cov.report(show_missing=False)
- return result.wasSuccessful()
+ def _all_test_names(self, suite):
+ names = []
+ if hasattr(suite, '_tests'):
+ for t in suite._tests:
+ names.extend(self._all_test_names(t))
+ else:
+ names.append(unit_test_name(suite))
+ return names
def _log_exception(self):
s = StringIO.StringIO()
Modified: trunk/Tools/Scripts/webkitpy/test/printer.py (124500 => 124501)
--- trunk/Tools/Scripts/webkitpy/test/printer.py 2012-08-02 21:34:51 UTC (rev 124500)
+++ trunk/Tools/Scripts/webkitpy/test/printer.py 2012-08-02 21:35:34 UTC (rev 124501)
@@ -37,6 +37,8 @@
self.options = options
self.num_tests = 0
self.num_completed = 0
+ self.num_errors = 0
+ self.num_failures = 0
self.running_tests = []
self.completed_tests = []
if options:
@@ -102,6 +104,9 @@
if self.options.pass_through:
outputcapture.OutputCapture.stream_wrapper = _CaptureAndPassThroughStream
+ def write_update(self, msg):
+ self.meter.write_update(msg)
+
def print_started_test(self, source, test_name):
self.running_tests.append(test_name)
if len(self.running_tests) > 1:
@@ -116,14 +121,16 @@
write(self._test_line(self.running_tests[0], suffix))
- def print_finished_test(self, result, test_name, test_time, failure, err):
+ def print_finished_test(self, source, test_name, test_time, failures, errors):
write = self.meter.writeln
- if failure:
- lines = failure[0][1].splitlines() + ['']
+ if failures:
+ lines = failures[0].splitlines() + ['']
suffix = ' failed:'
- elif err:
- lines = err[0][1].splitlines() + ['']
+ self.num_failures += 1
+ elif errors:
+ lines = errors[0].splitlines() + ['']
suffix = ' erred:'
+ self.num_errors += 1
else:
suffix = ' passed'
lines = []
@@ -154,13 +161,13 @@
def _test_line(self, test_name, suffix):
return '[%d/%d] %s%s' % (self.num_completed, self.num_tests, test_name, suffix)
- def print_result(self, result, run_time):
+ def print_result(self, run_time):
write = self.meter.writeln
- write('Ran %d test%s in %.3fs' % (result.testsRun, result.testsRun != 1 and "s" or "", run_time))
- if result.wasSuccessful():
+ write('Ran %d test%s in %.3fs' % (self.num_completed, self.num_completed != 1 and "s" or "", run_time))
+ if self.num_failures or self.num_errors:
+ write('FAILED (failures=%d, errors=%d)\n' % (self.num_failures, self.num_errors))
+ else:
write('\nOK\n')
- else:
- write('FAILED (failures=%d, errors=%d)\n' % (len(result.failures), len(result.errors)))
class _CaptureAndPassThroughStream(object):
Modified: trunk/Tools/Scripts/webkitpy/test/runner.py (124500 => 124501)
--- trunk/Tools/Scripts/webkitpy/test/runner.py 2012-08-02 21:34:51 UTC (rev 124500)
+++ trunk/Tools/Scripts/webkitpy/test/runner.py 2012-08-02 21:35:34 UTC (rev 124501)
@@ -22,61 +22,47 @@
"""code to actually run a list of python tests."""
-import logging
import re
import time
import unittest
from webkitpy.common import message_pool
-_log = logging.getLogger(__name__)
-
-
_test_description = re.compile("(\w+) \(([\w.]+)\)")
-def _test_name(test):
+def unit_test_name(test):
m = _test_description.match(str(test))
return "%s.%s" % (m.group(2), m.group(1))
class Runner(object):
- def __init__(self, printer, options, loader):
- self.options = options
+ def __init__(self, printer, loader):
self.printer = printer
self.loader = loader
- self.result = unittest.TestResult()
+ self.tests_run = 0
+ self.errors = []
+ self.failures = []
self.worker_factory = lambda caller: _Worker(caller, self.loader)
- def all_test_names(self, suite):
- names = []
- if hasattr(suite, '_tests'):
- for t in suite._tests:
- names.extend(self.all_test_names(t))
- else:
- names.append(_test_name(suite))
- return names
+ def run(self, test_names, num_workers):
+ if not test_names:
+ return
+ num_workers = min(num_workers, len(test_names))
+ with message_pool.get(self, self.worker_factory, num_workers) as pool:
+ pool.run(('test', test_name) for test_name in test_names)
- def run(self, suite):
- run_start_time = time.time()
- all_test_names = self.all_test_names(suite)
- self.printer.num_tests = len(all_test_names)
-
- with message_pool.get(self, self.worker_factory, int(self.options.child_processes)) as pool:
- pool.run(('test', test_name) for test_name in all_test_names)
-
- self.printer.print_result(self.result, time.time() - run_start_time)
- return self.result
-
- def handle(self, message_name, source, test_name, delay=None, result=None):
+ def handle(self, message_name, source, test_name, delay=None, failures=None, errors=None):
if message_name == 'started_test':
self.printer.print_started_test(source, test_name)
return
- self.result.testsRun += 1
- self.result.errors.extend(result.errors)
- self.result.failures.extend(result.failures)
- self.printer.print_finished_test(source, test_name, delay, result.failures, result.errors)
+ self.tests_run += 1
+ if failures:
+ self.failures.append((test_name, failures))
+ if errors:
+ self.errors.append((test_name, errors))
+ self.printer.print_finished_test(source, test_name, delay, failures, errors)
class _Worker(object):
@@ -89,13 +75,8 @@
result = unittest.TestResult()
start = time.time()
self._caller.post('started_test', test_name)
- self._loader.loadTestsFromName(test_name, None).run(result)
- # The tests in the TestResult contain file objects and other unpicklable things; we only
- # care about the test name, so we rewrite the result to replace the test with the test name.
- # FIXME: We need an automated test for this, but I don't know how to write an automated
- # test that will fail in this case that doesn't get picked up by test-webkitpy normally :(.
- result.failures = [(_test_name(failure[0]), failure[1]) for failure in result.failures]
- result.errors = [(_test_name(error[0]), error[1]) for error in result.errors]
-
- self._caller.post('finished_test', test_name, time.time() - start, result)
+ # We will need to rework this if a test_name results in multiple tests.
+ self._loader.loadTestsFromName(test_name, None).run(result)
+ self._caller.post('finished_test', test_name, time.time() - start,
+ [failure[1] for failure in result.failures], [error[1] for error in result.errors])
Modified: trunk/Tools/Scripts/webkitpy/test/runner_unittest.py (124500 => 124501)
--- trunk/Tools/Scripts/webkitpy/test/runner_unittest.py 2012-08-02 21:34:51 UTC (rev 124500)
+++ trunk/Tools/Scripts/webkitpy/test/runner_unittest.py 2012-08-02 21:35:34 UTC (rev 124501)
@@ -65,7 +65,7 @@
def top_suite(self):
return FakeTopSuite(self._tests)
- def loadTestsFromName(self, name, dummy):
+ def loadTestsFromName(self, name, _):
return FakeModuleSuite(*self._results[name])
@@ -84,29 +84,18 @@
for handler in self.log_handlers:
handler.level = self.log_levels.pop(0)
- def assert_run(self, verbose=0, timing=False, child_processes=1, quiet=False):
+ def test_run(self, verbose=0, timing=False, child_processes=1, quiet=False):
options = MockOptions(verbose=verbose, timing=timing, child_processes=child_processes, quiet=quiet, pass_through=False)
stream = StringIO.StringIO()
loader = FakeLoader(('test1 (Foo)', '.', ''),
('test2 (Foo)', 'F', 'test2\nfailed'),
('test3 (Foo)', 'E', 'test3\nerred'))
- runner = Runner(Printer(stream, options), options, loader)
- result = runner.run(loader.top_suite())
- self.assertFalse(result.wasSuccessful())
- self.assertEquals(result.testsRun, 3)
- self.assertEquals(len(result.failures), 1)
- self.assertEquals(len(result.errors), 1)
- # FIXME: check the output from the test
+ runner = Runner(Printer(stream, options), loader)
+ runner.run(['Foo.test1', 'Foo.test2', 'Foo.test3'], 1)
+ self.assertEquals(runner.tests_run, 3)
+ self.assertEquals(len(runner.failures), 1)
+ self.assertEquals(len(runner.errors), 1)
- def test_regular(self):
- self.assert_run()
- def test_verbose(self):
- self.assert_run(verbose=1)
-
- def test_timing(self):
- self.assert_run(timing=True)
-
-
if __name__ == '__main__':
unittest.main()
