Diff
Modified: trunk/Tools/ChangeLog (135652 => 135653)
--- trunk/Tools/ChangeLog 2012-11-24 16:46:03 UTC (rev 135652)
+++ trunk/Tools/ChangeLog 2012-11-24 17:15:11 UTC (rev 135653)
@@ -1,3 +1,90 @@
+2012-11-24 Zan Dobersek <[email protected]>
+
+ Add the --order option to NRWT
+ https://bugs.webkit.org/show_bug.cgi?id=102229
+
+ Reviewed by Dirk Pranke.
+
+ The --order option controls in what order the gathered tests will be run.
+ There are 3 options:
+ - natural, the tests are sorted in natural order,
+ - random, the tests are sorted in random order,
+ - none, the tests are kept in the order in which they were specified through
+ arguments or the test list.
+
+ The natural option is the default one and maintains the current behavior.
+
+ The none option is primarily intended to be used with the --test-list option,
+ with the test list file expected to list specific test names.
+ When a directory is specified as one of the paths from under which tests should
+ be gathered and the none option is used, that path will expand to all the tests
+ under it, with these tests sorted in the natural order.
+
+ Using --order=random duplicated the behavior of the --randomize-order option, so
+ the latter is removed.
+
+ This patch also incorporates the change with which makes it is possible to specify
+ a test via arguments or the test list multiple tests and then have the test run
+ that many times.
+
+ * Scripts/webkitpy/common/find_files.py:
+ (find): Accept an optional sorting key that is used to sort files found in the directories
+ that the given paths specify.
+ (_normalized_find): Both paths that are then searched for the tests and the tests that
+ were found are now kept in lists. This makes it possible to run a test multiple times
+ by specifying it multiple times through arguments or the test list file.
+ (_normalized_find.sort_by_directory_key):
+ * Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py:
+ (LayoutTestFinder.find_tests): Keep the paths where we should search for tests in a list.
+ (LayoutTestFinder.skip_tests): Subtracting from a set of paths is necessary.
+ * Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py:
+ (LayoutTestRunner.__init__): The Sharder class no longer needs a test path separator.
+ (Sharder.__init__): Ditto.
+ (Sharder._resize_shards):
+ * Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py:
+ (SharderTests.get_shards): Modify the Shader initializing as required.
+ (SharderTests.test_multiple_locked_shards): The natural sorting and test name key unit tests
+ are now moved to the base Port unit tests.
+ * Scripts/webkitpy/layout_tests/controllers/manager.py:
+ (Manager._prepare_lists): Keep the test names in a list. Order that list as specified by
+ the new --order option.
+ * Scripts/webkitpy/layout_tests/port/base.py:
+ (Port.tests): This method should now return a list of all the tests found in the specified
+ paths.
+ (Port._real_tests): A list is required, so no conversion to a set is made.
+ (Port):
+ (Port.test_key): Place this method here for now, but it should really move into a separate
+ module.
+ (Port._natural_sort_key): Ditto.
+ (Port._natural_sort_key.tryint):
+ (Port._virtual_tests): A list is required, so the function now operates on that rather than
+ on a set.
+ * Scripts/webkitpy/layout_tests/port/base_unittest.py:
+ (PortTest.test_find_with_skipped_directories_2): Adjust the test case to check for an empty
+ list rather than an empty set.
+ (PortTest.test_dont_require_http_server):
+ (NaturalCompareTest): The natural sorting unit tests are placed here for now.
+ (NaturalCompareTest.assert_cmp):
+ (NaturalCompareTest.test_natural_compare):
+ (KeyCompareTest): The test name key unit tests are placed here for now.
+ (KeyCompareTest.assert_cmp):
+ (KeyCompareTest.test_test_key):
+ * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+ (parse_args): Remove the --randomize-order option, it's now possible to replicate its behavior
+ by using --order=random.
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py: Add unit tests covering
+ various aspects of this change.
+ (MainTest.test_natural_order):
+ (MainTest):
+ (MainTest.test_natural_order_test_specified_multiple_times):
+ (MainTest.test_random_order):
+ (MainTest.test_random_order_test_specified_multiple_times):
+ (MainTest.test_no_order):
+ (MainTest.test_no_order_test_specified_multiple_times):
+ (MainTest.test_no_order_with_directory_entries_in_natural_order):
+ * Scripts/webkitpy/tool/commands/queries.py:
+ (PrintExpectations.execute): Operate on a set of tests when printing out expectations.
+
2012-11-24 Sheriff Bot <[email protected]>
Unreviewed, rolling out r135600.
Modified: trunk/Tools/Scripts/webkitpy/common/find_files.py (135652 => 135653)
--- trunk/Tools/Scripts/webkitpy/common/find_files.py 2012-11-24 16:46:03 UTC (rev 135652)
+++ trunk/Tools/Scripts/webkitpy/common/find_files.py 2012-11-24 17:15:11 UTC (rev 135653)
@@ -44,8 +44,10 @@
will be included into the result if the callback returns True.
The callback has to take three arguments: filesystem, dirname and filename."""
+import itertools
-def find(filesystem, base_dir, paths=None, skipped_directories=None, file_filter=None):
+
+def find(filesystem, base_dir, paths=None, skipped_directories=None, file_filter=None, directory_sort_key=None):
"""Finds the set of tests under a given list of sub-paths.
Args:
@@ -57,34 +59,27 @@
paths = paths or ['*']
skipped_directories = skipped_directories or set(['.svn', '_svn'])
- return _normalized_find(filesystem, _normalize(filesystem, base_dir, paths), skipped_directories, file_filter)
+ return _normalized_find(filesystem, _normalize(filesystem, base_dir, paths), skipped_directories, file_filter, directory_sort_key)
def _normalize(filesystem, base_dir, paths):
return [filesystem.normpath(filesystem.join(base_dir, path)) for path in paths]
-def _normalized_find(filesystem, paths, skipped_directories, file_filter):
+def _normalized_find(filesystem, paths, skipped_directories, file_filter, directory_sort_key):
"""Finds the set of tests under the list of paths.
Args:
paths: a list of absolute path expressions to search.
Glob patterns are ok.
"""
- paths_to_walk = set()
- for path in paths:
- # If there's an * in the name, assume it's a glob pattern.
- if path.find('*') > -1:
- filenames = filesystem.glob(path)
- paths_to_walk.update(filenames)
- else:
- paths_to_walk.add(path)
+ paths_to_walk = itertools.chain(*(filesystem.glob(path) for path in paths))
- # FIXME: I'm not sure there's much point in this being a set. A list would probably be faster.
- all_files = set()
- for path in paths_to_walk:
- files = filesystem.files_under(path, skipped_directories, file_filter)
- all_files.update(set(files))
+ def sort_by_directory_key(files_list):
+ if directory_sort_key:
+ files_list.sort(key=directory_sort_key)
+ return files_list
+ all_files = itertools.chain(*(sort_by_directory_key(filesystem.files_under(path, skipped_directories, file_filter)) for path in paths_to_walk))
return all_files
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py (135652 => 135653)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py 2012-11-24 16:46:03 UTC (rev 135652)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py 2012-11-24 17:15:11 UTC (rev 135653)
@@ -47,7 +47,6 @@
paths = self._strip_test_dir_prefixes(args)
if options.test_list:
paths += self._strip_test_dir_prefixes(self._read_test_names_from_file(options.test_list, self._port.TEST_PATH_SEPARATOR))
- paths = set(paths)
test_files = self._port.tests(paths)
return (paths, test_files)
@@ -108,7 +107,7 @@
tests_to_skip = set()
elif self._options.skipped != 'always':
# make sure we're explicitly running any tests passed on the command line; equivalent to 'default'.
- tests_to_skip -= paths
+ tests_to_skip -= set(paths)
# unless of course we don't want to run the HTTP tests :)
if not self._options.http:
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py (135652 => 135653)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py 2012-11-24 16:46:03 UTC (rev 135652)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py 2012-11-24 17:15:11 UTC (rev 135653)
@@ -68,7 +68,7 @@
self._results_directory = results_directory
self._expectations = None
self._test_is_slow = test_is_slow_fn
- self._sharder = Sharder(self._port.split_test, self._port.TEST_PATH_SEPARATOR, self._options.max_locked_shards)
+ self._sharder = Sharder(self._port.split_test, self._options.max_locked_shards)
self._current_result_summary = None
self._needs_http = None
@@ -80,9 +80,6 @@
self._worker_stats = {}
self._filesystem = self._port.host.filesystem
- def test_key(self, test_name):
- return self._sharder.test_key(test_name)
-
def run_tests(self, test_inputs, expectations, result_summary, num_workers, needs_http, needs_websockets, retrying):
"""Returns a tuple of (interrupted, keyboard_interrupted, thread_timings, test_timings, individual_test_timings):
interrupted is whether the run was interrupted
@@ -474,9 +471,8 @@
class Sharder(object):
- def __init__(self, test_split_fn, test_path_separator, max_locked_shards):
+ def __init__(self, test_split_fn, max_locked_shards):
self._split = test_split_fn
- self._sep = test_path_separator
self._max_locked_shards = max_locked_shards
def shard_tests(self, test_inputs, num_workers, fully_parallel):
@@ -603,29 +599,3 @@
some_shards, remaining_shards = split_at(remaining_shards, num_old_per_new)
new_shards.append(TestShard('%s_%d' % (shard_name_prefix, len(new_shards) + 1), extract_and_flatten(some_shards)))
return new_shards
-
- def test_key(self, test_name):
- """Turns a test name into a list with two sublists, the natural key of the
- dirname, and the natural key of the basename.
-
- This can be used when sorting paths so that files in a directory.
- directory are kept together rather than being mixed in with files in
- subdirectories."""
- dirname, basename = self._split(test_name)
- return (self.natural_sort_key(dirname + self._sep), self.natural_sort_key(basename))
-
- @staticmethod
- def natural_sort_key(string_to_split):
- """ Turns a string into a list of string and number chunks, i.e. "z23a" -> ["z", 23, "a"]
-
- This can be used to implement "natural sort" order. See:
- http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html
- http://nedbatchelder.com/blog/200712.html#e20071211T054956
- """
- def tryint(val):
- try:
- return int(val)
- except ValueError:
- return val
-
- return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)]
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py (135652 => 135653)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py 2012-11-24 16:46:03 UTC (rev 135652)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py 2012-11-24 17:15:11 UTC (rev 135653)
@@ -31,13 +31,15 @@
import unittest
from webkitpy.common.host_mock import MockHost
+from webkitpy.common.system.systemhost_mock import MockSystemHost
from webkitpy.layout_tests import run_webkit_tests
+from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner, Sharder, TestRunInterruptedException
from webkitpy.layout_tests.models import test_expectations
from webkitpy.layout_tests.models import test_failures
from webkitpy.layout_tests.models.result_summary import ResultSummary
from webkitpy.layout_tests.models.test_input import TestInput
from webkitpy.layout_tests.models.test_results import TestResult
-from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner, Sharder, TestRunInterruptedException
+from webkitpy.layout_tests.port.test import TestPort
TestExpectations = test_expectations.TestExpectations
@@ -235,12 +237,8 @@
return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf')))
def get_shards(self, num_workers, fully_parallel, test_list=None, max_locked_shards=1):
- def split(test_name):
- idx = test_name.rfind('/')
- if idx != -1:
- return (test_name[0:idx], test_name[idx + 1:])
-
- self.sharder = Sharder(split, '/', max_locked_shards)
+ port = TestPort(MockSystemHost())
+ self.sharder = Sharder(port.split_test, max_locked_shards)
test_list = test_list or self.test_list
return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel)
@@ -335,48 +333,3 @@
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'])])
-
-
-class NaturalCompareTest(unittest.TestCase):
- def assert_cmp(self, x, y, result):
- self.assertEqual(cmp(Sharder.natural_sort_key(x), Sharder.natural_sort_key(y)), result)
-
- def test_natural_compare(self):
- self.assert_cmp('a', 'a', 0)
- self.assert_cmp('ab', 'a', 1)
- self.assert_cmp('a', 'ab', -1)
- self.assert_cmp('', '', 0)
- self.assert_cmp('', 'ab', -1)
- self.assert_cmp('1', '2', -1)
- self.assert_cmp('2', '1', 1)
- self.assert_cmp('1', '10', -1)
- self.assert_cmp('2', '10', -1)
- self.assert_cmp('foo_1.html', 'foo_2.html', -1)
- self.assert_cmp('foo_1.1.html', 'foo_2.html', -1)
- self.assert_cmp('foo_1.html', 'foo_10.html', -1)
- self.assert_cmp('foo_2.html', 'foo_10.html', -1)
- self.assert_cmp('foo_23.html', 'foo_10.html', 1)
- self.assert_cmp('foo_23.html', 'foo_100.html', -1)
-
-
-class KeyCompareTest(unittest.TestCase):
- def setUp(self):
- def split(test_name):
- idx = test_name.rfind('/')
- if idx != -1:
- return (test_name[0:idx], test_name[idx + 1:])
-
- self.sharder = Sharder(split, '/', 1)
-
- def assert_cmp(self, x, y, result):
- self.assertEqual(cmp(self.sharder.test_key(x), self.sharder.test_key(y)), result)
-
- def test_test_key(self):
- self.assert_cmp('/a', '/a', 0)
- self.assert_cmp('/a', '/b', -1)
- self.assert_cmp('/a2', '/a10', -1)
- self.assert_cmp('/a2/foo', '/a10/foo', -1)
- self.assert_cmp('/a/foo11', '/a/foo2', 1)
- self.assert_cmp('/ab', '/a/a/b', -1)
- self.assert_cmp('/a/a/b', '/ab', 1)
- self.assert_cmp('/foo-bar/baz', '/foo/baz', -1)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (135652 => 135653)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2012-11-24 16:46:03 UTC (rev 135652)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2012-11-24 17:15:11 UTC (rev 135653)
@@ -305,14 +305,14 @@
def _prepare_lists(self):
tests_to_skip = self._finder.skip_tests(self._paths, self._test_names, self._expectations, self._http_tests())
- self._test_names = list(set(self._test_names) - tests_to_skip)
+ self._test_names = [test for test in self._test_names if test not in tests_to_skip]
# Create a sorted list of test files so the subset chunk,
# if used, contains alphabetically consecutive tests.
- if self._options.randomize_order:
+ if self._options.order == 'natural':
+ self._test_names.sort(key=self._port.test_key)
+ elif self._options.order == 'random':
random.shuffle(self._test_names)
- else:
- self._test_names.sort(key=self._runner.test_key)
self._test_names, tests_in_other_chunks = self._finder.split_into_chunks(self._test_names)
self._expectations.add_skipped_tests(tests_in_other_chunks)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py (135652 => 135653)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py 2012-11-24 16:46:03 UTC (rev 135652)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py 2012-11-24 17:15:11 UTC (rev 135653)
@@ -578,7 +578,9 @@
def tests(self, paths):
"""Return the list of tests found. Both generic and platform-specific tests matching paths should be returned."""
expanded_paths = self._expanded_paths(paths)
- return self._real_tests(expanded_paths).union(self._virtual_tests(expanded_paths, self.populated_virtual_test_suites()))
+ tests = self._real_tests(expanded_paths)
+ tests.extend(self._virtual_tests(expanded_paths, self.populated_virtual_test_suites()))
+ return tests
def _expanded_paths(self, paths):
expanded_paths = []
@@ -596,8 +598,8 @@
def _real_tests(self, paths):
# When collecting test cases, skip these directories
skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests', 'reference', 'reftest'])
- files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories, Port._is_test_file)
- return set([self.relative_test_filename(f) for f in files])
+ files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories, Port._is_test_file, self.test_key)
+ return [self.relative_test_filename(f) for f in files]
# When collecting test cases, we include any file with these extensions.
_supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl',
@@ -623,6 +625,31 @@
def _is_test_file(filesystem, dirname, filename):
return Port._has_supported_extension(filesystem, filename) and not Port.is_reference_html_file(filesystem, dirname, filename)
+ def test_key(self, test_name):
+ """Turns a test name into a list with two sublists, the natural key of the
+ dirname, and the natural key of the basename.
+
+ This can be used when sorting paths so that files in a directory.
+ directory are kept together rather than being mixed in with files in
+ subdirectories."""
+ dirname, basename = self.split_test(test_name)
+ return (self._natural_sort_key(dirname + self.TEST_PATH_SEPARATOR), self._natural_sort_key(basename))
+
+ def _natural_sort_key(self, string_to_split):
+ """ Turns a string into a list of string and number chunks, i.e. "z23a" -> ["z", 23, "a"]
+
+ This can be used to implement "natural sort" order. See:
+ http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html
+ http://nedbatchelder.com/blog/200712.html#e20071211T054956
+ """
+ def tryint(val):
+ try:
+ return int(val)
+ except ValueError:
+ return val
+
+ return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)]
+
def test_dirs(self):
"""Returns the list of top-level test directories."""
layout_tests_dir = self.layout_tests_dir()
@@ -1321,14 +1348,14 @@
return suites
def _virtual_tests(self, paths, suites):
- virtual_tests = set()
+ virtual_tests = list()
for suite in suites:
if paths:
for test in suite.tests:
if any(test.startswith(p) for p in paths):
- virtual_tests.add(test)
+ virtual_tests.append(test)
else:
- virtual_tests.update(set(suite.tests.keys()))
+ virtual_tests.extend(suite.tests.keys())
return virtual_tests
def lookup_virtual_test_base(self, test_name):
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py (135652 => 135653)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py 2012-11-24 16:46:03 UTC (rev 135652)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py 2012-11-24 17:15:11 UTC (rev 135653)
@@ -325,7 +325,7 @@
def test_find_with_skipped_directories_2(self):
port = self.make_port(with_tests=True)
tests = port.tests(['userscripts/resources'])
- self.assertEqual(tests, set([]))
+ self.assertEqual(tests, [])
def test_is_test_file(self):
filesystem = MockFileSystem()
@@ -455,5 +455,49 @@
port = self.make_port()
self.assertEqual(port.requires_http_server(), False)
+
+class NaturalCompareTest(unittest.TestCase):
+ def setUp(self):
+ self._port = TestPort(MockSystemHost())
+
+ def assert_cmp(self, x, y, result):
+ self.assertEqual(cmp(self._port._natural_sort_key(x), self._port._natural_sort_key(y)), result)
+
+ def test_natural_compare(self):
+ self.assert_cmp('a', 'a', 0)
+ self.assert_cmp('ab', 'a', 1)
+ self.assert_cmp('a', 'ab', -1)
+ self.assert_cmp('', '', 0)
+ self.assert_cmp('', 'ab', -1)
+ self.assert_cmp('1', '2', -1)
+ self.assert_cmp('2', '1', 1)
+ self.assert_cmp('1', '10', -1)
+ self.assert_cmp('2', '10', -1)
+ self.assert_cmp('foo_1.html', 'foo_2.html', -1)
+ self.assert_cmp('foo_1.1.html', 'foo_2.html', -1)
+ self.assert_cmp('foo_1.html', 'foo_10.html', -1)
+ self.assert_cmp('foo_2.html', 'foo_10.html', -1)
+ self.assert_cmp('foo_23.html', 'foo_10.html', 1)
+ self.assert_cmp('foo_23.html', 'foo_100.html', -1)
+
+
+class KeyCompareTest(unittest.TestCase):
+ def setUp(self):
+ self._port = TestPort(MockSystemHost())
+
+ def assert_cmp(self, x, y, result):
+ self.assertEqual(cmp(self._port.test_key(x), self._port.test_key(y)), result)
+
+ def test_test_key(self):
+ self.assert_cmp('/a', '/a', 0)
+ self.assert_cmp('/a', '/b', -1)
+ self.assert_cmp('/a2', '/a10', -1)
+ self.assert_cmp('/a2/foo', '/a10/foo', -1)
+ self.assert_cmp('/a/foo11', '/a/foo2', 1)
+ self.assert_cmp('/ab', '/a/a/b', -1)
+ self.assert_cmp('/a/a/b', '/ab', 1)
+ self.assert_cmp('/foo-bar/baz', '/foo/baz', -1)
+
+
if __name__ == '__main__':
unittest.main()
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py (135652 => 135653)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py 2012-11-24 16:46:03 UTC (rev 135652)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py 2012-11-24 17:15:11 UTC (rev 135653)
@@ -353,9 +353,11 @@
help="Run all tests, even those marked SKIP in the test list (same as --skipped=ignore)"),
optparse.make_option("--time-out-ms",
help="Set the timeout for each test"),
- optparse.make_option("--randomize-order", action=""
- default=False, help=("Run tests in random order (useful "
- "for tracking down corruption)")),
+ optparse.make_option("--order", action="" default="natural",
+ help=("determine the order in which the test cases will be run. "
+ "'none' == use the order in which the tests were listed either in arguments or test list, "
+ "'natural' == use the natural order (default), "
+ "'random' == randomize the test order.")),
optparse.make_option("--run-chunk",
help=("Run a specified chunk (n:l), the nth of len l, "
"of the layout tests")),
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (135652 => 135653)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-11-24 16:46:03 UTC (rev 135652)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-11-24 17:15:11 UTC (rev 135653)
@@ -389,10 +389,42 @@
self.assertEmpty(out)
self.assertContains(err, 'No tests to run.\n')
- def test_randomize_order(self):
- # FIXME: verify order was shuffled
- self.assertTrue(passing_run(['--randomize-order']))
+ def test_natural_order(self):
+ tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html']
+ tests_run = get_tests_run(['--order=natural'] + tests_to_run, tests_included=True, flatten_batches=True)
+ self.assertEqual(['failures/expected/missing_text.html', 'failures/expected/text.html', 'passes/args.html', 'passes/audio.html'], tests_run)
+ def test_natural_order_test_specified_multiple_times(self):
+ tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html']
+ tests_run = get_tests_run(['--order=natural'] + tests_to_run, tests_included=True, flatten_batches=True)
+ self.assertEqual(['passes/args.html', 'passes/args.html', 'passes/audio.html', 'passes/audio.html'], tests_run)
+
+ def test_random_order(self):
+ tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html']
+ tests_run = get_tests_run(['--order=random'] + tests_to_run, tests_included=True, flatten_batches=True)
+ self.assertEqual(sorted(tests_to_run), sorted(tests_run))
+
+ def test_random_order_test_specified_multiple_times(self):
+ tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html']
+ tests_run = get_tests_run(['--order=random'] + tests_to_run, tests_included=True, flatten_batches=True)
+ self.assertEqual(tests_run.count('passes/audio.html'), 2)
+ self.assertEqual(tests_run.count('passes/args.html'), 2)
+
+ def test_no_order(self):
+ tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html']
+ tests_run = get_tests_run(['--order=none'] + tests_to_run, tests_included=True, flatten_batches=True)
+ self.assertEqual(tests_to_run, tests_run)
+
+ def test_no_order_test_specified_multiple_times(self):
+ tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html']
+ tests_run = get_tests_run(['--order=none'] + tests_to_run, tests_included=True, flatten_batches=True)
+ self.assertEqual(tests_to_run, tests_run)
+
+ def test_no_order_with_directory_entries_in_natural_order(self):
+ tests_to_run = ['http/tests/ssl', 'perf/foo', 'http/tests/passes']
+ tests_run = get_tests_run(['--order=none'] + tests_to_run, tests_included=True, flatten_batches=True)
+ self.assertEqual(tests_run, ['http/tests/ssl/text.html', 'perf/foo/test.html', 'http/tests/passes/image.html', 'http/tests/passes/text.html'])
+
def test_gc_between_tests(self):
self.assertTrue(passing_run(['--gc-between-tests']))
Modified: trunk/Tools/Scripts/webkitpy/tool/commands/queries.py (135652 => 135653)
--- trunk/Tools/Scripts/webkitpy/tool/commands/queries.py 2012-11-24 16:46:03 UTC (rev 135652)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/queries.py 2012-11-24 17:15:11 UTC (rev 135653)
@@ -476,7 +476,7 @@
print file
return
- tests = default_port.tests(args)
+ tests = set(default_port.tests(args))
for port_name in port_names:
model = self._model(options, port_name, tests)
tests_to_print = self._filter_tests(options, model, tests)
