Diff
Modified: trunk/Tools/ChangeLog (240648 => 240649)
--- trunk/Tools/ChangeLog 2019-01-29 10:37:06 UTC (rev 240648)
+++ trunk/Tools/ChangeLog 2019-01-29 10:38:41 UTC (rev 240649)
@@ -1,3 +1,27 @@
+2019-01-29 Carlos Garcia Campos <[email protected]>
+
+ WebDriver: add support for running subtests
+ https://bugs.webkit.org/show_bug.cgi?id=193904
+
+ Reviewed by Michael Catanzaro.
+
+ pytest already supports it by passing test.py::subtest, so we can do the same.
+
+ * Scripts/webkitpy/webdriver_tests/pytest_runner.py:
+ (get_item_name): Made this global.
+ (CollectRecorder.__init__): Receive the parameter to ignore.
+ (CollectRecorder.pytest_collectreport): Collect also the subtests.
+ (TestExpectationsMarker.pytest_collection_modifyitems): Use get_item_name().
+ (collect): Add parameter to ignore.
+ * Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py:
+ (WebDriverSeleniumExecutor.collect): Pass the driver name as parameter to ignore.
+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py:
+ (WebDriverTestRunnerSelenium.collect_tests): Handle subtest name in test path.
+ (WebDriverTestRunnerSelenium.run): Ditto.
+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
+ (WebDriverTestRunnerW3C.collect_tests): Ditto.
+ (WebDriverTestRunnerW3C.run): Ditto.
+
2019-01-28 Ryosuke Niwa <[email protected]>
User agent string override for navigator.userAgent should be site specific quirks
Modified: trunk/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py (240648 => 240649)
--- trunk/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py 2019-01-29 10:37:06 UTC (rev 240648)
+++ trunk/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py 2019-01-29 10:38:41 UTC (rev 240649)
@@ -50,14 +50,32 @@
raise
+def get_item_name(item, ignore_param):
+ if ignore_param is None:
+ return item.name
+
+ single_param = '[%s]' % ignore_param
+ if item.name.endswith(single_param):
+ return item.name[:-len(single_param)]
+
+ param = '[%s-' % ignore_param
+ if param in item.name:
+ return item.name.replace('%s-' % ignore_param, '')
+
+ return item.name
+
+
class CollectRecorder(object):
- def __init__(self):
- self.tests = []
+ def __init__(self, ignore_param):
+ self._ignore_param = ignore_param
+ self.tests = {}
def pytest_collectreport(self, report):
if report.nodeid:
- self.tests.append(report.nodeid)
+ self.tests.setdefault(report.nodeid, [])
+ for subtest in report.result:
+ self.tests[report.nodeid].append(get_item_name(subtest, self._ignore_param))
class HarnessResultRecorder(object):
@@ -136,24 +154,10 @@
self._ignore_param = ignore_param
self._base_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests')
- def _item_name(self, item):
- if self._ignore_param is None:
- return item.name
-
- single_param = '[%s]' % self._ignore_param
- if item.name.endswith(single_param):
- return item.name[:-len(single_param)]
-
- param = '[%s-' % self._ignore_param
- if param in item.name:
- return item.name.replace('%s-' % self._ignore_param, '')
-
- return item.name
-
def pytest_collection_modifyitems(self, session, config, items):
for item in items:
test = os.path.relpath(str(item.fspath), self._base_dir)
- item_name = self._item_name(item)
+ item_name = get_item_name(item, self._ignore_param)
if self._expectations.is_slow(test, item_name):
item.add_marker(pytest.mark.timeout(self._timeout * 5))
expected = self._expectations.get_expectation(test, item_name)[0]
@@ -165,8 +169,8 @@
item.add_marker(pytest.mark.skip)
-def collect(directory, args):
- collect_recorder = CollectRecorder()
+def collect(directory, args, ignore_param=None):
+ collect_recorder = CollectRecorder(ignore_param)
stdout = sys.stdout
with open(os.devnull, 'wb') as devnull:
sys.stdout = devnull
Modified: trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py (240648 => 240649)
--- trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py 2019-01-29 10:37:06 UTC (rev 240648)
+++ trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py 2019-01-29 10:38:41 UTC (rev 240649)
@@ -57,7 +57,7 @@
do_delayed_imports()
def collect(self, directory):
- return pytest_runner.collect(directory, self._args)
+ return pytest_runner.collect(directory, self._args, self._driver_name)
def run(self, test, timeout, expectations):
return pytest_runner.run(test, self._args, timeout, self._env, expectations, self._driver_name)
Modified: trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py (240648 => 240649)
--- trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py 2019-01-29 10:37:06 UTC (rev 240648)
+++ trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py 2019-01-29 10:38:41 UTC (rev 240649)
@@ -51,18 +51,29 @@
executor = WebDriverSeleniumExecutor(self._driver, self._env)
# Collected tests are relative to test directory.
base_dir = os.path.join(self._tests_dir, os.path.dirname(relative_tests_dir))
- collected_tests = [os.path.join(base_dir, test) for test in executor.collect(os.path.join(self._tests_dir, relative_tests_dir))]
+ collected_tests = {}
+ for test, subtests in executor.collect(os.path.join(self._tests_dir, relative_tests_dir)).iteritems():
+ collected_tests[os.path.join(base_dir, test)] = subtests
selenium_tests = []
if not tests:
tests = [relative_tests_dir]
for test in tests:
+ subtest = None
+ if '::' in test:
+ test, subtest = test.split('::')
+
if not test.startswith(relative_tests_dir):
continue
+
test_path = os.path.join(self._tests_dir, test)
if os.path.isdir(test_path):
selenium_tests.extend([test for test in collected_tests if test.startswith(test_path) and test not in skipped])
elif test_path in collected_tests and test_path not in skipped:
- selenium_tests.append(test_path)
+ if subtest is not None:
+ if subtest in collected_tests[test_path]:
+ selenium_tests.append(test_path + '::' + subtest)
+ else:
+ selenium_tests.append(test_path)
return selenium_tests
def run(self, tests=[]):
@@ -72,7 +83,7 @@
executor = WebDriverSeleniumExecutor(self._driver, self._env)
timeout = self._port.get_option('timeout')
for test in tests:
- test_name = os.path.relpath(test, self._tests_dir)
+ test_name = os.path.relpath(test.split('::')[0], self._tests_dir)
harness_result, test_results = executor.run(test, timeout, self._expectations)
result = WebDriverTestResult(test_name, *harness_result)
if harness_result[0] == 'OK':
Modified: trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py (240648 => 240649)
--- trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py 2019-01-29 10:37:06 UTC (rev 240648)
+++ trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py 2019-01-29 10:38:41 UTC (rev 240649)
@@ -52,13 +52,21 @@
if not tests:
tests = [relative_tests_dir]
for test in tests:
+ subtest = None
+ if '::' in test:
+ test, subtest = test.split('::')
+
if not test.startswith(relative_tests_dir):
continue
+
test_path = os.path.join(self._tests_dir, test)
if os.path.isdir(test_path):
w3c_tests.extend(self._scan_directory(test_path, skipped))
elif self._is_test(test_path) and test_path not in skipped:
- w3c_tests.append(test_path)
+ if subtest is not None:
+ w3c_tests.append(test_path + '::' + subtest)
+ else:
+ w3c_tests.append(test_path)
return w3c_tests
def _is_test(self, test):
@@ -91,7 +99,7 @@
need_restart = False
try:
for test in tests:
- test_name = os.path.relpath(test, self._tests_dir)
+ test_name = os.path.relpath(test.split('::')[0], self._tests_dir)
harness_result, test_results = executor.run(test)
result = WebDriverTestResult(test_name, *harness_result)
if harness_result[0] == 'OK':
