Diff
Modified: releases/WebKitGTK/webkit-2.18/Tools/ChangeLog (227499 => 227500)
--- releases/WebKitGTK/webkit-2.18/Tools/ChangeLog 2018-01-24 10:01:53 UTC (rev 227499)
+++ releases/WebKitGTK/webkit-2.18/Tools/ChangeLog 2018-01-24 10:02:01 UTC (rev 227500)
@@ -1,3 +1,41 @@
+2018-01-17 Carlos Garcia Campos <cgar...@igalia.com>
+
+ WebDriver: add support for test expectations
+ https://bugs.webkit.org/show_bug.cgi?id=180420
+
+ Reviewed by Carlos Alberto Lopez Perez.
+
+ Add support for parsing test expectations from a JSON file and mark tests on collection accordingly.
+
+ * Scripts/run-webdriver-tests: Get the retval from process_results().
+ * Scripts/webkitpy/thirdparty/__init__.py:
+ (AutoinstallImportHook._install_pytest): Install also py because pytest needs it.
+ * Scripts/webkitpy/webdriver_tests/pytest_runner.py:
+ (TestExpectationsMarker): Plugin to mark tests based on given expectations.
+ (TestExpectationsMarker.__init__): Initialize expectations.
+ (TestExpectationsMarker.pytest_collection_modifyitems): Mark tests if needed,
+ (run): Create and use TestExpectationsMarker plugin.
+ * Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py:
+ (WebDriverSeleniumExecutor.run): Pass expectations to pytest_runner.
+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py:
+ (WebDriverTestRunner.__init__): Create a TestExpectations and pass it to the runners.
+ (WebDriverTestRunner.run): Do not count results here.
+ (WebDriverTestRunner.process_results): Rename print_results() as process_results() since it now returns the
+ amount of failures. Printing the test summary while processing results will be made optional in a follow up
+ patch.
+ (WebDriverTestRunner.process_results.report): Return the amount of failures.
+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py:
+ (WebDriverTestRunnerSelenium.__init__): Initialize _expectations.
+ (WebDriverTestRunnerSelenium.collect_tests): Do not include skipped tests.
+ (WebDriverTestRunnerSelenium.run): Stop returning the tests count.
+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
+ (WebDriverTestRunnerW3C.__init__): Initialize _expectations.
+ (WebDriverTestRunnerW3C.collect_tests): Do not include skipped tests.
+ (WebDriverTestRunnerW3C._scan_directory): Ditto.
+ (WebDriverTestRunnerW3C.run): Stop returning the tests count.
+ * Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py:
+ (WebDriverW3CExecutor.run): Pass expectations to pytest_runner.
+
2018-01-15 Carlos Garcia Campos <cgar...@igalia.com>
[GTK][WPE] Add support for unit test expectations
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/run-webdriver-tests (227499 => 227500)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/run-webdriver-tests 2018-01-24 10:01:53 UTC (rev 227499)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/run-webdriver-tests 2018-01-24 10:02:01 UTC (rev 227500)
@@ -65,8 +65,8 @@
port._display_server = options.display_server
runner = WebDriverTestRunner(port)
-retval = runner.run(args)
-runner.print_results()
+runner.run(args)
+retval = runner.process_results()
if options.json_output is not None:
runner.dump_results_to_json_file(options.json_output)
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/thirdparty/__init__.py (227499 => 227500)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/thirdparty/__init__.py 2018-01-24 10:01:53 UTC (rev 227499)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/thirdparty/__init__.py 2018-01-24 10:02:01 UTC (rev 227500)
@@ -142,6 +142,8 @@
"pytest-timeout-1.2.0/pytest_timeout.py")
def _install_pytest(self):
+ self._install("https://pypi.python.org/packages/90/e3/e075127d39d35f09a500ebb4a90afd10f9ef0a1d28a6d09abeec0e444fdd/py-1.5.2.tar.gz#md5=279ca69c632069e1b71e11b14641ca28",
+ "py-1.5.2/py")
self._install("https://pypi.python.org/packages/1f/f8/8cd74c16952163ce0db0bd95fdd8810cbf093c08be00e6e665ebf0dc3138/pytest-3.2.5.tar.gz#md5=6dbe9bb093883f75394a689a1426ac6f",
"pytest-3.2.5/_pytest")
self._install("https://pypi.python.org/packages/1f/f8/8cd74c16952163ce0db0bd95fdd8810cbf093c08be00e6e665ebf0dc3138/pytest-3.2.5.tar.gz#md5=6dbe9bb093883f75394a689a1426ac6f",
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py (227499 => 227500)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py 2018-01-24 10:01:53 UTC (rev 227499)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py 2018-01-24 10:02:01 UTC (rev 227500)
@@ -27,6 +27,8 @@
import sys
import tempfile
+from webkitpy.common.system.filesystem import FileSystem
+from webkitpy.common.webkit_finder import WebKitFinder
import webkitpy.thirdparty.autoinstalled.pytest
import webkitpy.thirdparty.autoinstalled.pytest_timeout
import pytest
@@ -126,6 +128,24 @@
self.results.append(new_result)
+class TestExpectationsMarker(object):
+
+ def __init__(self, expectations):
+ self._expectations = expectations
+ self._base_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests')
+
+ def pytest_collection_modifyitems(self, session, config, items):
+ for item in items:
+ test = os.path.relpath(str(item.fspath), self._base_dir)
+ expected = self._expectations.get_expectation(test, item.name)[0]
+ if expected == 'FAIL':
+ item.add_marker(pytest.mark.xfail)
+ elif expected == 'TIMEOUT':
+ item.add_marker(pytest.mark.xfail(reason="Timeout"))
+ elif expected == 'SKIP':
+ item.add_marker(pytest.mark.skip)
+
+
def collect(directory, args):
collect_recorder = CollectRecorder()
stdout = sys.stdout
@@ -141,9 +161,10 @@
return collect_recorder.tests
-def run(path, args, timeout, env={}):
+def run(path, args, timeout, env, expectations):
harness_recorder = HarnessResultRecorder()
subtests_recorder = SubtestResultRecorder()
+ expectations_marker = TestExpectationsMarker(expectations)
_environ = dict(os.environ)
os.environ.clear()
os.environ.update(env)
@@ -159,7 +180,8 @@
'-p', 'pytest_timeout']
cmd.extend(args)
cmd.append(path)
- result = pytest.main(cmd, plugins=[harness_recorder, subtests_recorder])
+ result = pytest.main(cmd, plugins=[harness_recorder, subtests_recorder, expectations_marker])
+
if result == EXIT_INTERNALERROR:
harness_recorder.outcome = ('ERROR', None)
except Exception as e:
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py (227499 => 227500)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py 2018-01-24 10:01:53 UTC (rev 227499)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py 2018-01-24 10:02:01 UTC (rev 227500)
@@ -58,5 +58,5 @@
def collect(self, directory):
return pytest_runner.collect(directory, self._args)
- def run(self, test, timeout):
- return pytest_runner.run(test, self._args, timeout, self._env)
+ def run(self, test, timeout, expectations):
+ return pytest_runner.run(test, self._args, timeout, self._env, expectations)
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py (227499 => 227500)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py 2018-01-24 10:01:53 UTC (rev 227499)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py 2018-01-24 10:02:01 UTC (rev 227500)
@@ -24,6 +24,8 @@
import logging
import os
+from webkitpy.common.webkit_finder import WebKitFinder
+from webkitpy.common.test_expectations import TestExpectations
from webkitpy.webdriver_tests.webdriver_driver import create_driver
from webkitpy.webdriver_tests.webdriver_test_runner_selenium import WebDriverTestRunnerSelenium
from webkitpy.webdriver_tests.webdriver_test_runner_w3c import WebDriverTestRunnerW3C
@@ -49,8 +51,14 @@
_log.info('Using driver at %s' % (driver.binary_path()))
_log.info('Browser: %s' % (driver.browser_name()))
- self._runners = [runner_cls(self._port, driver, self._display_driver) for runner_cls in self.RUNNER_CLASSES]
+ _log.info('Parsing expectations')
+ self._tests_dir = WebKitFinder(self._port.host.filesystem).path_from_webkit_base('WebDriverTests')
+ expectations_file = os.path.join(self._tests_dir, 'TestExpectations.json')
+ build_type = 'Debug' if self._port.get_option('debug') else 'Release'
+ self._expectations = TestExpectations(self._port.name(), expectations_file, build_type)
+ self._runners = [runner_cls(self._port, driver, self._display_driver, self._expectations) for runner_cls in self.RUNNER_CLASSES]
+
def run(self, tests=[]):
runner_tests = [runner.collect_tests(tests) for runner in self._runners]
collected_count = sum([len(tests) for tests in runner_tests])
@@ -59,13 +67,11 @@
return 0
_log.info('Collected %d test files' % collected_count)
- results_count = 0
for i in range(len(self._runners)):
if runner_tests[i]:
- results_count += self._runners[i].run(runner_tests[i])
- return results_count
+ self._runners[i].run(runner_tests[i])
- def print_results(self):
+ def process_results(self):
results = {}
expected_count = 0
passed_count = 0
@@ -93,31 +99,39 @@
pass
_log.info('')
+ retval = 0
if not results:
_log.info('All tests run as expected')
- return
+ return retval
_log.info('%d tests ran as expected, %d didn\'t\n' % (expected_count, failures_count + timeout_count + passed_count))
def report(status, actual, expected=None):
+ retval = 0
if status not in results:
- return
+ return retval
tests = results[status]
+ tests_count = len(tests)
if expected is None:
- _log.info('Unexpected %s (%d)' % (actual, len(tests)))
+ _log.info('Unexpected %s (%d)' % (actual, tests_count))
+ retval += tests_count
else:
- _log.info('Expected to %s, but %s (%d)' % (expected, actual, len(tests)))
+ _log.info('Expected to %s, but %s (%d)' % (expected, actual, tests_count))
for test in tests:
_log.info(' %s' % test)
_log.info('')
+ return retval
+
report('XPASS', 'passed', 'fail')
report('XPASS_TIMEOUT', 'passed', 'timeout')
- report('FAIL', 'failures')
- report('TIMEOUT', 'timeouts')
+ retval += report('FAIL', 'failures')
+ retval += report('TIMEOUT', 'timeouts')
+ return retval
+
def dump_results_to_json_file(self, output_path):
json_results = {}
json_results['results'] = []
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py (227499 => 227500)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py 2018-01-24 10:01:53 UTC (rev 227499)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py 2018-01-24 10:02:01 UTC (rev 227500)
@@ -34,24 +34,24 @@
class WebDriverTestRunnerSelenium(object):
- def __init__(self, port, driver, display_driver):
+ def __init__(self, port, driver, display_driver, expectations):
self._port = port
self._driver = driver
self._display_driver = display_driver
+ self._expectations = expectations
self._results = []
+ self._tests_dir = WebKitFinder(self._port.host.filesystem).path_from_webkit_base('WebDriverTests')
- def _tests_dir(self):
- return WebKitFinder(self._port.host.filesystem).path_from_webkit_base('WebDriverTests')
-
- def collect_tests(self, tests=[]):
+ def collect_tests(self, tests):
if self._driver.selenium_name() is None:
return 0
+ skipped = [os.path.join(self._tests_dir, test) for test in self._expectations.skipped_tests()]
relative_tests_dir = os.path.join('imported', 'selenium', 'py', 'test')
executor = WebDriverSeleniumExecutor(self._driver, self._display_driver)
# Collected tests are relative to test directory.
- base_dir = os.path.join(self._tests_dir(), os.path.dirname(relative_tests_dir))
- collected_tests = [os.path.join(base_dir, test) for test in executor.collect(os.path.join(self._tests_dir(), relative_tests_dir))]
+ base_dir = os.path.join(self._tests_dir, os.path.dirname(relative_tests_dir))
+ collected_tests = [os.path.join(base_dir, test) for test in executor.collect(os.path.join(self._tests_dir, relative_tests_dir))]
selenium_tests = []
if not tests:
tests = [relative_tests_dir]
@@ -58,22 +58,22 @@
for test in tests:
if not test.startswith(relative_tests_dir):
continue
- test_path = os.path.join(self._tests_dir(), test)
+ test_path = os.path.join(self._tests_dir, test)
if os.path.isdir(test_path):
- selenium_tests.extend([test for test in collected_tests if test.startswith(test_path)])
- elif test_path in collected_tests:
+ selenium_tests.extend([test for test in collected_tests if test.startswith(test_path) and test not in skipped])
+ elif test_path in collected_tests and test_path not in skipped:
selenium_tests.append(test_path)
return selenium_tests
def run(self, tests=[]):
if self._driver.selenium_name() is None:
- return 0
+ return
executor = WebDriverSeleniumExecutor(self._driver, self._display_driver)
timeout = self._port.get_option('timeout')
for test in tests:
- test_name = os.path.relpath(test, self._tests_dir())
- harness_result, test_results = executor.run(test, timeout)
+ test_name = os.path.relpath(test, self._tests_dir)
+ harness_result, test_results = executor.run(test, timeout, self._expectations)
result = WebDriverTestResult(test_name, *harness_result)
if harness_result[0] == 'OK':
for subtest, status, message, backtrace in test_results:
@@ -83,7 +83,5 @@
pass
self._results.append(result)
- return len(self._results)
-
def results(self):
return self._results
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py (227499 => 227500)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py 2018-01-24 10:01:53 UTC (rev 227499)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py 2018-01-24 10:02:01 UTC (rev 227500)
@@ -35,18 +35,18 @@
class WebDriverTestRunnerW3C(object):
- def __init__(self, port, driver, display_driver):
+ def __init__(self, port, driver, display_driver, expectations):
self._port = port
self._driver = driver
self._display_driver = display_driver
+ self._expectations = expectations
self._results = []
+ self._tests_dir = WebKitFinder(self._port.host.filesystem).path_from_webkit_base('WebDriverTests')
self._server = WebDriverW3CWebServer(self._port)
- def _tests_dir(self):
- return WebKitFinder(self._port.host.filesystem).path_from_webkit_base('WebDriverTests')
-
- def collect_tests(self, tests=[]):
+ def collect_tests(self, tests):
+ skipped = [os.path.join(self._tests_dir, test) for test in self._expectations.skipped_tests()]
relative_tests_dir = os.path.join('imported', 'w3c', 'webdriver', 'tests')
w3c_tests = []
if not tests:
@@ -54,10 +54,10 @@
for test in tests:
if not test.startswith(relative_tests_dir):
continue
- test_path = os.path.join(self._tests_dir(), test)
+ test_path = os.path.join(self._tests_dir, test)
if os.path.isdir(test_path):
- w3c_tests.extend(self._scan_directory(test_path))
- elif self._is_test(test_path):
+ w3c_tests.extend(self._scan_directory(test_path, skipped))
+ elif self._is_test(test_path) and test_path not in skipped:
w3c_tests.append(test_path)
return w3c_tests
@@ -72,10 +72,10 @@
return False
return True
- def _scan_directory(self, directory):
+ def _scan_directory(self, directory, skipped):
tests = []
for path in self._port.host.filesystem.files_under(directory):
- if self._is_test(path):
+ if self._is_test(path) and path not in skipped:
tests.append(path)
return tests
@@ -91,8 +91,8 @@
timeout = self._port.get_option('timeout')
try:
for test in tests:
- test_name = os.path.relpath(test, self._tests_dir())
- harness_result, test_results = executor.run(test, timeout)
+ test_name = os.path.relpath(test, self._tests_dir)
+ harness_result, test_results = executor.run(test, timeout, self._expectations)
result = WebDriverTestResult(test_name, *harness_result)
if harness_result[0] == 'OK':
for subtest, status, message, backtrace in test_results:
@@ -105,7 +105,5 @@
executor.teardown()
self._server.stop()
- return len(self._results)
-
def results(self):
return self._results
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py (227499 => 227500)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py 2018-01-24 10:01:53 UTC (rev 227499)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py 2018-01-24 10:02:01 UTC (rev 227500)
@@ -145,10 +145,10 @@
def teardown(self):
self.protocol.teardown()
- def run(self, test, timeout):
+ def run(self, test, timeout, expectations):
env = {'WD_HOST': self.protocol.session_config['host'],
'WD_PORT': str(self.protocol.session_config['port']),
'WD_CAPABILITIES': json.dumps(self.protocol.session_config['capabilities']),
'WD_SERVER_CONFIG': json.dumps(self.server_config)}
args = ['--strict', '-p', 'no:mozlog']
- return pytest_runner.run(test, args, timeout, env)
+ return pytest_runner.run(test, args, timeout, env, expectations)
Modified: releases/WebKitGTK/webkit-2.18/WebDriverTests/ChangeLog (227499 => 227500)
--- releases/WebKitGTK/webkit-2.18/WebDriverTests/ChangeLog 2018-01-24 10:01:53 UTC (rev 227499)
+++ releases/WebKitGTK/webkit-2.18/WebDriverTests/ChangeLog 2018-01-24 10:02:01 UTC (rev 227500)
@@ -1,3 +1,15 @@
+2018-01-17 Carlos Garcia Campos <cgar...@igalia.com>
+
+ WebDriver: add support for test expectations
+ https://bugs.webkit.org/show_bug.cgi?id=180420
+
+ Reviewed by Carlos Alberto Lopez Perez.
+
+ Add initial test expectations. For now I'm only adding the W3C test expectations, selenium ones will be added in
+ a follow up patch.
+
+ * TestExpectations.json: Added.
+
2018-01-11 Carlos Garcia Campos <cgar...@igalia.com>
Unreviewed. Update Selenium WebDriver imported tests.
Added: releases/WebKitGTK/webkit-2.18/WebDriverTests/TestExpectations.json (0 => 227500)
--- releases/WebKitGTK/webkit-2.18/WebDriverTests/TestExpectations.json (rev 0)
+++ releases/WebKitGTK/webkit-2.18/WebDriverTests/TestExpectations.json 2018-01-24 10:02:01 UTC (rev 227500)
@@ -0,0 +1,372 @@
+{
+ "imported/w3c/webdriver/tests/actions/key.py": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+ },
+ "imported/w3c/webdriver/tests/actions/key_shortcuts.py": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+ },
+ "imported/w3c/webdriver/tests/actions/modifier_click.py": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+ },
+ "imported/w3c/webdriver/tests/actions/mouse.py": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+ },
+ "imported/w3c/webdriver/tests/actions/mouse_dblclick.py": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+ },
+ "imported/w3c/webdriver/tests/actions/sequence.py": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+ },
+ "imported/w3c/webdriver/tests/actions/special_keys.py": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+ },
+ "imported/w3c/webdriver/tests/contexts/resizing_and_positioning.py": {
+ "subtests": {
+ "test_window_resize": {
+ "expected": {"gtk": {"status": ["FAIL"]}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/contexts/maximize_window.py": {
+ "subtests": {
+ "test_no_browsing_context": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}}
+ },
+ "test_handle_prompt_accept": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}}
+ },
+ "test_handle_prompt_missing_value": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}}
+ },
+ "test_fully_exit_fullscreen": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}}
+ },
+ "test_restore_the_window": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}}
+ },
+ "test_maximize": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}}
+ },
+ "test_payload": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}}
+ },
+ "test_maximize_twice_is_idempotent": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/contexts/json_serialize_windowproxy.py": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180397"}}
+ },
+ "imported/w3c/webdriver/tests/document_handling/page_source.py": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180399"}}
+ },
+ "imported/w3c/webdriver/tests/element_click/bubbling.py": {
+ "subtests": {
+ "test_spin_event_loop": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180400"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/element_send_keys/interactability.py": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181644"}}
+ },
+ "imported/w3c/webdriver/tests/element_send_keys/scroll_into_view.py": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181644"}}
+ },
+ "imported/w3c/webdriver/tests/interaction/send_keys_content_editable.py": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180403"}}
+ },
+ "imported/w3c/webdriver/tests/interaction/element_clear.py": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180404"}},
+ "subtests": {
+ "test_no_browsing_context": {
+ "expected": {"all": {"status": ["PASS"]}}
+ },
+ "test_disabled_element_not_resettable": {
+ "expected": {"all": {"status": ["PASS"]}}
+ },
+ "test_element_readonly": {
+ "expected": {"all": {"status": ["PASS"]}}
+ },
+ "test_element_disabled": {
+ "expected": {"all": {"status": ["PASS"]}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/navigation/current_url.py": {
+ "subtests": {
+ "test_set_malformed_url": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180405"}}
+ },
+ "test_get_current_url_after_modified_location": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180405"}}
+ },
+ "test_get_current_url_nested_browsing_context": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180405"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/sessions/new_session/merge.py": {
+ "subtests": {
+ "test_merge_invalid[unhandledPromptBehavior-value1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180407"}}
+ },
+ "test_merge_invalid[unhandledPromptBehavior-value2]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180407"}}
+ },
+ "test_merge_platformName": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180407"}}
+ },
+ "test_merge_browserName": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180407"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/sessions/new_session/invalid_capabilities.py": {
+ "subtests": {
+ "test_invalid_values[proxy-1-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-1-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value28-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value28-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-{}-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-{}-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value30-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value30-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value31-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value31-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value32-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value32-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value33-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value33-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value34-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value34-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value35-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value35-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value36-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value36-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value37-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value37-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value38-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value38-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value39-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value39-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value40-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value40-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value41-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value41-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value42-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value42-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value43-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value43-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value44-body0]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_invalid_values[proxy-value44-body1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/sessions/new_session/default_values.py": {
+ "subtests": {
+ "test_ignore_non_spec_fields_in_capabilities": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180409"}}
+ },
+ "test_valid_but_unmatchable_key": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180410"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/sessions/new_session/response.py": {
+ "subtests": {
+ "test_resp_capabilites": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180408"}}
+ },
+ "test_resp_data": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180411"}}
+ },
+ "test_timeouts": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180412"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/state/get_element_property.py": {
+ "subtests": {
+ "test_element_not_found": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180413"}}
+ },
+ "test_element": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180414"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/state/get_element_attribute.py": {
+ "subtests": {
+ "test_element_not_found": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180413"}}
+ },
+ "test_boolean_attribute[button-attrs1]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180415"}}
+ },
+ "test_boolean_attribute[dialog-attrs3]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180415"}}
+ },
+ "test_boolean_attribute[form-attrs5]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180415"}}
+ },
+ "test_boolean_attribute[iframe-attrs6]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180415"}}
+ },
+ "test_boolean_attribute[img-attrs7]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180415"}}
+ },
+ "test_boolean_attribute[input-attrs8]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180415"}}
+ },
+ "test_boolean_attribute[menuitem-attrs9]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180415"}}
+ },
+ "test_boolean_attribute[object-attrs10]": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180415"}}
+ },
+ "test_global_boolean_attributes": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180415"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/state/get_element_tag_name.py": {
+ "subtests": {
+ "test_element_not_found": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180413"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/state/text/get_text.py": {
+ "subtests": {
+ "test_getting_text_of_a_non_existant_element_is_an_error": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180413"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/fullscreen_window.py": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}},
+ "subtests": {
+ "test_handle_prompt_dismiss_and_notify": {
+ "expected": {"all": {"status": ["PASS"]}}
+ },
+ "test_handle_prompt_accept_and_notify": {
+ "expected": {"all": {"status": ["PASS"]}}
+ },
+ "test_handle_prompt_ignore": {
+ "expected": {"all": {"status": ["PASS"]}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/minimize_window.py": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}},
+ "subtests": {
+ "test_handle_prompt_dismiss_and_notify": {
+ "expected": {"all": {"status": ["PASS"]}}
+ },
+ "test_handle_prompt_accept_and_notify": {
+ "expected": {"all": {"status": ["PASS"]}}
+ },
+ "test_handle_prompt_ignore": {
+ "expected": {"all": {"status": ["PASS"]}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/set_window_rect.py": {
+ "subtests": {
+ "test_width_height_floats": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180416"}}
+ },
+ "test_fully_exit_fullscreen": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}}
+ },
+ "test_restore_from_minimized": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}}
+ },
+ "test_restore_from_maximized": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180398"}}
+ },
+ "test_height_width": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180417"}}
+ },
+ "test_height_width_larger_than_max": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180419"}}
+ },
+ "test_negative_x_y": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/180418"}}
+ }
+ }
+ }
+}
