Diff
Modified: trunk/WebDriverTests/ChangeLog (254191 => 254192)
--- trunk/WebDriverTests/ChangeLog 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/ChangeLog 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,3 +1,46 @@
+2020-01-08 Carlos Garcia Campos <[email protected]>
+
+ Unreviewed. Update W3C WebDriver imported tests.
+
+ * TestExpectations.json:
+ * imported/w3c/importer.json:
+ * imported/w3c/tools/webdriver/webdriver/__init__.py:
+ * imported/w3c/tools/webdriver/webdriver/client.py:
+ * imported/w3c/tools/webdriver/webdriver/error.py:
+ * imported/w3c/tools/webdriver/webdriver/transport.py:
+ * imported/w3c/tools/wptrunner/docs/expectation.rst:
+ * imported/w3c/tools/wptrunner/requirements.txt:
+ * imported/w3c/tools/wptrunner/requirements_firefox.txt:
+ * imported/w3c/tools/wptrunner/tox.ini:
+ * imported/w3c/tools/wptrunner/wptrunner/browsers/__init__.py:
+ * imported/w3c/tools/wptrunner/wptrunner/browsers/android_weblayer.py: Added.
+ * imported/w3c/tools/wptrunner/wptrunner/browsers/base.py:
+ * imported/w3c/tools/wptrunner/wptrunner/browsers/servo.py:
+ * imported/w3c/tools/wptrunner/wptrunner/browsers/servodriver.py:
+ * imported/w3c/tools/wptrunner/wptrunner/browsers/webkit.py:
+ * imported/w3c/tools/wptrunner/wptrunner/executors/base.py:
+ * imported/w3c/tools/wptrunner/wptrunner/executors/executormarionette.py:
+ * imported/w3c/tools/wptrunner/wptrunner/executors/executorselenium.py:
+ * imported/w3c/tools/wptrunner/wptrunner/executors/executorservo.py:
+ * imported/w3c/tools/wptrunner/wptrunner/executors/executorservodriver.py:
+ * imported/w3c/tools/wptrunner/wptrunner/executors/executorwebdriver.py:
+ * imported/w3c/tools/wptrunner/wptrunner/executors/protocol.py:
+ * imported/w3c/tools/wptrunner/wptrunner/executors/reftest-wait_marionette.js: Removed.
+ * imported/w3c/tools/wptrunner/wptrunner/executors/reftest-wait_webdriver.js:
+ * imported/w3c/tools/wptrunner/wptrunner/testdriver-extra.js:
+ * imported/w3c/tools/wptrunner/wptrunner/testrunner.py:
+ * imported/w3c/tools/wptrunner/wptrunner/tests/__init__.py:
+ * imported/w3c/tools/wptrunner/wptrunner/tests/browsers/test_webkitgtk.py: Added.
+ * imported/w3c/tools/wptrunner/wptrunner/tests/test_wpttest.py:
+ * imported/w3c/tools/wptrunner/wptrunner/wptrunner.py:
+ * imported/w3c/tools/wptrunner/wptrunner/wpttest.py:
+ * imported/w3c/webdriver/tests/find_element_from_element/find.py:
+ * imported/w3c/webdriver/tests/new_session/default_values.py:
+ * imported/w3c/webdriver/tests/perform_actions/support/keys.py:
+ * imported/w3c/webdriver/tests/perform_actions/validity.py:
+ * imported/w3c/webdriver/tests/send_alert_text/send.py:
+ * imported/w3c/webdriver/tests/set_window_rect/set.py:
+
2020-01-07 Carlos Garcia Campos <[email protected]>
WebDriver: several w3c collections tests added in r230953 are failing
Modified: trunk/WebDriverTests/TestExpectations.json (254191 => 254192)
--- trunk/WebDriverTests/TestExpectations.json 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/TestExpectations.json 2020-01-08 11:53:22 UTC (rev 254192)
@@ -675,13 +675,6 @@
}
}
},
- "imported/w3c/webdriver/tests/send_alert_text/send.py": {
- "subtests": {
- "test_alert_unsupported_operation": {
- "expected": {"all": {"status": ["SKIP"]}}
- }
- }
- },
"imported/w3c/webdriver/tests/permissions/set.py": {
"expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/192978"}}
},
Modified: trunk/WebDriverTests/imported/w3c/importer.json (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/importer.json 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/importer.json 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,6 +1,6 @@
{
"repository": "https://github.com/w3c/web-platform-tests.git",
- "revision": "2cc5a5ca83698ffd79abf7b6671aebc44c354d04",
+ "revision": "96203b2df545e5286b5912cd9f0ac95d04154303",
"paths_to_import": [
"tools/webdriver",
"tools/wptrunner",
Modified: trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/__init__.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/__init__.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/__init__.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,6 +1,6 @@
# flake8: noqa
-from client import (
+from .client import (
Cookies,
Element,
Find,
@@ -8,7 +8,7 @@
Session,
Timeouts,
Window)
-from error import (
+from .error import (
ElementNotSelectableException,
ElementNotVisibleException,
InvalidArgumentException,
Modified: trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/client.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/client.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/client.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,10 +1,9 @@
-import urlparse
+from . import error
+from . import protocol
+from . import transport
-import error
-import protocol
-import transport
-
from six import string_types
+from six.moves.urllib import parse as urlparse
def command(func):
Modified: trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/error.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/error.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/error.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,6 +1,7 @@
import collections
import json
+from six import itervalues
class WebDriverException(Exception):
http_status = None
@@ -205,6 +206,6 @@
_errors = collections.defaultdict()
-for item in locals().values():
+for item in list(itervalues(locals())):
if type(item) == type and issubclass(item, WebDriverException):
_errors[item.status_code] = item
Modified: trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/transport.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/transport.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/webdriver/webdriver/transport.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,9 +1,9 @@
-import httplib
import json
import select
-import urlparse
+from six.moves.urllib import parse as urlparse
+from six.moves import http_client as httplib
-import error
+from . import error
from six import text_type
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/docs/expectation.rst (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/docs/expectation.rst 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/docs/expectation.rst 2020-01-08 11:53:22 UTC (rev 254192)
@@ -248,7 +248,7 @@
base test URL, in which case the fuzziness applies to any
comparison with that URL, or takes the form lhs url, comparison,
rhs url, in which case the fuzziness only applies for any
- comparison involving that specifc pair of URLs. Some illustrative
+ comparison involving that specific pair of URLs. Some illustrative
examples are given below.
* Variables ``debug``, ``os``, ``version``, ``processor`` and
@@ -287,7 +287,7 @@
[canvas_test.html]
expected:
- if os == "osx": FAIL
+ if os == "mac": FAIL
if os == "windows" and version == "XP": FAIL
PASS
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/requirements.txt (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/requirements.txt 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/requirements.txt 2020-01-08 11:53:22 UTC (rev 254192)
@@ -3,6 +3,6 @@
mozlog==5.0
mozdebug==0.1.1
pillow==6.2.1
-urllib3[secure]==1.25.6
+urllib3[secure]==1.25.7
requests==2.22.0
six==1.13.0
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/requirements_firefox.txt (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/requirements_firefox.txt 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/requirements_firefox.txt 2020-01-08 11:53:22 UTC (rev 254192)
@@ -8,4 +8,4 @@
mozprofile==2.4.0
mozrunner==7.7.0
mozversion==2.2.0
-psutil==5.6.4
+psutil==5.6.7
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/tox.ini (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/tox.ini 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/tox.ini 2020-01-08 11:53:22 UTC (rev 254192)
@@ -2,7 +2,7 @@
xfail_strict=true
[tox]
-envlist = py27-{base,chrome,edge,firefox,ie,opera,safari,sauce,servo},py36-base
+envlist = py27-{base,chrome,edge,firefox,ie,opera,safari,sauce,servo,webkit,webkitgtk_minibrowser,epiphany},py36-base
skip_missing_interpreters = true
[testenv]
@@ -20,6 +20,9 @@
safari: -r{toxinidir}/requirements_safari.txt
sauce: -r{toxinidir}/requirements_sauce.txt
servo: -r{toxinidir}/requirements_servo.txt
+ webkit: -r{toxinidir}/requirements_webkit.txt
+ webkitgtk_minibrowser: -r{toxinidir}/requirements_webkit.txt
+ epiphany: -r{toxinidir}/requirements_epiphany.txt
commands = pytest {posargs}
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/__init__.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/__init__.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/__init__.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -22,7 +22,8 @@
module global scope.
"""
-product_list = ["android_webview",
+product_list = ["android_weblayer",
+ "android_webview",
"chrome",
"chrome_android",
"chrome_ios",
Added: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/android_weblayer.py (0 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/android_weblayer.py (rev 0)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/android_weblayer.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -0,0 +1,135 @@
+import subprocess
+
+from .base import Browser, ExecutorBrowser, require_arg
+from .base import get_timeout_multiplier # noqa: F401
+from .chrome import executor_kwargs as chrome_executor_kwargs
+from ..webdriver_server import ChromeDriverServer
+from ..executors.executorwebdriver import (WebDriverTestharnessExecutor, # noqa: F401
+ WebDriverRefTestExecutor) # noqa: F401
+from ..executors.executorchrome import ChromeDriverWdspecExecutor # noqa: F401
+
+
+__wptrunner__ = {"product": "android_weblayer",
+ "check_args": "check_args",
+ "browser": "WeblayerShell",
+ "executor": {"testharness": "WebDriverTestharnessExecutor",
+ "reftest": "WebDriverRefTestExecutor",
+ "wdspec": "ChromeDriverWdspecExecutor"},
+ "browser_kwargs": "browser_kwargs",
+ "executor_kwargs": "executor_kwargs",
+ "env_extras": "env_extras",
+ "env_options": "env_options",
+ "timeout_multiplier": "get_timeout_multiplier"}
+
+_wptserve_ports = set()
+
+
+def check_args(**kwargs):
+ require_arg(kwargs, "webdriver_binary")
+
+
+def browser_kwargs(test_type, run_info_data, config, **kwargs):
+ return {"binary": kwargs["binary"],
+ "device_serial": kwargs["device_serial"],
+ "webdriver_binary": kwargs["webdriver_binary"],
+ "webdriver_args": kwargs.get("webdriver_args")}
+
+
+def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
+ **kwargs):
+ # Use update() to modify the global list in place.
+ _wptserve_ports.update(set(
+ server_config['ports']['http'] + server_config['ports']['https'] +
+ server_config['ports']['ws'] + server_config['ports']['wss']
+ ))
+
+ executor_kwargs = chrome_executor_kwargs(test_type, server_config,
+ cache_manager, run_info_data,
+ **kwargs)
+ del executor_kwargs["capabilities"]["goog:chromeOptions"]["prefs"]
+ del executor_kwargs["capabilities"]["goog:chromeOptions"]["useAutomationExtension"]
+ capabilities = executor_kwargs["capabilities"]
+ # Note that for WebLayer, we launch a test shell and have the test shell use
+ # WebLayer.
+ # https://cs.chromium.org/chromium/src/weblayer/shell/android/shell_apk/
+ capabilities["goog:chromeOptions"]["androidPackage"] = \
+ "org.chromium.weblayer.shell"
+ capabilities["goog:chromeOptions"]["androidActivity"] = ".WebLayerShellActivity"
+ if kwargs.get('device_serial'):
+ capabilities["goog:chromeOptions"]["androidDeviceSerial"] = kwargs['device_serial']
+
+ # Workaround: driver.quit() cannot quit WeblayerShell.
+ executor_kwargs["pause_after_test"] = False
+ # Workaround: driver.close() is not supported.
+ executor_kwargs["restart_after_test"] = True
+ executor_kwargs["close_after_done"] = False
+ return executor_kwargs
+
+
+def env_extras(**kwargs):
+ return []
+
+
+def env_options():
+ # allow the use of host-resolver-rules in lieu of modifying /etc/hosts file
+ return {"server_host": "127.0.0.1"}
+
+
+#TODO: refactor common elements of WeblayerShell and ChromeAndroidBrowser
+class WeblayerShell(Browser):
+ """Chrome is backed by chromedriver, which is supplied through
+ ``wptrunner.webdriver.ChromeDriverServer``.
+ """
+
+ def __init__(self, logger, binary, webdriver_binary="chromedriver",
+ device_serial=None,
+ webdriver_args=None):
+ """Creates a new representation of Chrome. The `binary` argument gives
+ the browser binary to use for testing."""
+ Browser.__init__(self, logger)
+ self.binary = binary
+ self.device_serial = device_serial
+ self.server = ChromeDriverServer(self.logger,
+ binary=webdriver_binary,
+ args=webdriver_args)
+ self.setup_adb_reverse()
+
+ def _adb_run(self, args):
+ cmd = ['adb']
+ if self.device_serial:
+ cmd.extend(['-s', self.device_serial])
+ cmd.extend(args)
+ self.logger.info(' '.join(cmd))
+ subprocess.check_call(cmd)
+
+ def setup_adb_reverse(self):
+ self._adb_run(['wait-for-device'])
+ self._adb_run(['forward', '--remove-all'])
+ self._adb_run(['reverse', '--remove-all'])
+ # "adb reverse" basically forwards network connection from device to
+ # host.
+ for port in _wptserve_ports:
+ self._adb_run(['reverse', 'tcp:%d' % port, 'tcp:%d' % port])
+
+ def start(self, **kwargs):
+ self.server.start(block=False)
+
+ def stop(self, force=False):
+ self.server.stop(force=force)
+
+ def pid(self):
+ return self.server.pid
+
+ def is_alive(self):
+ # TODO(ato): This only indicates the driver is alive,
+ # and doesn't say anything about whether a browser session
+ # is active.
+ return self.server.is_alive
+
+ def cleanup(self):
+ self.stop()
+ self._adb_run(['forward', '--remove-all'])
+ self._adb_run(['reverse', '--remove-all'])
+
+ def executor_browser(self):
+ return ExecutorBrowser, {"webdriver_url": self.server.url}
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/base.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/base.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/base.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -50,6 +50,14 @@
return current_args
+def certificate_domain_list(list_of_domains, certificate_file):
+ """Build a list of domains where certificate_file should be used"""
+ cert_list = []
+ for domain in list_of_domains:
+ cert_list.append({"host": domain, "certificateFile": certificate_file})
+ return cert_list
+
+
def get_free_port():
"""Get a random unbound port"""
while True:
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/servo.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/servo.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/servo.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -46,6 +46,7 @@
rv["pause_after_test"] = kwargs["pause_after_test"]
if test_type == "wdspec":
rv["capabilities"] = {}
+ rv["webdriver_binary"] = kwargs["binary"]
return rv
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/servodriver.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/servodriver.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/servodriver.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -163,9 +163,7 @@
command=" ".join(self.command))
def is_alive(self):
- if self.runner:
- return self.runner.is_running()
- return False
+ return self.proc.poll() is None
def cleanup(self):
self.stop()
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/webkit.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/webkit.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/browsers/webkit.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,5 +1,5 @@
from .base import Browser, ExecutorBrowser, require_arg
-from .base import get_timeout_multiplier # noqa: F401
+from .base import get_timeout_multiplier, certificate_domain_list # noqa: F401
from ..executors import executor_kwargs as base_executor_kwargs
from ..executors.executorwebdriver import (WebDriverTestharnessExecutor, # noqa: F401
WebDriverRefTestExecutor) # noqa: F401
@@ -47,9 +47,7 @@
browser_options_key: {
"binary": kwargs["binary"],
"args": kwargs.get("binary_args", []),
- "certificates": [
- {"host": server_config["browser_host"],
- "certificateFile": kwargs["host_cert_path"]}]}}
+ "certificates": certificate_domain_list(server_config.domains_set, kwargs["host_cert_path"])}}
return {}
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/base.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/base.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/base.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -7,6 +7,7 @@
import threading
import traceback
import socket
+import sys
from six.moves.urllib.parse import urljoin, urlsplit, urlunsplit
from abc import ABCMeta, abstractmethod
@@ -15,11 +16,7 @@
here = os.path.split(__file__)[0]
-# Extra timeout to use after internal test timeout at which the harness
-# should force a timeout
-extra_timeout = 5 # seconds
-
def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
**kwargs):
timeout_multiplier = kwargs["timeout_multiplier"]
@@ -130,6 +127,63 @@
self.message = message
+class TimedRunner(object):
+ def __init__(self, logger, func, protocol, url, timeout, extra_timeout):
+ self.func = func
+ self.logger = logger
+ self.result = None
+ self.protocol = protocol
+ self.url = ""
+ self.timeout = timeout
+ self.extra_timeout = extra_timeout
+ self.result_flag = threading.Event()
+
+ def run(self):
+ if self.set_timeout() is Stop:
+ return Stop
+
+ if self.before_run() is Stop:
+ return Stop
+
+ executor = threading.Thread(target=self.run_func)
+ executor.start()
+
+ # Add twice the extra timeout since the called function is expected to
+ # wait at least self.timeout + self.extra_timeout and this gives some leeway
+ timeout = self.timeout + 2 * self.extra_timeout if self.timeout else None
+ finished = self.result_flag.wait(timeout)
+ if self.result is None:
+ if finished:
+ # flag is True unless we timeout; this *shouldn't* happen, but
+ # it can if self.run_func fails to set self.result due to raising
+ self.result = False, ("INTERNAL-ERROR", "%s.run_func didn't set a result" %
+ self.__class__.__name__)
+ else:
+ message = "Executor hit external timeout (this may indicate a hang)\n"
+ # get a traceback for the current stack of the executor thread
+ message += "".join(traceback.format_stack(sys._current_frames()[executor.ident]))
+ self.result = False, ("EXTERNAL-TIMEOUT", message)
+ elif self.result[1] is None:
+ # We didn't get any data back from the test, so check if the
+ # browser is still responsive
+ if self.protocol.is_alive:
+ self.result = False, ("INTERNAL-ERROR", None)
+ else:
+ self.logger.info("Browser not responding, setting status to CRASH")
+ self.result = False, ("CRASH", None)
+
+ return self.result
+
+ def set_timeout(self):
+ raise NotImplementedError
+
+ def before_run(self):
+ pass
+
+ def run_func(self):
+ raise NotImplementedError
+
+
class TestExecutor(object):
"""Abstract Base class for object that actually executes the tests in a
specific browser. Typically there will be a different TestExecutor
@@ -148,7 +202,11 @@
convert_result = None
supports_testdriver = False
supports_jsshell = False
+ # Extra timeout to use after internal test timeout at which the harness
+ # should force a timeout
+ extra_timeout = 5 # seconds
+
def __init__(self, browser, server_config, timeout_multiplier=1,
debug_info=None, **kwargs):
self.runner = None
@@ -302,7 +360,7 @@
def reset(self):
self.screenshot_cache.clear()
- def is_pass(self, hashes, screenshots, relation, fuzzy):
+ def is_pass(self, hashes, screenshots, urls, relation, fuzzy):
assert relation in ("==", "!=")
if not fuzzy or fuzzy == ((0,0), (0,0)):
equal = hashes[0] == hashes[1]
@@ -309,10 +367,10 @@
# sometimes images can have different hashes, but pixels can be identical.
if not equal:
self.logger.info("Image hashes didn't match, checking pixel differences")
- max_per_channel, pixels_different = self.get_differences(screenshots)
+ max_per_channel, pixels_different = self.get_differences(screenshots, urls)
equal = pixels_different == 0 and max_per_channel == 0
else:
- max_per_channel, pixels_different = self.get_differences(screenshots)
+ max_per_channel, pixels_different = self.get_differences(screenshots, urls)
allowed_per_channel, allowed_different = fuzzy
self.logger.info("Allowed %s pixels different, maximum difference per channel %s" %
("-".join(str(item) for item in allowed_different),
@@ -323,11 +381,13 @@
allowed_different[0] <= pixels_different <= allowed_different[1]))
return equal if relation == "==" else not equal
- def get_differences(self, screenshots):
+ def get_differences(self, screenshots, urls):
from PIL import Image, ImageChops, ImageStat
lhs = Image.open(io.BytesIO(base64.b64decode(screenshots[0]))).convert("RGB")
rhs = Image.open(io.BytesIO(base64.b64decode(screenshots[1]))).convert("RGB")
+ self.check_if_solid_color(lhs, urls[0])
+ self.check_if_solid_color(rhs, urls[1])
diff = ImageChops.difference(lhs, rhs)
minimal_diff = diff.crop(diff.getbbox())
mask = minimal_diff.convert("L", dither=None)
@@ -338,6 +398,12 @@
(count, per_channel))
return per_channel, count
+ def check_if_solid_color(self, image, url):
+ extrema = image.getextrema()
+ if all(min == max for min, max in extrema):
+ color = ''.join('%02X' % value for value, _ in extrema)
+ self.message.append("Screenshot is solid color 0x%s for %s\n" % (color, url))
+
def run_test(self, test):
viewport_size = test.viewport_size
dpi = test.dpi
@@ -350,6 +416,7 @@
while stack:
hashes = [None, None]
screenshots = [None, None]
+ urls = [None, None]
nodes, relation = stack.pop()
fuzzy = self.get_fuzzy(test, nodes, relation)
@@ -360,8 +427,9 @@
return {"status": data[0], "message": data[1]}
hashes[i], screenshots[i] = data
+ urls[i] = node.url
- if self.is_pass(hashes, screenshots, relation, fuzzy):
+ if self.is_pass(hashes, screenshots, urls, relation, fuzzy):
fuzzy = self.get_fuzzy(test, nodes, relation)
if nodes[1].references:
stack.extend(list(((nodes[1], item[0]), item[1]) for item in reversed(nodes[1].references)))
@@ -441,7 +509,7 @@
pass
def do_test(self, test):
- timeout = test.timeout * self.timeout_multiplier + extra_timeout
+ timeout = test.timeout * self.timeout_multiplier + self.extra_timeout
success, data = ""
self.protocol.session_config,
@@ -586,6 +654,8 @@
WebDriver. Things that are more different to WebDriver may need to create a
fully custom implementation."""
+ unimplemented_exc = (NotImplementedError,)
+
def __init__(self, logger, protocol, test_window):
self.protocol = protocol
self.test_window = test_window
@@ -600,6 +670,7 @@
"send_keys": SendKeysAction(self.logger, self.protocol),
"action_sequence": ActionSequenceAction(self.logger, self.protocol),
"generate_test_report": GenerateTestReportAction(self.logger, self.protocol),
+ "set_permission": SetPermissionAction(self.logger, self.protocol),
"add_virtual_authenticator": AddVirtualAuthenticatorAction(self.logger, self.protocol),
"remove_virtual_authenticator": RemoveVirtualAuthenticatorAction(self.logger, self.protocol),
"add_credential": AddCredentialAction(self.logger, self.protocol),
@@ -631,6 +702,9 @@
raise ValueError("Unknown action %s" % action)
try:
result = action_handler(payload)
+ except self.unimplemented_exc:
+ self.logger.warning("Action %s not implemented" % action)
+ self._send_message("complete", "error", "Action %s not implemented" % action)
except Exception:
self.logger.warning("Action %s failed" % action)
self.logger.warning(traceback.format_exc())
@@ -702,6 +776,20 @@
self.logger.debug("Generating test report: %s" % message)
self.protocol.generate_test_report.generate_test_report(message)
+class SetPermissionAction(object):
+ def __init__(self, logger, protocol):
+ self.logger = logger
+ self.protocol = protocol
+
+ def __call__(self, payload):
+ permission_params = payload["permission_params"]
+ descriptor = permission_params["descriptor"]
+ name = descriptor["name"]
+ state = permission_params["state"]
+ _one_realm_ = permission_params.get("oneRealm", False)
+ self.logger.debug("Setting permission %s to %s, _oneRealm_=%s" % (name, state, one_realm))
+ self.protocol.set_permission.set_permission(descriptor, state, one_realm)
+
class AddVirtualAuthenticatorAction(object):
def __init__(self, logger, protocol):
self.logger = logger
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executormarionette.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executormarionette.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executormarionette.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -16,9 +16,9 @@
RefTestExecutor,
RefTestImplementation,
TestharnessExecutor,
+ TimedRunner,
WdspecExecutor,
WebDriverProtocol,
- extra_timeout,
strip_server)
from .protocol import (ActionSequenceProtocolPart,
AssertsProtocolPart,
@@ -31,7 +31,10 @@
ClickProtocolPart,
SendKeysProtocolPart,
TestDriverProtocolPart,
- CoverageProtocolPart)
+ CoverageProtocolPart,
+ GenerateTestReportProtocolPart,
+ VirtualAuthenticatorProtocolPart,
+ SetPermissionProtocolPart)
from ..testrunner import Stop
from ..webdriver_server import GeckoDriverServer
@@ -213,7 +216,7 @@
handles = self.marionette.window_handles
if len(handles) == 2:
test_window = next(iter(set(handles) - {parent}))
- elif handles[0] == parent and len(handles) > 2:
+ elif len(handles) > 2 and handles[0] == parent:
# Hope the first one here is the test window
test_window = handles[1]
@@ -481,7 +484,45 @@
# This usually happens if the process crashed
pass
+class MarionetteGenerateTestReportProtocolPart(GenerateTestReportProtocolPart):
+ def setup(self):
+ self.marionette = self.parent.marionette
+ def generate_test_report(self, config):
+ raise NotImplementedError("generate_test_report not yet implemented")
+
+class MarionetteVirtualAuthenticatorProtocolPart(VirtualAuthenticatorProtocolPart):
+ def setup(self):
+ self.marionette = self.parent.marionette
+
+ def add_virtual_authenticator(self, config):
+ raise NotImplementedError("add_virtual_authenticator not yet implemented")
+
+ def remove_virtual_authenticator(self, authenticator_id):
+ raise NotImplementedError("remove_virtual_authenticator not yet implemented")
+
+ def add_credential(self, authenticator_id, credential):
+ raise NotImplementedError("add_credential not yet implemented")
+
+ def get_credentials(self, authenticator_id):
+ raise NotImplementedError("get_credentials not yet implemented")
+
+ def remove_credential(self, authenticator_id, credential_id):
+ raise NotImplementedError("remove_credential not yet implemented")
+
+ def remove_all_credentials(self, authenticator_id):
+ raise NotImplementedError("remove_all_credentials not yet implemented")
+
+ def set_user_verified(self, authenticator_id, uv):
+ raise NotImplementedError("set_user_verified not yet implemented")
+
+class MarionetteSetPermissionProtocolPart(SetPermissionProtocolPart):
+ def setup(self):
+ self.marionette = self.parent.marionette
+
+ def set_permission(self, name, state, one_realm):
+ raise NotImplementedError("set_permission not yet implemented")
+
class MarionetteProtocol(Protocol):
implements = [MarionetteBaseProtocolPart,
MarionetteTestharnessProtocolPart,
@@ -493,7 +534,10 @@
MarionetteActionSequenceProtocolPart,
MarionetteTestDriverProtocolPart,
MarionetteAssertsProtocolPart,
- MarionetteCoverageProtocolPart]
+ MarionetteCoverageProtocolPart,
+ MarionetteGenerateTestReportProtocolPart,
+ MarionetteVirtualAuthenticatorProtocolPart,
+ MarionetteSetPermissionProtocolPart]
def __init__(self, executor, browser, capabilities=None, timeout_multiplier=1, e10s=True, ccov=False):
do_delayed_imports()
@@ -566,27 +610,14 @@
self.prefs.set(name, value)
-class ExecuteAsyncScriptRun(object):
- def __init__(self, logger, func, protocol, url, timeout):
- self.logger = logger
- self.result = (None, None)
- self.protocol = protocol
- self.func = func
- self.url = ""
- self.timeout = timeout
- self.result_flag = threading.Event()
+class ExecuteAsyncScriptRun(TimedRunner):
- def run(self):
- index = self.url.rfind("/storage/")
- if index != -1:
- # Clear storage
- self.protocol.storage.clear_origin(self.url)
-
+ def set_timeout(self):
timeout = self.timeout
try:
if timeout is not None:
- self.protocol.base.set_timeout(timeout + extra_timeout)
+ self.protocol.base.set_timeout(timeout + self.extra_timeout)
else:
# We just want it to never time out, really, but marionette doesn't
# make that possible. It also seems to time out immediately if the
@@ -596,33 +627,13 @@
self.logger.error("Lost marionette connection before starting test")
return Stop
- if timeout is not None:
- wait_timeout = timeout + 2 * extra_timeout
- else:
- wait_timeout = None
+ def before_run(self):
+ index = self.url.rfind("/storage/")
+ if index != -1:
+ # Clear storage
+ self.protocol.storage.clear_origin(self.url)
- timer = threading.Timer(wait_timeout, self._timeout)
- timer.start()
-
- self._run()
-
- self.result_flag.wait()
- timer.cancel()
-
- if self.result == (None, None):
- self.logger.debug("Timed out waiting for a result")
- self.result = False, ("EXTERNAL-TIMEOUT", None)
- elif self.result[1] is None:
- # We didn't get any data back from the test, so check if the
- # browser is still responsive
- if self.protocol.is_alive:
- self.result = False, ("INTERNAL-ERROR", None)
- else:
- self.logger.info("Browser not responding, setting status to CRASH")
- self.result = False, ("CRASH", None)
- return self.result
-
- def _run(self):
+ def run_func(self):
try:
self.result = True, self.func(self.protocol, self.url, self.timeout)
except errors.ScriptTimeoutException:
@@ -650,11 +661,7 @@
finally:
self.result_flag.set()
- def _timeout(self):
- self.result = False, ("EXTERNAL-TIMEOUT", None)
- self.result_flag.set()
-
class MarionetteTestharnessExecutor(TestharnessExecutor):
supports_testdriver = True
@@ -703,7 +710,8 @@
self.do_testharness,
self.protocol,
self.test_url(test),
- timeout).run()
+ timeout,
+ self.extra_timeout).run()
# The format of data depends on whether the test ran to completion or not
# For asserts we only care about the fact that if it didn't complete, the
# status is in the first field.
@@ -782,7 +790,7 @@
with open(os.path.join(here, "reftest.js")) as f:
self.script = f.read()
- with open(os.path.join(here, "reftest-wait_marionette.js")) as f:
+ with open(os.path.join(here, "reftest-wait_webdriver.js")) as f:
self.wait_script = f.read()
def setup(self, runner):
@@ -853,7 +861,8 @@
self._screenshot,
self.protocol,
test_url,
- timeout).run()
+ timeout,
+ self.extra_timeout).run()
def _screenshot(self, protocol, url, timeout):
protocol.marionette.navigate(url)
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorselenium.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorselenium.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorselenium.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -11,7 +11,7 @@
RefTestExecutor,
RefTestImplementation,
TestharnessExecutor,
- extra_timeout,
+ TimedRunner,
strip_server)
from .protocol import (BaseProtocolPart,
TestharnessProtocolPart,
@@ -237,39 +237,17 @@
self.testharness.load_runner(self.executor.last_environment["protocol"])
-class SeleniumRun(object):
- def __init__(self, func, protocol, url, timeout):
- self.func = func
- self.result = None
- self.protocol = protocol
- self.url = ""
- self.timeout = timeout
- self.result_flag = threading.Event()
-
- def run(self):
+class SeleniumRun(TimedRunner):
+ def set_timeout(self):
timeout = self.timeout
try:
- self.protocol.base.set_timeout(timeout + extra_timeout)
+ self.protocol.base.set_timeout(timeout + self.extra_timeout)
except exceptions.ErrorInResponseException:
self.logger.error("Lost WebDriver connection")
return Stop
- executor = threading.Thread(target=self._run)
- executor.start()
-
- flag = self.result_flag.wait(timeout + 2 * extra_timeout)
- if self.result is None:
- if flag:
- # flag is True unless we timeout; this *shouldn't* happen, but
- # it can if self._run fails to set self.result due to raising
- self.result = False, ("INTERNAL-ERROR", "self._run didn't set a result")
- else:
- self.result = False, ("EXTERNAL-TIMEOUT", None)
-
- return self.result
-
- def _run(self):
+ def run_func(self):
try:
self.result = True, self.func(self.protocol, self.url, self.timeout)
except exceptions.TimeoutException:
@@ -312,10 +290,12 @@
def do_test(self, test):
url = ""
- success, data = ""
+ success, data = ""
+ self.do_testharness,
self.protocol,
url,
- test.timeout * self.timeout_multiplier).run()
+ test.timeout * self.timeout_multiplier,
+ self.extra_timeout).run()
if success:
return self.convert_result(test, data)
@@ -387,10 +367,12 @@
assert viewport_size is None
assert dpi is None
- return SeleniumRun(self._screenshot,
+ return SeleniumRun(self.logger,
+ self._screenshot,
self.protocol,
self.test_url(test),
- test.timeout).run()
+ test.timeout,
+ self.extra_timeout).run()
def _screenshot(self, protocol, url, timeout):
webdriver = protocol.webdriver
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorservo.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorservo.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorservo.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -24,7 +24,6 @@
pytestrunner = None
webdriver = None
-extra_timeout = 5 # seconds
def write_hosts_file(config):
hosts_fd, hosts_path = tempfile.mkstemp()
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorservodriver.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorservodriver.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorservodriver.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,8 +1,6 @@
import json
import os
import socket
-import threading
-import time
import traceback
from .base import (Protocol,
@@ -10,6 +8,7 @@
RefTestExecutor,
RefTestImplementation,
TestharnessExecutor,
+ TimedRunner,
strip_server)
from ..testrunner import Stop
from ..webdriver_server import wait_for_service
@@ -19,9 +18,7 @@
here = os.path.join(os.path.split(__file__)[0])
-extra_timeout = 5
-
def do_delayed_imports():
global webdriver
import webdriver
@@ -131,29 +128,13 @@
break
-class ServoWebDriverRun(object):
- def __init__(self, func, session, url, timeout, current_timeout=None):
- self.func = func
- self.result = None
- self.session = session
- self.url = ""
- self.timeout = timeout
- self.result_flag = threading.Event()
+class ServoWebDriverRun(TimedRunner):
+ def set_timeout(self):
+ pass
- def run(self):
- executor = threading.Thread(target=self._run)
- executor.start()
-
- flag = self.result_flag.wait(self.timeout + extra_timeout)
- if self.result is None:
- assert not flag
- self.result = False, ("EXTERNAL-TIMEOUT", None)
-
- return self.result
-
- def _run(self):
+ def run_func(self):
try:
- self.result = True, self.func(self.session, self.url, self.timeout)
+ self.result = True, self.func(self.protocol.session, self.url, self.timeout)
except webdriver.TimeoutException:
self.result = False, ("EXTERNAL-TIMEOUT", None)
except (socket.timeout, IOError):
@@ -168,14 +149,6 @@
self.result_flag.set()
-def timeout_func(timeout):
- if timeout:
- t0 = time.time()
- return lambda: time.time() - t0 > timeout + extra_timeout
- else:
- return lambda: False
-
-
class ServoWebDriverTestharnessExecutor(TestharnessExecutor):
supports_testdriver = True
@@ -198,7 +171,7 @@
def do_test(self, test):
url = ""
- timeout = test.timeout * self.timeout_multiplier + extra_timeout
+ timeout = test.timeout * self.timeout_multiplier + self.extra_timeout
if timeout != self.timeout:
try:
@@ -208,10 +181,12 @@
self.logger.error("Lost webdriver connection")
return Stop
- success, data = ""
- self.protocol.session,
+ success, data = ""
+ self.do_testharness,
+ self.protocol,
url,
- timeout).run()
+ timeout,
+ self.extra_timeout).run()
if success:
return self.convert_result(test, data)
@@ -286,7 +261,7 @@
assert viewport_size is None
assert dpi is None
- timeout = (test.timeout * self.timeout_multiplier + extra_timeout
+ timeout = (test.timeout * self.timeout_multiplier + self.extra_timeout
if self.debug_info is None else None)
if self.timeout != timeout:
@@ -297,10 +272,12 @@
self.logger.error("Lost webdriver connection")
return Stop
- return ServoWebDriverRun(self._screenshot,
+ return ServoWebDriverRun(self.logger,
+ self._screenshot,
self.protocol.session,
self.test_url(test),
- timeout).run()
+ timeout,
+ self.extra_timeout).run()
def _screenshot(self, session, url, timeout):
session.url = ""
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorwebdriver.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorwebdriver.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/executorwebdriver.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,7 +1,6 @@
import json
import os
import socket
-import sys
import threading
import time
import traceback
@@ -12,7 +11,7 @@
RefTestExecutor,
RefTestImplementation,
TestharnessExecutor,
- extra_timeout,
+ TimedRunner,
strip_server)
from .protocol import (BaseProtocolPart,
TestharnessProtocolPart,
@@ -23,6 +22,7 @@
ActionSequenceProtocolPart,
TestDriverProtocolPart,
GenerateTestReportProtocolPart,
+ SetPermissionProtocolPart,
VirtualAuthenticatorProtocolPart)
from ..testrunner import Stop
@@ -31,6 +31,10 @@
here = os.path.join(os.path.split(__file__)[0])
+class WebDriverCallbackHandler(CallbackHandler):
+ unimplemented_exc = (NotImplementedError, client.UnknownCommandException)
+
+
class WebDriverBaseProtocolPart(BaseProtocolPart):
def setup(self):
self.webdriver = self.parent.webdriver
@@ -203,6 +207,21 @@
json_message = {"message": message}
self.webdriver.send_session_command("POST", "reporting/generate_test_report", json_message)
+
+class WebDriverSetPermissionProtocolPart(SetPermissionProtocolPart):
+ def setup(self):
+ self.webdriver = self.parent.webdriver
+
+ def set_permission(self, descriptor, state, one_realm):
+ permission_params_dict = {
+ "descriptor": descriptor,
+ "state": state,
+ }
+ if one_realm is not None:
+ permission_params_dict["oneRealm"] = one_realm
+ self.webdriver.send_session_command("POST", "permissions", permission_params_dict)
+
+
class WebDriverVirtualAuthenticatorProtocolPart(VirtualAuthenticatorProtocolPart):
def setup(self):
self.webdriver = self.parent.webdriver
@@ -238,6 +257,7 @@
WebDriverActionSequenceProtocolPart,
WebDriverTestDriverProtocolPart,
WebDriverGenerateTestReportProtocolPart,
+ WebDriverSetPermissionProtocolPart,
WebDriverVirtualAuthenticatorProtocolPart]
def __init__(self, executor, browser, capabilities, **kwargs):
@@ -256,7 +276,6 @@
self.webdriver = client.Session(host, port, capabilities=capabilities)
self.webdriver.start()
-
def teardown(self):
self.logger.debug("Hanging up on WebDriver session")
try:
@@ -281,42 +300,15 @@
self.testharness.load_runner(self.executor.last_environment["protocol"])
-class WebDriverRun(object):
- def __init__(self, func, protocol, url, timeout):
- self.func = func
- self.result = None
- self.protocol = protocol
- self.url = ""
- self.timeout = timeout
- self.result_flag = threading.Event()
-
- def run(self):
- timeout = self.timeout
-
+class WebDriverRun(TimedRunner):
+ def set_timeout(self):
try:
- self.protocol.base.set_timeout(timeout + extra_timeout)
+ self.protocol.base.set_timeout(self.timeout + self.extra_timeout)
except client.UnknownErrorException:
self.logger.error("Lost WebDriver connection")
return Stop
- executor = threading.Thread(target=self._run)
- executor.start()
-
- flag = self.result_flag.wait(timeout + 2 * extra_timeout)
- if self.result is None:
- if flag:
- # flag is True unless we timeout; this *shouldn't* happen, but
- # it can if self._run fails to set self.result due to raising
- self.result = False, ("INTERNAL-ERROR", "self._run didn't set a result")
- else:
- message = "Waiting on browser:\n"
- # get a traceback for the current stack of the executor thread
- message += "".join(traceback.format_stack(sys._current_frames()[executor.ident]))
- self.result = False, ("EXTERNAL-TIMEOUT", message)
-
- return self.result
-
- def _run(self):
+ def run_func(self):
try:
self.result = True, self.func(self.protocol, self.url, self.timeout)
except (client.TimeoutException, client.ScriptTimeoutException):
@@ -366,10 +358,12 @@
def do_test(self, test):
url = ""
- success, data = ""
- self.protocol,
- url,
- test.timeout * self.timeout_multiplier).run()
+ success, data = ""
+ self.do_testharness,
+ self.protocol,
+ url,
+ test.timeout * self.timeout_multiplier,
+ self.extra_timeout).run()
if success:
return self.convert_result(test, data)
@@ -386,7 +380,7 @@
parent_window,
timeout=5*self.timeout_multiplier)
self.protocol.base.set_window(test_window)
- handler = CallbackHandler(self.logger, protocol, test_window)
+ handler = WebDriverCallbackHandler(self.logger, protocol, test_window)
protocol.webdriver.url = ""
if not self.supports_eager_pageload:
@@ -485,10 +479,12 @@
assert viewport_size is None
assert dpi is None
- return WebDriverRun(self._screenshot,
- self.protocol,
- self.test_url(test),
- test.timeout).run()
+ return WebDriverRun(self.logger,
+ self._screenshot,
+ self.protocol,
+ self.test_url(test),
+ test.timeout,
+ self.extra_timeout).run()
def _screenshot(self, protocol, url, timeout):
webdriver = protocol.webdriver
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/protocol.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/protocol.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/protocol.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -305,6 +305,22 @@
pass
+class SetPermissionProtocolPart(ProtocolPart):
+ """Protocol part for setting permissions"""
+ __metaclass__ = ABCMeta
+
+ name = "set_permission"
+
+ @abstractmethod
+ def set_permission(self, descriptor, state, _one_realm_=False):
+ """Set permission state.
+
+ :param descriptor: A PermissionDescriptor object.
+ :param state: The state to set the permission to.
+ :param one_realm: Whether to set the permission for only one realm."""
+ pass
+
+
class ActionSequenceProtocolPart(ProtocolPart):
"""Protocol part for performing trusted clicks"""
__metaclass__ = ABCMeta
Deleted: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/reftest-wait_marionette.js (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/reftest-wait_marionette.js 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/reftest-wait_marionette.js 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,19 +0,0 @@
-var callback = arguments[arguments.length - 1];
-
-function test(x) {
- if (!root.classList.contains("reftest-wait")) {
- observer.disconnect();
- callback();
- }
-}
-
-var root = document.documentElement;
-var observer = new MutationObserver(test);
-
-observer.observe(root, {attributes: true});
-
-if (document.readyState != "complete") {
- addEventListener('load', test);
-} else {
- test();
-}
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/reftest-wait_webdriver.js (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/reftest-wait_webdriver.js 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/executors/reftest-wait_webdriver.js 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,21 +1,20 @@
var callback = arguments[arguments.length - 1];
+var observer = null;
+var root = document.documentElement;
-function root_wait() {
- if (!root.classList.contains("reftest-wait")) {
- observer.disconnect();
-
- if (Document.prototype.hasOwnProperty("fonts")) {
- document.fonts.ready.then(ready_for_screenshot);
- } else {
- // This might take the screenshot too early, depending on whether the
- // load event is blocked on fonts being loaded. See:
- // https://github.com/w3c/csswg-drafts/issues/1088
- ready_for_screenshot();
- }
+function wait_load() {
+ if (Document.prototype.hasOwnProperty("fonts")) {
+ document.fonts.ready.then(wait_paints);
+ } else {
+ // This might take the screenshot too early, depending on whether the
+ // load event is blocked on fonts being loaded. See:
+ // https://github.com/w3c/csswg-drafts/issues/1088
+ wait_paints();
}
}
-function ready_for_screenshot() {
+
+function wait_paints() {
// As of 2017-04-05, the Chromium web browser exhibits a rendering bug
// (https://bugs.chromium.org/p/chromium/issues/detail?id=708757) that
// produces instability during screen capture. The following use of
@@ -27,18 +26,29 @@
requestAnimationFrame(function() {
requestAnimationFrame(function() {
- callback();
+ screenshot_if_ready();
});
});
}
-var root = document.documentElement;
-var observer = new MutationObserver(root_wait);
+function screenshot_if_ready() {
+ if (root.classList.contains("reftest-wait") &&
+ observer === null) {
+ observer = new MutationObserver(wait_paints);
+ observer.observe(root, {attributes: true});
+ var event = new Event("TestRendered", {bubbles: true});
+ root.dispatchEvent(event);
+ return;
+ }
+ if (observer !== null) {
+ observer.disconnect();
+ }
+ callback();
+}
-observer.observe(root, {attributes: true});
if (document.readyState != "complete") {
- addEventListener('load', root_wait);
+ addEventListener('load', wait_load);
} else {
- root_wait();
+ wait_load();
}
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/testdriver-extra.js (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/testdriver-extra.js 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/testdriver-extra.js 2020-01-08 11:53:22 UTC (rev 254192)
@@ -19,7 +19,7 @@
result = JSON.parse(data.message).result
pending_resolve(result);
} else {
- pending_reject();
+ pending_reject(`${data.status}: ${data.message}`);
}
});
@@ -126,6 +126,15 @@
return pending_promise;
};
+ window.test_driver_internal.set_permission = function(permission_params) {
+ const pending_promise = new Promise(function(resolve, reject) {
+ pending_resolve = resolve;
+ pending_reject = reject;
+ });
+ window.__wptrunner_message_queue.push({"type": "action", "action": "set_permission", permission_params});
+ return pending_promise;
+ };
+
window.test_driver_internal.add_virtual_authenticator = function(config) {
const pending_promise = new Promise(function(resolve, reject) {
pending_resolve = resolve;
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/testrunner.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/testrunner.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/testrunner.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -334,6 +334,8 @@
# This may not really be what we want
self.daemon = True
+ self.timer = None
+
self.max_restarts = 5
self.browser = None
@@ -559,8 +561,30 @@
self.logger.info("Run %d/%d" % (self.run_count, self.rerun))
self.send_message("reset")
self.run_count += 1
+ if self.debug_info is None:
+ # Factor of 3 on the extra timeout here is based on allowing the executor
+ # at least test.timeout + 2 * extra_timeout to complete,
+ # which in turn is based on having several layers of timeout inside the executor
+ wait_timeout = (self.state.test.timeout * self.executor_kwargs['timeout_multiplier'] +
+ 3 * self.executor_cls.extra_timeout)
+ self.timer = threading.Timer(wait_timeout, self._timeout)
+
self.send_message("run_test", self.state.test)
+ if self.timer:
+ self.timer.start()
+ def _timeout(self):
+ # This is executed in a different thread (threading.Timer).
+ self.logger.info("Got timeout in harness")
+ test = self.state.test
+ self.inject_message(
+ "test_ended",
+ test,
+ (test.result_cls("EXTERNAL-TIMEOUT",
+ "TestRunner hit external timeout "
+ "(this may indicate a hang)"), []),
+ )
+
def test_ended(self, test, results):
"""Handle the end of a test.
@@ -567,15 +591,28 @@
Output the result of each subtest, and the result of the overall
harness to the logs.
"""
- assert isinstance(self.state, RunnerManagerState.running)
- assert test == self.state.test
+ if ((not isinstance(self.state, RunnerManagerState.running)) or
+ (test != self.state.test)):
+ # Due to inherent race conditions in EXTERNAL-TIMEOUT, we might
+ # receive multiple test_ended for a test (e.g. from both Executor
+ # and TestRunner), in which case we ignore the duplicate message.
+ self.logger.error("Received unexpected test_ended for %s" % test)
+ return
+ if self.timer is not None:
+ self.timer.cancel()
# Write the result of each subtest
file_result, test_results = results
subtest_unexpected = False
+ expect_any_subtest_status = test.expect_any_subtest_status()
+ if expect_any_subtest_status:
+ self.logger.debug("Ignoring subtest statuses for test %s" % test.id)
for result in test_results:
if test.disabled(result.name):
continue
- expected = test.expected(result.name)
+ if expect_any_subtest_status:
+ expected = result.status
+ else:
+ expected = test.expected(result.name)
known_intermittent = test.known_intermittent(result.name)
is_unexpected = expected != result.status and result.status not in known_intermittent
@@ -733,8 +770,13 @@
return RunnerManagerState.stop()
def send_message(self, command, *args):
+ """Send a message to the remote queue (to Executor)."""
self.remote_queue.put((command, args))
+ def inject_message(self, command, *args):
+ """Inject a message to the command queue (from Executor)."""
+ self.command_queue.put((command, args))
+
def cleanup(self):
self.logger.debug("TestRunnerManager cleanup")
if self.browser:
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/tests/__init__.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/tests/__init__.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/tests/__init__.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -0,0 +1,7 @@
+import os
+import sys
+
+here = os.path.abspath(os.path.split(__file__)[0])
+sys.path.insert(0, os.path.join(here, os.pardir, os.pardir, os.pardir))
+
+import localpaths as _localpaths # noqa: F401
Added: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/tests/browsers/test_webkitgtk.py (0 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/tests/browsers/test_webkitgtk.py (rev 0)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/tests/browsers/test_webkitgtk.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -0,0 +1,57 @@
+from os.path import join, dirname
+
+import pytest
+
+from wptserve.config import ConfigBuilder
+from ..base import active_products
+from wptrunner import environment, products
+
+test_paths = {"/": {"tests_path": join(dirname(__file__), "..", "..", "..", "..", "..")}} # repo root
+environment.do_delayed_imports(None, test_paths)
+
+
+@active_products("product")
+def test_webkitgtk_certificate_domain_list(product):
+
+ def domain_is_inside_certificate_list_cert(domain_to_find, webkitgtk_certificate_list, cert_file):
+ for domain in webkitgtk_certificate_list:
+ if domain["host"] == domain_to_find and domain["certificateFile"] == cert_file:
+ return True
+ return False
+
+ if product not in ["epiphany", "webkit", "webkitgtk_minibrowser"]:
+ pytest.skip("%s doesn't support certificate_domain_list" % product)
+
+ (check_args,
+ target_browser_cls, get_browser_kwargs,
+ executor_classes, get_executor_kwargs,
+ env_options, get_env_extras, run_info_extras) = products.load_product({}, product)
+
+ cert_file = "/home/user/wpt/tools/certs/cacert.pem"
+ valid_domains_test = ["a.example.org", "b.example.org", "example.org",
+ "a.example.net", "b.example.net", "example.net"]
+ invalid_domains_test = ["x.example.org", "y.example.org", "example.it",
+ "x.example.net", "y.example.net", "z.example.net"]
+ kwargs = {}
+ kwargs["timeout_multiplier"] = 1
+ kwargs["debug_info"] = None
+ kwargs["host_cert_path"] = cert_file
+ kwargs["webkit_port"] = "gtk"
+ kwargs["binary"] = None
+ kwargs["webdriver_binary"] = None
+ with ConfigBuilder(browser_host="example.net",
+ alternate_hosts={"alt": "example.org"},
+ subdomains={"a", "b"},
+ not_subdomains={"x", "y"}) as env_config:
+
+ executor_args = get_executor_kwargs(None, env_config, None, None, **kwargs)
+ assert('capabilities' in executor_args)
+ assert('webkitgtk:browserOptions' in executor_args['capabilities'])
+ assert('certificates' in executor_args['capabilities']['webkitgtk:browserOptions'])
+ cert_list = executor_args['capabilities']['webkitgtk:browserOptions']['certificates']
+ for valid_domain in valid_domains_test:
+ assert(domain_is_inside_certificate_list_cert(valid_domain, cert_list, cert_file))
+ assert(not domain_is_inside_certificate_list_cert(valid_domain, cert_list, cert_file + ".backup_non_existent"))
+ for invalid_domain in invalid_domains_test:
+ assert(not domain_is_inside_certificate_list_cert(invalid_domain, cert_list, cert_file))
+ assert(not domain_is_inside_certificate_list_cert(invalid_domain, cert_list, cert_file + ".backup_non_existent"))
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/tests/test_wpttest.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/tests/test_wpttest.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/tests/test_wpttest.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -68,6 +68,11 @@
expected: [OK, FAIL]
"""
+test_7 = """\
+[7.html]
+ blink_expect_any_subtest_status: yep
+"""
+
test_fuzzy = """\
[fuzzy.html]
fuzzy: fuzzy-ref.html:1;200
@@ -205,6 +210,14 @@
@pytest.mark.xfail(sys.version[0] == "3",
reason="bytes/text confusion in py3")
+def test_expect_any_subtest_status():
+ test_obj = make_test_object(test_7, "a/7.html", 7, ("test", "a", 8), None, False)
+ assert test_obj.expected() == "OK"
+ assert test_obj.expect_any_subtest_status() is True
+
+
[email protected](sys.version[0] == "3",
+ reason="bytes/text confusion in py3")
def test_metadata_fuzzy():
manifest_data = {
"items": {"reftest": {"a/fuzzy.html": [["a/fuzzy.html",
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/wptrunner.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/wptrunner.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/wptrunner.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -342,6 +342,7 @@
handler = handlers.LogLevelFilter(logged_critical, "CRITICAL")
logger.add_handler(handler)
+ rv = False
try:
if kwargs["list_test_groups"]:
list_test_groups(**kwargs)
@@ -350,11 +351,12 @@
elif kwargs["list_tests"]:
list_tests(**kwargs)
elif kwargs["verify"] or kwargs["stability"]:
- return check_stability(**kwargs) or logged_critical.has_log
+ rv = check_stability(**kwargs) or logged_critical.has_log
else:
- return not run_tests(**kwargs) or logged_critical.has_log
+ rv = not run_tests(**kwargs) or logged_critical.has_log
finally:
logger.remove_handler(handler)
+ return rv
def main():
Modified: trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/wpttest.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/wpttest.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/tools/wptrunner/wptrunner/wpttest.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -164,8 +164,14 @@
self.environment = {"protocol": protocol, "prefs": self.prefs}
def __eq__(self, other):
+ if not isinstance(other, Test):
+ return False
return self.id == other.id
+ # Python 2 does not have this delegation, while Python 3 does.
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
def update_metadata(self, metadata=None):
if metadata is None:
metadata = {}
@@ -347,6 +353,17 @@
except KeyError:
return []
+ def expect_any_subtest_status(self):
+ metadata = self._get_metadata()
+ if metadata is None:
+ return False
+ try:
+ # This key is used by the Blink CI to ignore subtest statuses
+ metadata.get("blink_expect_any_subtest_status")
+ return True
+ except KeyError:
+ return False
+
def __repr__(self):
return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.id)
Modified: trunk/WebDriverTests/imported/w3c/webdriver/tests/find_element_from_element/find.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/webdriver/tests/find_element_from_element/find.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/webdriver/tests/find_element_from_element/find.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -1,4 +1,3 @@
-# META: timeout=long
import pytest
from webdriver.transport import Response
Modified: trunk/WebDriverTests/imported/w3c/webdriver/tests/new_session/default_values.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/webdriver/tests/new_session/default_values.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/webdriver/tests/new_session/default_values.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -44,12 +44,3 @@
}})
value = assert_success(response)
assert value["capabilities"]["pageLoadStrategy"] == "normal"
-
-
-def test_valid_but_unmatchable_key(new_session, add_browser_capabilities):
- response, _ = new_session({"capabilities": {"firstMatch": [
- add_browser_capabilities({"pageLoadStrategy": "eager", "foo:unmatchable": True}),
- {"pageLoadStrategy": "none"},
- ]}})
- value = assert_success(response)
- assert value["capabilities"]["pageLoadStrategy"] == "none"
Modified: trunk/WebDriverTests/imported/w3c/webdriver/tests/perform_actions/support/keys.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/webdriver/tests/perform_actions/support/keys.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/webdriver/tests/perform_actions/support/keys.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -379,7 +379,7 @@
"value": u"\ue012",
},
"META": {
- "code": "OSLeft",
+ "code": "MetaLeft",
"ctrl": False,
"key": "Meta",
"location": 1,
@@ -631,7 +631,7 @@
"value": u"\ue05c",
},
"R_META": {
- "code": "OSRight",
+ "code": "MetaRight",
"ctrl": False,
"key": "Meta",
"location": 2,
Modified: trunk/WebDriverTests/imported/w3c/webdriver/tests/perform_actions/validity.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/webdriver/tests/perform_actions/validity.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/webdriver/tests/perform_actions/validity.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -12,7 +12,7 @@
@pytest.mark.parametrize("action_type", ["none", "key", "pointer"])
def test_pause_positive_integer(session, action_type):
- for valid_duration in [0, 1]:
+ for valid_duration in [0.0, 1]:
actions = [{
"type": action_type,
"id": "foobar",
@@ -38,7 +38,7 @@
@pytest.mark.parametrize("action_type", ["none", "key", "pointer"])
def test_pause_invalid_types(session, action_type):
- for invalid_type in [0.0, None, "foo", True, [], {}]:
+ for invalid_type in [0.1, None, "foo", True, [], {}]:
actions = [{
"type": action_type,
"id": "foobar",
Modified: trunk/WebDriverTests/imported/w3c/webdriver/tests/send_alert_text/send.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/webdriver/tests/send_alert_text/send.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/webdriver/tests/send_alert_text/send.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -4,7 +4,6 @@
from webdriver.transport import Response
from tests.support.asserts import assert_error, assert_success
-from tests.support.authentication import basic_authentication
from tests.support.inline import inline
from tests.support.sync import Poll
@@ -58,13 +57,6 @@
assert_error(response, "element not interactable")
-def test_alert_unsupported_operation(session):
- session.url = ""
-
- response = send_alert_text(session, "Federer")
- assert_error(response, "unsupported operation")
-
-
@pytest.mark.parametrize("text", ["", "Federer", " Fed erer ", "Fed\terer"])
def test_send_alert_text(session, page, text):
send_response = send_alert_text(session, text)
Modified: trunk/WebDriverTests/imported/w3c/webdriver/tests/set_window_rect/set.py (254191 => 254192)
--- trunk/WebDriverTests/imported/w3c/webdriver/tests/set_window_rect/set.py 2020-01-08 10:29:40 UTC (rev 254191)
+++ trunk/WebDriverTests/imported/w3c/webdriver/tests/set_window_rect/set.py 2020-01-08 11:53:22 UTC (rev 254192)
@@ -135,9 +135,9 @@
session.window.fullscreen()
assert is_fullscreen(session)
- response = set_window_rect(session, {"width": 400, "height": 400})
+ response = set_window_rect(session, {"width": 600, "height": 400})
value = assert_success(response)
- assert value["width"] == 400
+ assert value["width"] == 600
assert value["height"] == 400
assert not is_fullscreen(session)
