Diff
Modified: releases/WebKitGTK/webkit-2.18/Tools/ChangeLog (226056 => 226057)
--- releases/WebKitGTK/webkit-2.18/Tools/ChangeLog 2017-12-18 17:40:54 UTC (rev 226056)
+++ releases/WebKitGTK/webkit-2.18/Tools/ChangeLog 2017-12-18 17:41:02 UTC (rev 226057)
@@ -1,3 +1,56 @@
+2017-12-14 Carlos Garcia Campos <cgar...@igalia.com>
+
+ WebDriver: add a common way to run tests with pytest
+ https://bugs.webkit.org/show_bug.cgi?id=180800
+
+ Reviewed by Carlos Alberto Lopez Perez.
+
+ We currently use pytestrunner from wpt for w3c tests and our own code for selenium tests. Using the same code
+ for both would simplify everything, but also allows us to have a custom results recorder to support other test
+ expectations like TIMEOUT. The code to run selenium tests with pytest has been moved to a new file
+ pytest_runner.py and made generic to be used also for w3c tests.
+
+ * Scripts/webkitpy/webdriver_tests/pytest_runner.py: Added.
+ (TemporaryDirectory):
+ (TemporaryDirectory.__enter__):
+ (TemporaryDirectory.__exit__):
+ (CollectRecorder):
+ (CollectRecorder.__init__):
+ (CollectRecorder.pytest_collectreport):
+ (HarnessResultRecorder):
+ (HarnessResultRecorder.__init__):
+ (HarnessResultRecorder.pytest_collectreport):
+ (SubtestResultRecorder):
+ (SubtestResultRecorder.__init__):
+ (SubtestResultRecorder.pytest_runtest_logreport):
+ (SubtestResultRecorder._was_timeout):
+ (SubtestResultRecorder.record_pass):
+ (SubtestResultRecorder.record_fail):
+ (SubtestResultRecorder.record_error):
+ (SubtestResultRecorder.record_skip):
+ (SubtestResultRecorder.record):
+ (collect):
+ (run):
+ * Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py:
+ (do_delayed_imports): Import pytest_runner here to avoid cycles.
+ (WebDriverSeleniumExecutor.__init__): Save the driver parameter as args member and call do_delayed_imports() if
+ needed.
+ (WebDriverSeleniumExecutor.collect): Use pytest_runner.
+ (WebDriverSeleniumExecutor.run): Ditto.
+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py:
+ (WebDriverTestRunner.print_results): Handle all possible tests results.
+ (WebDriverTestRunner.print_results.report): Helper to dump test results.
+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py:
+ (WebDriverTestRunnerSelenium.run):
+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
+ (WebDriverTestRunnerW3C.__init__): Do not set PYTEST_TIMEOUT env var.
+ (WebDriverTestRunnerW3C._is_test): Fix check for support files.
+ (WebDriverTestRunnerW3C.run): Pass the timeout as parameter to WebDriverW3CExecutor.run().
+ * Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py:
+ (do_delayed_imports): Import pytest_runner here to avoid cycles.
+ (WebDriverW3CExecutor.__init__): Call do_delayed_imports() if needed.
+ (WebDriverW3CExecutor.run): Use pytest_runner.
+
2017-12-11 Carlos Garcia Campos <cgar...@igalia.com>
[GTK] WebDriver: run-webdriver-tests is leaking a DumpRenderTree directory in tmp
Added: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py (0 => 226057)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py (rev 0)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py 2017-12-18 17:41:02 UTC (rev 226057)
@@ -0,0 +1,169 @@
+# Copyright (C) 2017 Igalia S.L.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import errno
+import json
+import os
+import shutil
+import sys
+import tempfile
+
+import webkitpy.thirdparty.autoinstalled.pytest
+import webkitpy.thirdparty.autoinstalled.pytest_timeout
+import pytest
+from _pytest.main import EXIT_INTERNALERROR
+
+
+class TemporaryDirectory(object):
+
+ def __enter__(self):
+ self.path = tempfile.mkdtemp(prefix="pytest-")
+ return self.path
+
+ def __exit__(self, *args):
+ try:
+ shutil.rmtree(self.path)
+ except OSError as e:
+ # no such file or directory
+ if e.errno != errno.ENOENT:
+ raise
+
+
+class CollectRecorder(object):
+
+ def __init__(self):
+ self.tests = []
+
+ def pytest_collectreport(self, report):
+ if report.nodeid:
+ self.tests.append(report.nodeid)
+
+
+class HarnessResultRecorder(object):
+
+ def __init__(self):
+ self.outcome = ('OK', None)
+
+ def pytest_collectreport(self, report):
+ if report.outcome == 'failed':
+ self.outcome = ('ERROR', None)
+ elif report.outcome == 'skipped':
+ self.outcome = ('SKIP', None)
+
+
+class SubtestResultRecorder(object):
+
+ def __init__(self):
+ self.results = []
+
+ def pytest_runtest_logreport(self, report):
+ if report.passed and report.when == 'call':
+ self.record_pass(report)
+ elif report.failed:
+ if report.when != 'call':
+ self.record_error(report)
+ else:
+ self.record_fail(report)
+ elif report.skipped:
+ self.record_skip(report)
+
+ def _was_timeout(self, report):
+ return hasattr(report.longrepr, 'reprcrash') and report.longrepr.reprcrash.message.startswith('Failed: Timeout >')
+
+ def record_pass(self, report):
+ if hasattr(report, 'wasxfail'):
+ if report.wasxfail == 'Timeout':
+ self.record(report.nodeid, 'XPASS_TIMEOUT')
+ else:
+ self.record(report.nodeid, 'XPASS')
+ else:
+ self.record(report.nodeid, 'PASS')
+
+ def record_fail(self, report):
+ if self._was_timeout(report):
+ self.record(report.nodeid, 'TIMEOUT', stack=report.longrepr)
+ else:
+ self.record(report.nodeid, 'FAIL', stack=report.longrepr)
+
+ def record_error(self, report):
+ # error in setup/teardown
+ if report.when != 'call':
+ message = '%s error' % report.when
+ self.record(report.nodeid, 'ERROR', message, report.longrepr)
+
+ def record_skip(self, report):
+ if hasattr(report, 'wasxfail'):
+ if self._was_timeout(report) and report.wasxfail != 'Timeout':
+ self.record(report.nodeid, 'TIMEOUT', stack=report.longrepr)
+ else:
+ self.record(report.nodeid, 'XFAIL')
+ else:
+ self.record(report.nodeid, 'SKIP')
+
+ def record(self, test, status, message=None, stack=None):
+ if stack is not None:
+ stack = str(stack)
+ new_result = (test, status, message, stack)
+ self.results.append(new_result)
+
+
+def collect(directory, args):
+ collect_recorder = CollectRecorder()
+ stdout = sys.stdout
+ with open(os.devnull, 'wb') as devnull:
+ sys.stdout = devnull
+ with TemporaryDirectory() as cache_directory:
+ pytest.main(args + ['--collect-only',
+ '--basetemp', cache_directory,
+ directory],
+ plugins=[collect_recorder])
+ sys.stdout = stdout
+ return collect_recorder.tests
+
+
+def run(path, args, timeout, env={}):
+ harness_recorder = HarnessResultRecorder()
+ subtests_recorder = SubtestResultRecorder()
+ _environ = dict(os.environ)
+ os.environ.clear()
+ os.environ.update(env)
+
+ with TemporaryDirectory() as cache_directory:
+ try:
+ result = pytest.main(args + ['--verbose',
+ '--capture=no',
+ '--basetemp', cache_directory,
+ '--showlocals',
+ '--timeout=%s' % timeout,
+ '-p', 'no:cacheprovider',
+ '-p', 'pytest_timeout',
+ path],
+ plugins=[harness_recorder, subtests_recorder])
+ if result == EXIT_INTERNALERROR:
+ harness_recorder.outcome = ('ERROR', None)
+ except Exception as e:
+ harness_recorder.outcome = ('ERROR', str(e))
+
+ os.environ.clear()
+ os.environ.update(_environ)
+
+ return harness_recorder.outcome, subtests_recorder.results
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py (226056 => 226057)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py 2017-12-18 17:40:54 UTC (rev 226056)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py 2017-12-18 17:41:02 UTC (rev 226057)
@@ -26,35 +26,18 @@
from webkitpy.common.system.filesystem import FileSystem
from webkitpy.common.webkit_finder import WebKitFinder
-import webkitpy.thirdparty.autoinstalled.mozlog
-import webkitpy.thirdparty.autoinstalled.pytest
-import webkitpy.thirdparty.autoinstalled.pytest_timeout
-import pytest
-# Since W3C tests also use pytest, we use pytest and some other tools for selenium too.
-w3c_tools_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests', 'imported', 'w3c', 'tools')
+pytest_runner = None
-def _ensure_directory_in_path(directory):
- if not directory in sys.path:
- sys.path.insert(0, directory)
-_ensure_directory_in_path(os.path.join(w3c_tools_dir, 'wptrunner'))
+def do_delayed_imports():
+ global pytest_runner
+ import webkitpy.webdriver_tests.pytest_runner as pytest_runner
-from wptrunner.executors.pytestrunner.runner import HarnessResultRecorder, SubtestResultRecorder, TemporaryDirectory
_log = logging.getLogger(__name__)
-class CollectRecorder(object):
-
- def __init__(self):
- self.tests = []
-
- def pytest_collectreport(self, report):
- if report.nodeid:
- self.tests.append(report.nodeid)
-
-
class WebDriverSeleniumExecutor(object):
def __init__(self, driver, display_driver):
@@ -69,43 +52,13 @@
self._env.update(display_driver._setup_environ_for_test())
self._env.update(driver.browser_env())
- self._name = driver.selenium_name()
+ self._args = ['--driver=%s' % driver.selenium_name()]
+ if pytest_runner is None:
+ do_delayed_imports()
+
def collect(self, directory):
- collect_recorder = CollectRecorder()
- stdout = sys.stdout
- with open(os.devnull, 'wb') as devnull:
- sys.stdout = devnull
- with TemporaryDirectory() as cache_directory:
- pytest.main(['--driver=%s' % self._name,
- '--collect-only',
- '--basetemp', cache_directory,
- directory], plugins=[collect_recorder])
- sys.stdout = stdout
- return collect_recorder.tests
+ return pytest_runner.collect(directory, self._args)
- def run(self, test, timeout=0):
- harness_recorder = HarnessResultRecorder()
- subtests_recorder = SubtestResultRecorder()
- _environ = dict(os.environ)
- os.environ.clear()
- os.environ.update(self._env)
-
- with TemporaryDirectory() as cache_directory:
- try:
- pytest.main(['--driver=%s' % self._name,
- '--verbose',
- '--capture=no',
- '--basetemp', cache_directory,
- '--showlocals',
- '--timeout=%s' % timeout,
- '-p', 'no:cacheprovider',
- '-p', 'pytest_timeout',
- test], plugins=[harness_recorder, subtests_recorder])
- except Exception as e:
- harness_recorder.outcome = ("ERROR", str(e))
-
- os.environ.clear()
- os.environ.update(_environ)
-
- return harness_recorder.outcome, subtests_recorder.results
+ def run(self, test, timeout):
+ return pytest_runner.run(test, self._args, timeout, self._env)
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py (226056 => 226057)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py 2017-12-18 17:40:54 UTC (rev 226056)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py 2017-12-18 17:41:02 UTC (rev 226057)
@@ -67,8 +67,10 @@
def print_results(self):
results = {}
+ expected_count = 0
passed_count = 0
failures_count = 0
+ timeout_count = 0
test_results = []
for runner in self._runners:
test_results.extend(runner.results())
@@ -75,11 +77,17 @@
for result in test_results:
if result.status == 'OK':
for subtest, status, _, _ in result.subtest_results:
- if status == 'PASS':
- passed_count += 1
+ if status in ['PASS', 'SKIP', 'XFAIL']:
+ expected_count += 1
elif status in ['FAIL', 'ERROR']:
results.setdefault('FAIL', []).append(os.path.join(os.path.dirname(result.test), subtest))
failures_count += 1
+ elif status == 'TIMEOUT':
+ results.setdefault('TIMEOUT', []).append(os.path.join(os.path.dirname(result.test), subtest))
+ timeout_count += 1
+ elif status in ['XPASS', 'XPASS_TIMEOUT']:
+ results.setdefault(status, []).append(os.path.join(os.path.dirname(result.test), subtest))
+ passed_count += 1
else:
# FIXME: handle other results.
pass
@@ -90,14 +98,26 @@
_log.info('All tests run as expected')
return
- _log.info('%d tests ran as expected, %d didn\'t\n' % (passed_count, failures_count))
+ _log.info('%d tests ran as expected, %d didn\'t\n' % (expected_count, failures_count + timeout_count + passed_count))
- if 'FAIL' in results:
- failed = results['FAIL']
- _log.info('Unexpected failures (%d)' % len(failed))
- for test in failed:
+ def report(status, actual, expected=None):
+ if status not in results:
+ return
+
+ tests = results[status]
+ if expected is None:
+ _log.info('Unexpected %s (%d)' % (actual, len(tests)))
+ else:
+ _log.info('Expected to %s, but %s (%d)' % (expected, actual, len(tests)))
+ for test in tests:
_log.info(' %s' % test)
+ _log.info('')
+ report('XPASS', 'passed', 'fail')
+ report('XPASS_TIMEOUT', 'passed', 'timeout')
+ report('FAIL', 'failures')
+ report('TIMEOUT', 'timeouts')
+
def dump_results_to_json_file(self, output_path):
json_results = {}
json_results['results'] = []
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py (226056 => 226057)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py 2017-12-18 17:40:54 UTC (rev 226056)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py 2017-12-18 17:41:02 UTC (rev 226057)
@@ -73,7 +73,6 @@
timeout = self._port.get_option('timeout')
for test in tests:
test_name = os.path.relpath(test, self._tests_dir())
- print test_name
harness_result, test_results = executor.run(test, timeout)
result = WebDriverTestResult(test_name, *harness_result)
if harness_result[0] == 'OK':
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py (226056 => 226057)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py 2017-12-18 17:40:54 UTC (rev 226056)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py 2017-12-18 17:41:02 UTC (rev 226057)
@@ -39,12 +39,8 @@
self._port = port
self._driver = driver
self._display_driver = display_driver
+ self._results = []
- timeout = self._port.get_option('timeout')
- if timeout > 0:
- os.environ['PYTEST_TIMEOUT'] = str(timeout)
-
- self._results = []
self._server = WebDriverW3CWebServer(self._port)
def _tests_dir(self):
@@ -72,7 +68,7 @@
return False
if os.path.basename(test) in ['conftest.py', '__init__.py']:
return False
- if os.path.dirname(test) == 'support':
+ if os.path.basename(os.path.dirname(test)) == 'support':
return False
return True
@@ -88,10 +84,11 @@
executor = WebDriverW3CExecutor(self._driver, self._server, self._display_driver)
executor.setup()
+ timeout = self._port.get_option('timeout')
try:
for test in tests:
test_name = os.path.relpath(test, self._tests_dir())
- harness_result, test_results = executor.run(test)
+ harness_result, test_results = executor.run(test, timeout)
result = WebDriverTestResult(test_name, *harness_result)
if harness_result[0] == 'OK':
for subtest, status, message, backtrace in test_results:
Modified: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py (226056 => 226057)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py 2017-12-18 17:40:54 UTC (rev 226056)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/webdriver_tests/webdriver_w3c_executor.py 2017-12-18 17:41:02 UTC (rev 226057)
@@ -22,6 +22,7 @@
import logging
import os
+import json
import sys
from webkitpy.common.system.filesystem import FileSystem
@@ -28,8 +29,6 @@
from webkitpy.common.webkit_finder import WebKitFinder
import webkitpy.thirdparty.autoinstalled.mozlog
import webkitpy.thirdparty.autoinstalled.mozprocess
-import webkitpy.thirdparty.autoinstalled.pytest
-import webkitpy.thirdparty.autoinstalled.pytest_timeout
from mozlog import structuredlog
w3c_tools_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests', 'imported', 'w3c', 'tools')
@@ -44,6 +43,14 @@
from wptrunner.executors.base import WdspecExecutor, WebDriverProtocol
from wptrunner.webdriver_server import WebDriverServer
+pytest_runner = None
+
+
+def do_delayed_imports():
+ global pytest_runner
+ import webkitpy.webdriver_tests.pytest_runner as pytest_runner
+
+
_log = logging.getLogger(__name__)
@@ -128,6 +135,9 @@
server_config = {'host': server.host(), 'ports': {'http': [str(server.port())]}}
WdspecExecutor.__init__(self, driver.browser_name(), server_config, driver.binary_path(), None, capabilities=driver.capabilities())
+ if pytest_runner is None:
+ do_delayed_imports()
+
def setup(self):
self.runner = TestRunner()
self.protocol.setup(self.runner)
@@ -135,6 +145,10 @@
def teardown(self):
self.protocol.teardown()
- def run(self, path):
- # Timeout here doesn't really matter because it's ignored, so we pass 0.
- return self.do_wdspec(self.protocol.session_config, path, 0)
+ def run(self, test, timeout):
+ env = {'WD_HOST': self.protocol.session_config['host'],
+ 'WD_PORT': str(self.protocol.session_config['port']),
+ 'WD_CAPABILITIES': json.dumps(self.protocol.session_config['capabilities']),
+ 'WD_SERVER_CONFIG': json.dumps(self.server_config)}
+ args = ['--strict', '-p', 'no:mozlog']
+ return pytest_runner.run(test, args, timeout, env)
Modified: releases/WebKitGTK/webkit-2.18/WebDriverTests/ChangeLog (226056 => 226057)
--- releases/WebKitGTK/webkit-2.18/WebDriverTests/ChangeLog 2017-12-18 17:40:54 UTC (rev 226056)
+++ releases/WebKitGTK/webkit-2.18/WebDriverTests/ChangeLog 2017-12-18 17:41:02 UTC (rev 226057)
@@ -1,3 +1,14 @@
+2017-12-14 Carlos Garcia Campos <cgar...@igalia.com>
+
+ WebDriver: add a common way to run tests with pytest
+ https://bugs.webkit.org/show_bug.cgi?id=180800
+
+ Reviewed by Carlos Alberto Lopez Perez.
+
+ Remove conftest.py since pytest_timeout plugin is now always loaded from the command line.
+
+ * imported/w3c/conftest.py: Removed.
+
2017-12-04 Carlos Garcia Campos <cgar...@igalia.com>
Unreviewed. Update W3C WebDriver imported tests.
Deleted: releases/WebKitGTK/webkit-2.18/WebDriverTests/imported/w3c/conftest.py (226056 => 226057)
--- releases/WebKitGTK/webkit-2.18/WebDriverTests/imported/w3c/conftest.py 2017-12-18 17:40:54 UTC (rev 226056)
+++ releases/WebKitGTK/webkit-2.18/WebDriverTests/imported/w3c/conftest.py 2017-12-18 17:41:02 UTC (rev 226057)
@@ -1 +0,0 @@
-pytest_plugins = 'pytest_timeout',
