Diff
Modified: trunk/Tools/ChangeLog (227126 => 227127)
--- trunk/Tools/ChangeLog 2018-01-18 06:50:12 UTC (rev 227126)
+++ trunk/Tools/ChangeLog 2018-01-18 06:51:30 UTC (rev 227127)
@@ -1,5 +1,25 @@
2018-01-17 Carlos Garcia Campos <[email protected]>
+ WebDriver: ignore the driver in selenium test names when getting expectations
+ https://bugs.webkit.org/show_bug.cgi?id=181738
+
+ Reviewed by Carlos Alberto Lopez Perez.
+
+ In selenium tests, the driver is added as a parameter to every test which results in tests names like
+ foo[DriverName] or foo[DriverName-param2] in case of tests using more parameters. We don't want to include the
+ driver name in the test expectations file, so we need to remove it when querying the expectations.
+
+ * Scripts/webkitpy/webdriver_tests/pytest_runner.py:
+ (TestExpectationsMarker.__init__): Save the param to ignore.
+ (TestExpectationsMarker._item_name): Return the name of the test without the para to ignore.
+ (TestExpectationsMarker.pytest_collection_modifyitems): Use _item_name().
+ (run): Pass param to ignore to TestExpectationsMarker().
+ * Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py:
+ (WebDriverSeleniumExecutor.__init__): Save the driver name.
+ (WebDriverSeleniumExecutor.run): Pass the driver name as param to ignore.
+
+2018-01-17 Carlos Garcia Campos <[email protected]>
+
[GTK][WPE] Show unit tests unexpectedly passing in the bots
https://bugs.webkit.org/show_bug.cgi?id=181726
Modified: trunk/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py (227126 => 227127)
--- trunk/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py 2018-01-18 06:50:12 UTC (rev 227126)
+++ trunk/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py 2018-01-18 06:51:30 UTC (rev 227127)
@@ -130,14 +130,29 @@
class TestExpectationsMarker(object):
- def __init__(self, expectations):
+ def __init__(self, expectations, ignore_param):
self._expectations = expectations
+ self._ignore_param = ignore_param
self._base_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests')
+ def _item_name(self, item):
+ if self._ignore_param is None:
+ return item.name
+
+ single_param = '[%s]' % self._ignore_param
+ if item.name.endswith(single_param):
+ return item.name[:-len(single_param)]
+
+ param = '[%s-' % self._ignore_param
+ if param in item.name:
+ return item.name.replace('%s-' % self._ignore_param, '')
+
+ return item.name
+
def pytest_collection_modifyitems(self, session, config, items):
for item in items:
test = os.path.relpath(str(item.fspath), self._base_dir)
- expected = self._expectations.get_expectation(test, item.name)[0]
+ expected = self._expectations.get_expectation(test, self._item_name(item))[0]
if expected == 'FAIL':
item.add_marker(pytest.mark.xfail)
elif expected == 'TIMEOUT':
@@ -161,10 +176,10 @@
return collect_recorder.tests
-def run(path, args, timeout, env, expectations):
+def run(path, args, timeout, env, expectations, ignore_param=None):
harness_recorder = HarnessResultRecorder()
subtests_recorder = SubtestResultRecorder()
- expectations_marker = TestExpectationsMarker(expectations)
+ expectations_marker = TestExpectationsMarker(expectations, ignore_param)
_environ = dict(os.environ)
os.environ.clear()
os.environ.update(env)
Modified: trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py (227126 => 227127)
--- trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py 2018-01-18 06:50:12 UTC (rev 227126)
+++ trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py 2018-01-18 06:51:30 UTC (rev 227127)
@@ -41,10 +41,11 @@
class WebDriverSeleniumExecutor(object):
def __init__(self, driver, display_driver):
+ self._driver_name = driver.selenium_name()
self._env = display_driver._setup_environ_for_test()
self._env.update(driver.browser_env())
- self._args = ['--driver=%s' % driver.selenium_name(), '--driver-binary=%s' % driver.binary_path().encode()]
+ self._args = ['--driver=%s' % self._driver_name, '--driver-binary=%s' % driver.binary_path().encode()]
browser_path = driver.browser_path().encode()
if browser_path:
self._args.extend(['--browser-binary=%s' % browser_path])
@@ -59,4 +60,4 @@
return pytest_runner.collect(directory, self._args)
def run(self, test, timeout, expectations):
- return pytest_runner.run(test, self._args, timeout, self._env, expectations)
+ return pytest_runner.run(test, self._args, timeout, self._env, expectations, self._driver_name)
Modified: trunk/WebDriverTests/ChangeLog (227126 => 227127)
--- trunk/WebDriverTests/ChangeLog 2018-01-18 06:50:12 UTC (rev 227126)
+++ trunk/WebDriverTests/ChangeLog 2018-01-18 06:51:30 UTC (rev 227127)
@@ -1,5 +1,16 @@
2018-01-17 Carlos Garcia Campos <[email protected]>
+ WebDriver: ignore the driver in selenium test names when getting expectations
+ https://bugs.webkit.org/show_bug.cgi?id=181738
+
+ Reviewed by Carlos Alberto Lopez Perez.
+
+ Add some expectations for selenium tests.
+
+ * TestExpectations.json:
+
+2018-01-17 Carlos Garcia Campos <[email protected]>
+
WebDriver: add support for test expectations
https://bugs.webkit.org/show_bug.cgi?id=180420
Modified: trunk/WebDriverTests/TestExpectations.json (227126 => 227127)
--- trunk/WebDriverTests/TestExpectations.json 2018-01-18 06:50:12 UTC (rev 227126)
+++ trunk/WebDriverTests/TestExpectations.json 2018-01-18 06:51:30 UTC (rev 227127)
@@ -1,4 +1,111 @@
{
+ "imported/selenium/py/test/selenium/webdriver/common/alerts_tests.py": {
+ "subtests": {
+ "testShouldAllowUsersToAcceptAnAlertInAFrame": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181727"}}
+ },
+ "testShouldAllowUsersToAcceptAnAlertInANestedFrame": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181727"}}
+ }
+ }
+ },
+ "imported/selenium/py/test/selenium/webdriver/common/api_example_tests.py": {
+ "subtests": {
+ "testChangeWindowSize": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181728"}}
+ },
+ "testGetLogTypes": {
+ "expected": {"all": {"status": ["SKIP"]}}
+ },
+ "testGetLog": {
+ "expected": {"all": {"status": ["SKIP"]}}
+ }
+ }
+ },
+ "imported/selenium/py/test/selenium/webdriver/common/appcache_tests.py": {
+ "expected": {"all": {"status": ["SKIP"]}}
+ },
+ "imported/selenium/py/test/selenium/webdriver/common/click_scrolling_tests.py": {
+ "subtests": {
+ "testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/174902"}}
+ },
+ "testShouldNotScrollIfAlreadyScrolledAndElementIsInView": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181729"}}
+ },
+ "testShouldScrollOverflowElementsIfClickPointIsOutOfViewButElementIsInView": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181730"}}
+ },
+ "testShouldBeAbleToClickElementThatIsOutOfViewInAFrameThatIsOutOfView": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+ },
+ "testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+ },
+ "testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+ },
+ "testShouldNotScrollWhenGettingElementSize": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+ },
+ "testShouldBeAbleToClickElementInATallFrame": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181731"}}
+ }
+ }
+ },
+ "imported/selenium/py/test/selenium/webdriver/common/correct_event_firing_tests.py": {
+ "subtests": {
+ "testShouldEmitOnChangeEventsWhenChangingTheStateOfACheckbox": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181732"}}
+ },
+ "testClearingAnElementShouldCauseTheOnChangeHandlerToFire": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181732"}}
+ }
+ }
+ },
+ "imported/selenium/py/test/selenium/webdriver/common/form_handling_tests.py": {
+ "subtests": {
+ "testShouldThrowAnExceptionWhenSelectingAnUnselectableElement": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181733"}}
+ }
+ }
+ },
+ "imported/selenium/py/test/selenium/webdriver/common/interactions_tests.py": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+ },
+ "imported/selenium/py/test/selenium/webdriver/common/position_and_size_tests.py": {
+ "subtests": {
+ "testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181734"}}
+ },
+ "testShouldGetCoordinatesOfAnElementInAFrame": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181735"}}
+ },
+ "testShouldGetCoordinatesOfAnElementInANestedFrame": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181735"}}
+ }
+ }
+ },
+ "imported/selenium/py/test/selenium/webdriver/common/rendered_webelement_tests.py": {
+ "subtests": {
+ "testShouldPickUpStyleOfAnElement": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181736"}}
+ },
+ "testShouldAllowInheritedStylesToBeUsed": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181736"}}
+ }
+ }
+ },
+ "imported/selenium/py/test/selenium/webdriver/common/text_handling_tests.py": {
+ "subtests": {
+ "testReadALargeAmountOfData": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/181737"}}
+ }
+ }
+ },
+ "imported/selenium/py/test/selenium/webdriver/common/w3c_interaction_tests.py": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
+ },
"imported/w3c/webdriver/tests/actions/key.py": {
"expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
},
