Diff
Modified: trunk/Tools/ChangeLog (233679 => 233680)
--- trunk/Tools/ChangeLog 2018-07-10 08:21:26 UTC (rev 233679)
+++ trunk/Tools/ChangeLog 2018-07-10 08:23:53 UTC (rev 233680)
@@ -1,5 +1,23 @@
2018-07-10 Thibault Saunier <[email protected]>
+ [Flatpak] Fix running wpt tests inside flatpak
+ https://bugs.webkit.org/show_bug.cgi?id=187490
+
+ Reviewed by Philippe Normand.
+
+ Still not *all* passing but a big majority do.
+
+ * Scripts/webkitpy/w3c/wpt_runner.py:
+ (main): Generate the /etc/hosts and mount it in flatpak as required.
+ (WPTRunner.prepare_wpt_checkout): Renamed as we need to first checkout WPT code before generating /etc/hosts.
+ (WPTRunner._prepare_wpt_checkout): Deleted.
+ * flatpak/flatpakutils.py: Added a stdout= argument to run_in_sandbox() and
+ added a way to pass extra env to flatpak itself (so we can mount /etc/hosts).
+ Stopped passing `--device=all` as it is not required since r233638 (Bug 187400).
+ * flatpak/org.webkit.WebKit.yaml: Added virtualenv for python2 as required by wpt.
+
+2018-07-10 Thibault Saunier <[email protected]>
+
[Flatpak] Use logging to print debug information
https://bugs.webkit.org/show_bug.cgi?id=187496
Modified: trunk/Tools/Scripts/webkitpy/w3c/wpt_runner.py (233679 => 233680)
--- trunk/Tools/Scripts/webkitpy/w3c/wpt_runner.py 2018-07-10 08:21:26 UTC (rev 233679)
+++ trunk/Tools/Scripts/webkitpy/w3c/wpt_runner.py 2018-07-10 08:23:53 UTC (rev 233680)
@@ -50,7 +50,8 @@
_log.error(str(e))
sys.exit(-1)
- # If necessary, inject the jhbuild wrapper.
+ runner = WPTRunner(port, port.host, script_name, options)
+ # If necessary, inject the jhbuild wrapper or re-run in flatpak sandbox.
if port.name() in ['gtk', 'wpe']:
filesystem = host.filesystem
@@ -57,12 +58,23 @@
top_level_directory = filesystem.normpath(filesystem.join(filesystem.dirname(__file__), '..', '..', '..', '..'))
sys.path.insert(0, filesystem.join(top_level_directory, 'Tools', 'flatpak'))
import flatpakutils
- flatpakutils.run_in_sandbox_if_available(sys.argv)
+ if not flatpakutils.is_sandboxed() and flatpakutils.check_flatpak(verbose=False):
+ flatpak_runner = flatpakutils.WebkitFlatpak.load_from_args(sys.argv)
+ if flatpak_runner.clean_args() and flatpak_runner.has_environment():
+ if not runner.prepare_wpt_checkout():
+ sys.exit(1)
+ hostfilename = os.path.join(flatpak_runner.build_path, "wpt_etc_hosts")
+ with open(hostfilename, "w") as stdout:
+ flatpak_runner.run_in_sandbox(os.path.join(options.wpt_checkout, "wpt"), "make-hosts-file", stdout=stdout)
+
+ sys.exit(flatpak_runner.run_in_sandbox(*sys.argv,
+ extra_flatpak_args=['--bind-mount=/etc/hosts=%s' % hostfilename]))
+
sys.path.insert(0, filesystem.join(top_level_directory, 'Tools', 'jhbuild'))
import jhbuildutils
- if flatpakutils.is_sandboxed() and not jhbuildutils.enter_jhbuild_environment_if_available(port.name()):
+ if not flatpakutils.is_sandboxed() and not jhbuildutils.enter_jhbuild_environment_if_available(port.name()):
_log.warning('jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing.')
# Create the Port-specific driver.
@@ -76,7 +88,6 @@
options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
str(port.default_child_processes()))
- runner = WPTRunner(port, port.host, script_name, options)
if not runner.run(args):
sys.exit(1)
@@ -130,7 +141,7 @@
self._create_webdriver_func = create_webdriver_func
self._spawn_wpt_func = spawn_wpt_func
- def _prepare_wpt_checkout(self):
+ def prepare_wpt_checkout(self):
if not self._options.wpt_checkout:
test_downloader = self._downloader_class(WPTPaths.checkout_directory(self._finder),
self._host, self._downloader_class.default_options())
@@ -177,7 +188,7 @@
return True
def run(self, args):
- if not self._prepare_wpt_checkout():
+ if not self.prepare_wpt_checkout():
return False
# Parse the test expectations JSON and construct corresponding metadata files.
Modified: trunk/Tools/Scripts/webkitpy/w3c/wpt_runner_unittest.py (233679 => 233680)
--- trunk/Tools/Scripts/webkitpy/w3c/wpt_runner_unittest.py 2018-07-10 08:21:26 UTC (rev 233679)
+++ trunk/Tools/Scripts/webkitpy/w3c/wpt_runner_unittest.py 2018-07-10 08:23:53 UTC (rev 233680)
@@ -153,12 +153,12 @@
"/mock-checkout/WebPlatformTests/MockPort/TestManifest.ini", "{}")
def test_prepare_wpt_checkout(self):
- # Tests the _prepare_wpt_checkout() method with no WPT checkout specified in options.
+ # Tests the prepare_wpt_checkout() method with no WPT checkout specified in options.
options, _ = parse_args([])
instance = WPTRunnerTest.TestInstance(options)
- self.assertTrue(instance.runner._prepare_wpt_checkout())
+ self.assertTrue(instance.runner.prepare_wpt_checkout())
expected_wpt_checkout = "/mock-checkout/WebKitBuild/w3c-tests/web-platform-tests"
self.assertEquals(instance.runner._options.wpt_checkout, expected_wpt_checkout)
@@ -165,7 +165,7 @@
self.assertTrue(instance.host.filesystem.isdir(expected_wpt_checkout))
def test_prepare_wpt_checkout_specified_path(self):
- # Tests the _prepare_wpt_checkout() method with WPT checkout specified in options.
+ # Tests the prepare_wpt_checkout() method with WPT checkout specified in options.
specified_wpt_checkout = "/mock-path/web-platform-tests"
options, _ = parse_args(["--wpt-checkout", specified_wpt_checkout])
@@ -172,7 +172,7 @@
instance = WPTRunnerTest.TestInstance(options)
instance.host.filesystem.maybe_make_directory(specified_wpt_checkout)
- self.assertTrue(instance.runner._prepare_wpt_checkout())
+ self.assertTrue(instance.runner.prepare_wpt_checkout())
self.assertEquals(instance.runner._options.wpt_checkout, specified_wpt_checkout)
def test_generate_metadata_directory(self):
Modified: trunk/Tools/flatpak/flatpakutils.py (233679 => 233680)
--- trunk/Tools/flatpak/flatpakutils.py 2018-07-10 08:21:26 UTC (rev 233679)
+++ trunk/Tools/flatpak/flatpakutils.py 2018-07-10 08:23:53 UTC (rev 233680)
@@ -634,11 +634,11 @@
return True
def _cleanup_faltpak_args_for_tests_if_needed(self, args):
- if args and not args[0].endswith('run-webkit-tests'):
+ if not args or not args[0].endswith('run-webkit-tests'):
return self.finish_args
# We are going to run our own Xvfb server in the sandbox
- unwanted_args = ["--socket=x11", "--device=all"]
+ unwanted_args = ["--socket=x11"]
finish_args = [e for e in self.finish_args if e not in unwanted_args]
return finish_args
@@ -645,7 +645,8 @@
def run_in_sandbox(self, *args, **kwargs):
cwd = kwargs.pop("cwd", None)
- remove_devices = kwargs.pop("remove_devices", False)
+ stdout = kwargs.pop("stdout", sys.stdout)
+ extra_flatpak_args = kwargs.pop("extra_flatpak_args", [])
if not isinstance(args, list):
args = list(args)
@@ -684,7 +685,7 @@
flatpak_command.append("--env=%s=%s" % (envvar, value))
finish_args = self._cleanup_faltpak_args_for_tests_if_needed(args)
- flatpak_command += finish_args + [self.flatpak_build_path]
+ flatpak_command += finish_args + extra_flatpak_args + [self.flatpak_build_path]
shell_string = ""
if args:
@@ -704,7 +705,7 @@
flatpak_command.extend(['sh', "/run/host/" + tmpscript.name])
try:
- subprocess.check_call(flatpak_command)
+ subprocess.check_call(flatpak_command, stdout=stdout)
except subprocess.CalledProcessError as e:
sys.stderr.write(str(e) + "\n")
return e.returncode
Modified: trunk/Tools/flatpak/org.webkit.WebKit.yaml (233679 => 233680)
--- trunk/Tools/flatpak/org.webkit.WebKit.yaml 2018-07-10 08:21:26 UTC (rev 233679)
+++ trunk/Tools/flatpak/org.webkit.WebKit.yaml 2018-07-10 08:23:53 UTC (rev 233680)
@@ -89,8 +89,7 @@
- name: python3-pyaml
buildsystem: simple
build-commands:
- - pip3 install --no-index --find-links="file://${PWD}" --prefix=${FLATPAK_DEST}
- pyaml
+ - pip3 install --no-index --find-links="file://${PWD}" --prefix=${FLATPAK_DEST} pyaml
sources:
- type: file
url: https://files.pythonhosted.org/packages/4a/85/db5a2df477072b2902b0eb892feb37d88ac635d36245a72a6a69b23b383a/PyYAML-3.12.tar.gz
@@ -102,8 +101,7 @@
- name: python2-pyaml
buildsystem: simple
build-commands:
- - pip2 install --no-index --find-links="file://${PWD}" --target=/app/lib/python2.7/site-packages/
- pyaml
+ - pip2 install --no-index --find-links="file://${PWD}" --target=/app/lib/python2.7/site-packages/ pyaml
sources:
- type: file
url: https://files.pythonhosted.org/packages/4a/85/db5a2df477072b2902b0eb892feb37d88ac635d36245a72a6a69b23b383a/PyYAML-3.12.tar.gz
@@ -211,6 +209,17 @@
url: https://cpan.metacpan.org/authors/id/L/LE/LEEJO/CGI-4.38.tar.gz
sha256: 8c58f4a529bb92a914b22b7e64c5e31185c9854a4070a6dfad44fe5cc248e7d4
+ - name: python2-virtualenv
+ buildsystem: simple
+ build-commands:
+ - pip2 install --no-index --find-links="file://${PWD}" --target=/app/lib/python2.7/site-packages/ virtualenv
+ - ln -s /app/lib/python2.7/site-packages/virtualenv.py /app/bin/virtualenv
+ - chmod +x /app/bin/virtualenv
+ sources:
+ - type: file
+ url: https://files.pythonhosted.org/packages/33/bc/fa0b5347139cd9564f0d44ebd2b147ac97c36b2403943dbee8a25fd74012/virtualenv-16.0.0.tar.gz
+ sha256: ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752
+
# Port specific components.
- %(PORTNAME)s.yaml
