Diff
Modified: trunk/Tools/ChangeLog (252442 => 252443)
--- trunk/Tools/ChangeLog 2019-11-14 01:59:52 UTC (rev 252442)
+++ trunk/Tools/ChangeLog 2019-11-14 02:08:56 UTC (rev 252443)
@@ -1,3 +1,40 @@
+2019-11-13 Jonathan Bedard <[email protected]>
+
+ Python 3: Add support in webkitpy.layout_tests.controllers
+ https://bugs.webkit.org/show_bug.cgi?id=204180
+
+ Reviewed by Stephanie Lewis.
+
+ * Scripts/test-webkitpy-python3: Add webkitpy.layout_tests.controllers.
+ * Scripts/webkitpy/common/message_pool.py:
+ (_MessagePool.__init__): Use Python 3 queue syntax.
+ (_MessagePool._can_pickle): Use Python 3 pickle syntax.
+ (_MessagePool._loop): Use Python 3 queue syntax.
+ (_Worker.run): Use Python 3 queue syntax.
+ (_Worker._raise): Python 2 and Python 3 have different semantics for raising an exception
+ With a stack trace. However, 'raise' will raise the exception we are in the process of capturing,
+ Which is exactly what we want in this case.
+ * Scripts/webkitpy/common/read_checksum_from_png.py:
+ (read_checksum): Standardize checksum as a string.
+ * Scripts/webkitpy/common/system/filesystem.py:
+ (FileSystem.write_binary_file): Binary files should be written with bytes, not strings.
+ * Scripts/webkitpy/common/system/filesystem_mock.py:
+ (MockFileSystem.write_binary_file): Binary files should be written with bytes, not strings.
+ * Scripts/webkitpy/layout_tests/controllers/layout_test_finder_unittest.py: assertItemsEqual is not
+ Defined in Python 3.
+ * Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py:
+ (Worker._do_post_tests_work): Use compatible iteritems.
+ (Sharder._shard_by_directory): Ditto.
+ * Scripts/webkitpy/layout_tests/controllers/manager.py:
+ (Manager.run): Use compatible itervalues.
+ (Manager._look_for_new_crash_logs): Use Python 3 item iteration.
+ (Manager._results_to_upload_json_trie): Use compatible itervalues.
+ (Manager._stats_trie): Use compatible iteritems.
+ * Scripts/webkitpy/port/base.py:
+ (Port.expected_text): Be explicit about decoding text expectations.
+ * Scripts/webkitpy/port/mock_drt.py:
+ (MockDRT.write_test_output):
+
2019-11-13 Per Arne Vollan <[email protected]>
REGRESSION: WKWebView navigation fails when navigating from about:blank
Modified: trunk/Tools/Scripts/test-webkitpy-python3 (252442 => 252443)
--- trunk/Tools/Scripts/test-webkitpy-python3 2019-11-14 01:59:52 UTC (rev 252442)
+++ trunk/Tools/Scripts/test-webkitpy-python3 2019-11-14 02:08:56 UTC (rev 252443)
@@ -35,6 +35,7 @@
PYTHON3_COMPATIBLE_DIRECTORIES = [
'webkitpy.common',
+ 'webkitpy.layout_tests.controllers',
'webkitpy.layout_tests.models',
'webkitpy.port',
'webkitpy.results',
Modified: trunk/Tools/Scripts/webkitpy/common/message_pool.py (252442 => 252443)
--- trunk/Tools/Scripts/webkitpy/common/message_pool.py 2019-11-14 01:59:52 UTC (rev 252442)
+++ trunk/Tools/Scripts/webkitpy/common/message_pool.py 2019-11-14 02:08:56 UTC (rev 252443)
@@ -40,16 +40,20 @@
"""
-import cPickle
import logging
import multiprocessing
import os
-import Queue
import signal
import sys
import time
import traceback
+if sys.version_info > (3, 0):
+ import pickle
+ import queue
+else:
+ import cPickle as pickle
+ import Queue as queue
from webkitpy.common.host import Host
from webkitpy.common.system import stack_utils
@@ -76,8 +80,8 @@
self._running_inline = (self._num_workers == 1)
self._timeout = timeout
if self._running_inline:
- self._messages_to_worker = Queue.Queue()
- self._messages_to_manager = Queue.Queue()
+ self._messages_to_worker = queue.Queue()
+ self._messages_to_manager = queue.Queue()
else:
self._messages_to_worker = multiprocessing.Queue()
self._messages_to_manager = multiprocessing.Queue()
@@ -175,7 +179,7 @@
def _can_pickle(self, host):
try:
- cPickle.dumps(host)
+ pickle.dumps(host)
return True
except TypeError:
return False
@@ -193,7 +197,7 @@
method = getattr(self, '_handle_' + message.name)
assert method, 'bad message %s' % repr(message)
method(message.src, *message.args)
- except Queue.Empty:
+ except queue.Empty:
pass
@@ -272,7 +276,7 @@
break
_log.debug("%s exiting" % self.name)
- except Queue.Empty:
+ except queue.Empty:
assert False, '%s: ran out of messages in worker queue.' % self.name
except KeyboardInterrupt as e:
self._raise(sys.exc_info())
@@ -302,7 +306,7 @@
def _raise(self, exc_info):
exception_type, exception_value, exception_traceback = exc_info
if self._running_inline:
- raise exception_type, exception_value, exception_traceback
+ raise
if exception_type == KeyboardInterrupt:
_log.debug("%s: interrupted, exiting" % self.name)
Modified: trunk/Tools/Scripts/webkitpy/common/read_checksum_from_png.py (252442 => 252443)
--- trunk/Tools/Scripts/webkitpy/common/read_checksum_from_png.py 2019-11-14 01:59:52 UTC (rev 252442)
+++ trunk/Tools/Scripts/webkitpy/common/read_checksum_from_png.py 2019-11-14 02:08:56 UTC (rev 252443)
@@ -27,13 +27,15 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from webkitpy.common.unicode_compatibility import encode_if_necessary, decode_for
+
def read_checksum(filehandle):
# We expect the comment to be at the beginning of the file.
- data = ""
- comment_key = 'tEXtchecksum\x00'
+ data = ""
+ comment_key = b'tEXtchecksum\x00'
comment_pos = data.find(comment_key)
if comment_pos == -1:
return
checksum_pos = comment_pos + len(comment_key)
- return data[checksum_pos:checksum_pos + 32]
+ return decode_for(data[checksum_pos:checksum_pos + 32], str)
Modified: trunk/Tools/Scripts/webkitpy/common/system/filesystem.py (252442 => 252443)
--- trunk/Tools/Scripts/webkitpy/common/system/filesystem.py 2019-11-14 01:59:52 UTC (rev 252442)
+++ trunk/Tools/Scripts/webkitpy/common/system/filesystem.py 2019-11-14 02:08:56 UTC (rev 252443)
@@ -39,7 +39,7 @@
import tempfile
import time
-from webkitpy.common.unicode_compatibility import decode_if_necessary
+from webkitpy.common.unicode_compatibility import decode_if_necessary, encode_for
class FileSystem(object):
@@ -227,7 +227,7 @@
def write_binary_file(self, path, contents):
with open(path, 'wb') as f:
- f.write(contents)
+ f.write(encode_for(contents, bytes))
def open_text_file_for_reading(self, path, errors='strict'):
# Note: There appears to be an issue with the returned file objects
Modified: trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock.py (252442 => 252443)
--- trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock.py 2019-11-14 01:59:52 UTC (rev 252442)
+++ trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock.py 2019-11-14 02:08:56 UTC (rev 252443)
@@ -329,8 +329,8 @@
def write_binary_file(self, path, contents):
# FIXME: should this assert if dirname(path) doesn't exist?
self.maybe_make_directory(self.dirname(path))
- self.files[path] = contents
- self.written_files[path] = contents
+ self.files[path] = unicode_compatibility.encode_for(contents, bytes)
+ self.written_files[path] = unicode_compatibility.encode_for(contents, bytes)
def open_text_file_for_reading(self, path, errors='strict'):
if self.files[path] is None:
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder_unittest.py (252442 => 252443)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder_unittest.py 2019-11-14 01:59:52 UTC (rev 252442)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder_unittest.py 2019-11-14 02:08:56 UTC (rev 252443)
@@ -55,7 +55,7 @@
def test_touched_test(self):
paths = ['LayoutTests/test.html', 'LayoutTests/test', 'test2.html', 'Source/test1.html']
fs, touched_tests = self.touched_files(paths)
- self.assertItemsEqual(touched_tests, ['test.html'])
+ self.assertEqual(touched_tests, ['test.html'])
def test_expected_touched_test(self):
paths = ['LayoutTests/test-expected.txt', 'LayoutTests/no-test-expected.txt']
@@ -62,7 +62,7 @@
fs = MockFileSystem()
fs.write_text_file('/test.checkout/LayoutTests/test.html', 'This is a test')
fs, touched_tests = self.touched_files(paths, fs)
- self.assertItemsEqual(touched_tests, ['test.html'])
+ self.assertEqual(touched_tests, ['test.html'])
def test_platform_expected_touched_test(self):
paths = ['LayoutTests/platform/mock/test-expected.txt', 'LayoutTests/platform/mock/no-test-expected.txt']
@@ -69,7 +69,7 @@
fs = MockFileSystem()
fs.write_text_file('/test.checkout/LayoutTests/test.html', 'This is a test')
fs, touched_tests = self.touched_files(paths, fs)
- self.assertItemsEqual(touched_tests, ['test.html'])
+ self.assertEqual(touched_tests, ['test.html'])
def test_platform_duplicate_touched_test(self):
paths = ['LayoutTests/test1.html', 'LayoutTests/test1.html', 'LayoutTests/platform/mock1/test2-expected.txt', 'LayoutTests/platform/mock2/test2-expected.txt']
@@ -76,7 +76,7 @@
fs = MockFileSystem()
fs.write_text_file('/test.checkout/LayoutTests/test2.html', 'This is a test')
fs, touched_tests = self.touched_files(paths, fs)
- self.assertItemsEqual(touched_tests, ['test1.html', 'test2.html'])
+ self.assertEqual(sorted(touched_tests), sorted(['test1.html', 'test2.html']))
def test_touched_but_skipped_test(self):
host = MockHost()
@@ -92,4 +92,4 @@
host.filesystem.write_text_file('/test.checkout/LayoutTests/test3.html', 'This is a test to be skipped')
touched_tests = LayoutTestFinder(port, optparse.Values({'skipped': 'always', 'skip_failing_tests': False, 'http': True})).find_touched_tests(paths)
- self.assertItemsEqual(touched_tests, ['test0.html', 'test2.html'])
+ self.assertEqual(sorted(touched_tests), sorted(['test0.html', 'test2.html']))
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py (252442 => 252443)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py 2019-11-14 01:59:52 UTC (rev 252442)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py 2019-11-14 02:08:56 UTC (rev 252443)
@@ -33,6 +33,7 @@
import time
from webkitpy.common import message_pool
+from webkitpy.common.iteration_compatibility import iteritems
from webkitpy.layout_tests.controllers import single_test_runner
from webkitpy.layout_tests.models.test_run_results import TestRunResults
from webkitpy.layout_tests.models import test_expectations
@@ -331,7 +332,7 @@
post_test_output = driver.do_post_tests_work()
if post_test_output:
- for test_name, doc_list in post_test_output.world_leaks_dict.iteritems():
+ for test_name, doc_list in iteritems(post_test_output.world_leaks_dict):
additional_results.append(test_results.TestResult(test_name, [test_failures.FailureDocumentLeak(doc_list)]))
return additional_results
@@ -536,7 +537,7 @@
tests_by_dir.setdefault(directory, [])
tests_by_dir[directory].append(test_input)
- for directory, test_inputs in tests_by_dir.iteritems():
+ for directory, test_inputs in iteritems(tests_by_dir):
shard = TestShard(directory, test_inputs)
shards.append(shard)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (252442 => 252443)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2019-11-14 01:59:52 UTC (rev 252442)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2019-11-14 02:08:56 UTC (rev 252443)
@@ -44,6 +44,7 @@
from webkitpy.common.checkout.scm.detection import SCMDetector
from webkitpy.common.net.file_uploader import FileUploader
+from webkitpy.common.iteration_compatibility import iteritems, itervalues
from webkitpy.layout_tests.controllers.layout_test_finder import LayoutTestFinder
from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner
from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
@@ -218,13 +219,13 @@
start_time = time.time()
# Check to make sure we're not skipping every test.
- if not sum([len(tests) for tests in tests_to_run_by_device.itervalues()]):
+ if not sum([len(tests) for tests in itervalues(tests_to_run_by_device)]):
_log.critical('No tests to run.')
return test_run_results.RunDetails(exit_code=-1)
- needs_http = any((self._is_http_test(test) and not self._needs_web_platform_test(test)) for tests in tests_to_run_by_device.itervalues() for test in tests)
- needs_web_platform_test_server = any(self._needs_web_platform_test(test) for tests in tests_to_run_by_device.itervalues() for test in tests)
- needs_websockets = any(self._is_websocket_test(test) for tests in tests_to_run_by_device.itervalues() for test in tests)
+ needs_http = any((self._is_http_test(test) and not self._needs_web_platform_test(test)) for tests in itervalues(tests_to_run_by_device) for test in tests)
+ needs_web_platform_test_server = any(self._needs_web_platform_test(test) for tests in itervalues(tests_to_run_by_device) for test in tests)
+ needs_websockets = any(self._is_websocket_test(test) for tests in itervalues(tests_to_run_by_device) for test in tests)
self._runner = LayoutTestRunner(self._options, self._port, self._printer, self._results_directory, self._test_is_slow,
needs_http=needs_http, needs_web_platform_test_server=needs_web_platform_test_server, needs_websockets=needs_websockets)
@@ -427,7 +428,7 @@
logs after that time.
"""
crashed_processes = []
- for test, result in run_results.unexpected_results_by_name.iteritems():
+ for test, result in run_results.unexpected_results_by_name.items():
if (result.type != test_expectations.CRASH):
continue
for failure in result.failures:
@@ -437,13 +438,13 @@
sample_files = self._port.look_for_new_samples(crashed_processes, start_time)
if sample_files:
- for test, sample_file in sample_files.iteritems():
+ for test, sample_file in sample_files.items():
writer = TestResultWriter(self._port._filesystem, self._port, self._port.results_directory(), test)
writer.copy_sample_file(sample_file)
crash_logs = self._port.look_for_new_crash_logs(crashed_processes, start_time)
if crash_logs:
- for test, crash_log in crash_logs.iteritems():
+ for test, crash_log in crash_logs.items():
writer = TestResultWriter(self._port._filesystem, self._port, self._port.results_directory(), test)
writer.write_crash_log(crash_log)
@@ -491,7 +492,7 @@
}
results_trie = {}
- for result in results.results_by_name.itervalues():
+ for result in itervalues(results.results_by_name):
if result.type == test_expectations.SKIP:
continue
@@ -645,7 +646,7 @@
if result.type != test_expectations.SKIP:
stats[result.test_name] = {'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(result.test_run_time * 1000), int(result.total_run_time * 1000))}
stats_trie = {}
- for name, value in stats.iteritems():
+ for name, value in iteritems(stats):
json_results_generator.add_path_to_trie(name, value, stats_trie)
return stats_trie
Modified: trunk/Tools/Scripts/webkitpy/port/base.py (252442 => 252443)
--- trunk/Tools/Scripts/webkitpy/port/base.py 2019-11-14 01:59:52 UTC (rev 252442)
+++ trunk/Tools/Scripts/webkitpy/port/base.py 2019-11-14 02:08:56 UTC (rev 252443)
@@ -500,7 +500,7 @@
baseline_path = self.expected_filename(test_name, '.webarchive', device_type=device_type)
if not self._filesystem.exists(baseline_path):
return None
- text = self._filesystem.read_binary_file(baseline_path)
+ text = decode_for(self._filesystem.read_binary_file(baseline_path), str)
return text.replace("\r\n", "\n")
def _get_reftest_list(self, test_name):
Modified: trunk/Tools/Scripts/webkitpy/port/mock_drt.py (252442 => 252443)
--- trunk/Tools/Scripts/webkitpy/port/mock_drt.py 2019-11-14 01:59:52 UTC (rev 252442)
+++ trunk/Tools/Scripts/webkitpy/port/mock_drt.py 2019-11-14 02:08:56 UTC (rev 252443)
@@ -48,6 +48,7 @@
if script_dir not in sys.path:
sys.path.append(script_dir)
+from webkitpy.common.unicode_compatibility import decode_for
from webkitpy.common.system.systemhost import SystemHost
from webkitpy.port.driver import DriverInput, DriverOutput, DriverProxy
from webkitpy.port.factory import PortFactory
@@ -230,7 +231,7 @@
if output.image_hash != test_input.image_hash:
self._stdout.write('Content-Type: image/png\n')
self._stdout.write('Content-Length: %s\n' % len(output.image))
- self._stdout.write(output.image)
+ self._stdout.write(decode_for(output.image, str))
self._stdout.write('#EOF\n')
self._stdout.flush()
self._stderr.write('#EOF\n')
