- Revision
- 220150
- Author
- [email protected]
- Date
- 2017-08-02 13:01:32 -0700 (Wed, 02 Aug 2017)
Log Message
webkitpy: Allow caller to specify response to unicode encode/decode error in filesystem
https://bugs.webkit.org/show_bug.cgi?id=175075
Reviewed by David Kilzer.
We have no way of handling text files with illegal unicode characters. Allow the callers of
filesystem.read_text_file to specify one of Python 2.7's supported responses ('strict', 'ignore', or
'replace'). See https://docs.python.org/2/howto/unicode.html for details on these responses.
* Scripts/webkitpy/common/system/filesystem.py:
(FileSystem.read_text_file): Allow caller to specify unicode error handling.
(FileSystem.write_text_file): Ditto.
* Scripts/webkitpy/common/system/filesystem_mock.py:
(MockFileSystem.read_text_file): Allow caller to specify unicode error handling.
(MockFileSystem.write_text_file): Ditto.
* Scripts/webkitpy/common/system/filesystem_unittest.py:
(RealFileSystemTest.test_read_text_file_unicode_decode_error): Test reading a file with illegal unicode content.
(RealFileSystemTest.test_write_text_file_unicode_encode_error): Test writing illegal unicode content to a file.
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (220149 => 220150)
--- trunk/Tools/ChangeLog 2017-08-02 19:59:09 UTC (rev 220149)
+++ trunk/Tools/ChangeLog 2017-08-02 20:01:32 UTC (rev 220150)
@@ -1,3 +1,24 @@
+2017-08-02 Jonathan Bedard <[email protected]>
+
+ webkitpy: Allow caller to specify response to unicode encode/decode error in filesystem
+ https://bugs.webkit.org/show_bug.cgi?id=175075
+
+ Reviewed by David Kilzer.
+
+ We have no way of handling text files with illegal unicode characters. Allow the callers of
+ filesystem.read_text_file to specify one of Python 2.7's supported responses ('strict', 'ignore', or
+ 'replace'). See https://docs.python.org/2/howto/unicode.html for details on these responses.
+
+ * Scripts/webkitpy/common/system/filesystem.py:
+ (FileSystem.read_text_file): Allow caller to specify unicode error handling.
+ (FileSystem.write_text_file): Ditto.
+ * Scripts/webkitpy/common/system/filesystem_mock.py:
+ (MockFileSystem.read_text_file): Allow caller to specify unicode error handling.
+ (MockFileSystem.write_text_file): Ditto.
+ * Scripts/webkitpy/common/system/filesystem_unittest.py:
+ (RealFileSystemTest.test_read_text_file_unicode_decode_error): Test reading a file with illegal unicode content.
+ (RealFileSystemTest.test_write_text_file_unicode_encode_error): Test writing illegal unicode content to a file.
+
2017-08-02 Aakash Jain <[email protected]>
Intermittent exception in buildPageURLForIteration for Buildbot 0.9 dashboard
Modified: trunk/Tools/Scripts/webkitpy/common/system/filesystem.py (220149 => 220150)
--- trunk/Tools/Scripts/webkitpy/common/system/filesystem.py 2017-08-02 19:59:09 UTC (rev 220149)
+++ trunk/Tools/Scripts/webkitpy/common/system/filesystem.py 2017-08-02 20:01:32 UTC (rev 220150)
@@ -246,19 +246,19 @@
codecs.getwriter('utf8'),
'replace')
- def read_text_file(self, path):
+ def read_text_file(self, path, errors='strict'):
"""Return the contents of the file at the given path as a Unicode string.
The file is read assuming it is a UTF-8 encoded file with no BOM."""
- with codecs.open(path, 'r', 'utf8') as f:
+ with codecs.open(path, 'r', 'utf8', errors=errors) as f:
return f.read()
- def write_text_file(self, path, contents):
+ def write_text_file(self, path, contents, errors='strict'):
"""Write the contents to the file at the given location.
The file is written encoded as UTF-8 with no BOM."""
- with codecs.open(path, 'w', 'utf-8') as f:
- f.write(contents.decode('utf-8') if type(contents) == str else contents)
+ with codecs.open(path, 'w', 'utf-8', errors=errors) as f:
+ f.write(contents.decode('utf-8', errors=errors) if type(contents) == str else contents)
def sha1(self, path):
contents = self.read_binary_file(path)
Modified: trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock.py (220149 => 220150)
--- trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock.py 2017-08-02 19:59:09 UTC (rev 220149)
+++ trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock.py 2017-08-02 20:01:32 UTC (rev 220150)
@@ -340,11 +340,11 @@
def open_text_file_for_writing(self, path, should_append=False):
return WritableTextFileObject(self, path)
- def read_text_file(self, path):
- return self.read_binary_file(path).decode('utf-8')
+ def read_text_file(self, path, errors='strict'):
+ return self.read_binary_file(path).decode('utf-8', errors=errors)
- def write_text_file(self, path, contents):
- return self.write_binary_file(path, contents.encode('utf-8'))
+ def write_text_file(self, path, contents, errors='strict'):
+ return self.write_binary_file(path, contents.encode('utf-8', errors=errors))
def sha1(self, path):
contents = self.read_binary_file(path)
Modified: trunk/Tools/Scripts/webkitpy/common/system/filesystem_unittest.py (220149 => 220150)
--- trunk/Tools/Scripts/webkitpy/common/system/filesystem_unittest.py 2017-08-02 19:59:09 UTC (rev 220149)
+++ trunk/Tools/Scripts/webkitpy/common/system/filesystem_unittest.py 2017-08-02 20:01:32 UTC (rev 220150)
@@ -202,6 +202,40 @@
if text_path and fs.isfile(text_path):
os.remove(text_path)
+ def test_read_text_file_unicode_decode_error(self):
+ fs = FileSystem()
+ text_path = None
+ try:
+ text_path = tempfile.mktemp(prefix='write_text_unittest_')
+ fs.write_binary_file(text_path, bytearray(b'\x73\x74\x72\x8b'))
+
+ self.assertRaises(UnicodeDecodeError, fs.read_text_file, text_path)
+ self.assertEqual(u'str\ufffd', fs.read_text_file(text_path, errors='replace'))
+ self.assertEqual('str', fs.read_text_file(text_path, errors='ignore'))
+ finally:
+ if text_path and fs.isfile(text_path):
+ os.remove(text_path)
+
+ def test_write_text_file_unicode_encode_error(self):
+ fs = FileSystem()
+ text_path = None
+ try:
+ text_path = tempfile.mktemp(prefix='write_text_unittest_')
+ bin_path = tempfile.mktemp(prefix='write_bin_unittest_')
+ fs.write_binary_file(bin_path, bytearray(b'\x73\x74\x72\x8b'))
+ data_to_write = fs.read_binary_file(bin_path)
+
+ self.assertRaises(UnicodeDecodeError, fs.write_text_file, text_path, data_to_write)
+ fs.write_text_file(text_path, data_to_write, 'replace')
+ self.assertEqual(u'str\ufffd', fs.read_text_file(text_path))
+ fs.write_text_file(text_path, data_to_write, 'ignore')
+ self.assertEqual('str', fs.read_text_file(text_path))
+ finally:
+ if text_path and fs.isfile(text_path):
+ os.remove(text_path)
+ if bin_path and fs.isfile(bin_path):
+ os.remove(bin_path)
+
def test_append_to_text_file(self):
fs = FileSystem()
text_path = None
