- Revision
- 108414
- Author
- [email protected]
- Date
- 2012-02-21 16:32:33 -0800 (Tue, 21 Feb 2012)
Log Message
webkitpy: speed up hot filesystem_mock functions
https://bugs.webkit.org/show_bug.cgi?id=79159
Reviewed by Adam Barth.
Profiling run_webkit_tests_integrationtest.py revealed that
isdir(), normpath(), and join() are called a lot and were very
slow.This patch speeds them up substantially, shaving 20 seconds
off of the execution time.
* Scripts/webkitpy/common/system/filesystem_mock.py:
(MockFileSystem.isdir):
(MockFileSystem._slow_but_correct_join):
(MockFileSystem.join):
(MockFileSystem.listdir):
(MockFileSystem._slow_but_correct_normpath):
(MockFileSystem.normpath):
(MockFileSystem.write_binary_file):
* Scripts/webkitpy/common/system/filesystem_mock_unittest.py:
(MockFileSystemTest.quick_check):
(MockFileSystemTest):
(MockFileSystemTest.test_join):
(MockFileSystemTest.test_normpath):
* Scripts/webkitpy/tool/servers/rebaselineserver_unittest.py:
(get_test_config): Call write_binary_file() instead of updating
filesystem.files directly, so that we create directories as
needed.
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (108413 => 108414)
--- trunk/Tools/ChangeLog 2012-02-22 00:29:33 UTC (rev 108413)
+++ trunk/Tools/ChangeLog 2012-02-22 00:32:33 UTC (rev 108414)
@@ -1,5 +1,35 @@
2012-02-21 Dirk Pranke <[email protected]>
+ webkitpy: speed up hot filesystem_mock functions
+ https://bugs.webkit.org/show_bug.cgi?id=79159
+
+ Reviewed by Adam Barth.
+
+ Profiling run_webkit_tests_integrationtest.py revealed that
+ isdir(), normpath(), and join() are called a lot and were very
+ slow.This patch speeds them up substantially, shaving 20 seconds
+ off of the execution time.
+
+ * Scripts/webkitpy/common/system/filesystem_mock.py:
+ (MockFileSystem.isdir):
+ (MockFileSystem._slow_but_correct_join):
+ (MockFileSystem.join):
+ (MockFileSystem.listdir):
+ (MockFileSystem._slow_but_correct_normpath):
+ (MockFileSystem.normpath):
+ (MockFileSystem.write_binary_file):
+ * Scripts/webkitpy/common/system/filesystem_mock_unittest.py:
+ (MockFileSystemTest.quick_check):
+ (MockFileSystemTest):
+ (MockFileSystemTest.test_join):
+ (MockFileSystemTest.test_normpath):
+ * Scripts/webkitpy/tool/servers/rebaselineserver_unittest.py:
+ (get_test_config): Call write_binary_file() instead of updating
+ filesystem.files directly, so that we create directories as
+ needed.
+
+2012-02-21 Dirk Pranke <[email protected]>
+
nrwt: make the delay between starting workers configurable per-port
https://bugs.webkit.org/show_bug.cgi?id=79148
Modified: trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock.py (108413 => 108414)
--- trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock.py 2012-02-22 00:29:33 UTC (rev 108413)
+++ trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock.py 2012-02-22 00:32:33 UTC (rev 108414)
@@ -173,40 +173,45 @@
return path in self.files and self.files[path] is not None
def isdir(self, path):
- if path in self.files:
- return False
- path = self.normpath(path)
- if path in self.dirs:
- return True
+ return self.normpath(path) in self.dirs
- # We need to use a copy of the keys here in order to avoid switching
- # to a different thread and potentially modifying the dict in
- # mid-iteration.
- files = self.files.keys()[:]
- result = any(f.startswith(path) and len(self.split(f)[0]) >= len(path) for f in files)
- if result:
- self.dirs.add(path)
- return result
+ def _slow_but_correct_join(self, *comps):
+ return re.sub(re.escape(os.path.sep), self.sep, os.path.join(*comps))
def join(self, *comps):
- # FIXME: might want tests for this and/or a better comment about how
- # it works.
- return re.sub(re.escape(os.path.sep), self.sep, os.path.join(*comps))
+ # This function is called a lot, so we optimize it; there are
+ # unittests to check that we match _slow_but_correct_join(), above.
+ path = ''
+ sep = self.sep
+ for comp in comps:
+ if not comp:
+ continue
+ if comp[0] == sep:
+ path = comp
+ continue
+ if path:
+ path += sep
+ path += comp
+ if comps[-1] == '' and path:
+ path += '/'
+ path = path.replace(sep + sep, sep)
+ return path
def listdir(self, path):
+ sep = self.sep
if not self.isdir(path):
raise OSError("%s is not a directory" % path)
- if not path.endswith(self.sep):
- path += self.sep
+ if not path.endswith(sep):
+ path += sep
dirs = []
files = []
for f in self.files:
if self.exists(f) and f.startswith(path):
remaining = f[len(path):]
- if self.sep in remaining:
- dir = remaining[:remaining.index(self.sep)]
+ if sep in remaining:
+ dir = remaining[:remaining.index(sep)]
if not dir in dirs:
dirs.append(dir)
else:
@@ -262,11 +267,27 @@
self.files[source] = None
self.written_files[source] = None
- def normpath(self, path):
- # Like join(), relies on os.path functionality but normalizes the
- # path separator to the mock one.
+ def _slow_but_correct_normpath(self, path):
return re.sub(re.escape(os.path.sep), self.sep, os.path.normpath(path))
+ def normpath(self, path):
+ # This function is called a lot, so we try to optimize the common cases
+ # instead of always calling _slow_but_correct_normpath(), above.
+ if '..' in path:
+ # This doesn't happen very often; don't bother trying to optimize it.
+ return self._slow_but_correct_normpath(path)
+ if not path:
+ return '.'
+ if path == '/':
+ return path
+ if path == '/.':
+ return '/'
+ if path.endswith('/.'):
+ return path[:-2]
+ if path.endswith('/'):
+ return path[:-1]
+ return path
+
def open_binary_tempfile(self, suffix=''):
path = self._mktemp(suffix)
return (WritableBinaryFileObject(self, path), path)
@@ -284,6 +305,7 @@
def write_binary_file(self, path, contents):
# FIXME: should this assert if dirname(path) doesn't exist?
+ self.maybe_make_directory(self.dirname(path))
self.files[path] = contents
self.written_files[path] = contents
Modified: trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock_unittest.py (108413 => 108414)
--- trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock_unittest.py 2012-02-22 00:29:33 UTC (rev 108413)
+++ trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock_unittest.py 2012-02-22 00:32:33 UTC (rev 108414)
@@ -26,8 +26,11 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import os
+import re
import unittest
+
from webkitpy.common.system import filesystem_mock
from webkitpy.common.system import filesystem_unittest
@@ -41,6 +44,45 @@
self.teardown_generic_test_dir()
self.fs = None
+ def quick_check(self, test_fn, good_fn, *tests):
+ for test in tests:
+ if hasattr(test, '__iter__'):
+ expected = good_fn(*test)
+ actual = test_fn(*test)
+ else:
+ expected = good_fn(test)
+ actual = test_fn(test)
+ self.assertEquals(expected, actual, 'given %s, expected %s, got %s' % (repr(test), repr(expected), repr(actual)))
+ def test_join(self):
+ self.quick_check(self.fs.join,
+ self.fs._slow_but_correct_join,
+ ('',),
+ ('', 'bar'),
+ ('foo',),
+ ('foo/',),
+ ('foo', ''),
+ ('foo/', ''),
+ ('foo', 'bar'),
+ ('foo', '/bar'),
+ )
+
+ def test_normpath(self):
+ self.quick_check(self.fs.normpath,
+ self.fs._slow_but_correct_normpath,
+ '',
+ '/',
+ '.',
+ '/.',
+ 'foo',
+ 'foo/',
+ 'foo/.',
+ 'foo/bar',
+ '/foo',
+ 'foo/../bar',
+ 'foo/../bar/baz',
+ '../foo')
+
+
if __name__ == '__main__':
unittest.main()
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/rebaselineserver_unittest.py (108413 => 108414)
--- trunk/Tools/Scripts/webkitpy/tool/servers/rebaselineserver_unittest.py 2012-02-22 00:29:33 UTC (rev 108413)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/rebaselineserver_unittest.py 2012-02-22 00:32:33 UTC (rev 108414)
@@ -295,11 +295,9 @@
results_directory = '/WebKitBuild/Debug/layout-test-results'
host = MockHost()
for file in test_files:
- file_path = host.filesystem.join(layout_tests_directory, file)
- host.filesystem.files[file_path] = ''
+ host.filesystem.write_binary_file(host.filesystem.join(layout_tests_directory, file), '')
for file in result_files:
- file_path = host.filesystem.join(results_directory, file)
- host.filesystem.files[file_path] = ''
+ host.filesystem.write_binary_file(host.filesystem.join(results_directory, file), '')
class TestMacPort(WebKitPort):
port_name = "mac"
