Diff
Modified: trunk/Tools/ChangeLog (120737 => 120738)
--- trunk/Tools/ChangeLog 2012-06-19 18:05:58 UTC (rev 120737)
+++ trunk/Tools/ChangeLog 2012-06-19 18:21:16 UTC (rev 120738)
@@ -1,5 +1,52 @@
2012-06-19 Dirk Pranke <[email protected]>
+ NRWT should not take memory used as disk cache into account when deciding how many processes to launch
+ https://bugs.webkit.org/show_bug.cgi?id=81379
+
+ Reviewed by Eric Seidel.
+
+ The 'free memory' calculation we were running on the mac seemed
+ to underestimate how many children we can run in parallel, and
+ it was complex. This patch replaces that calculation with a
+ simpler one that reserves 2GB for overhead and assumes 256MB per
+ DRT/WTR, so if we had 4GB of RAM we can run up to 8 DRTs.
+
+ Also, there was a bug where we were truncating the memory
+ installed on the machine to 4GB by casting to an int instead of
+ a long; this was probably the source of some of the earlier
+ problems when using total memory.
+
+ This patch also removes the no-longer-needed restrictions on the
+ number of workers on beefy Lion boxes for both Apple Mac and Chromium
+ Mac; we should now use all of the cores by default.
+
+ The memory calculations have only been implemented on the mac;
+ having the calculation in base.default_child_processes() was IMO
+ misleading, and so this patch also moves the computation into
+ the MacPort. I have not heard of the # of workers being an issue
+ on any other ports, so this should be fine.
+
+ * Scripts/webkitpy/common/system/platforminfo.py:
+ (PlatformInfo.total_bytes_memory):
+ * Scripts/webkitpy/common/system/platforminfo_mock.py:
+ (PlatformInfo.total_bytes_memory):
+ (PlatformInfo.total_bytes_memory):
+ (PlatformInfo._win_version_tuple_from_cmd):
+ * Scripts/webkitpy/common/system/platforminfo_unittest.py:
+ (TestPlatformInfo.test_total_bytes_memory):
+ * Scripts/webkitpy/layout_tests/port/base.py:
+ (Port.default_child_processes):
+ * Scripts/webkitpy/layout_tests/port/base_unittest.py:
+ (PortTest.test_default_child_processes):
+ * Scripts/webkitpy/layout_tests/port/chromium_mac.py:
+ (ChromiumMacPort.operating_system):
+ * Scripts/webkitpy/layout_tests/port/mac.py:
+ (MacPort.default_child_processes):
+ * Scripts/webkitpy/layout_tests/port/mac_unittest.py:
+ (TestMacPort.test_default_child_processes):
+
+2012-06-19 Dirk Pranke <[email protected]>
+
new-run-webkit-tests reports unexpected pass of pixel tests when pixel testing is disabled
https://bugs.webkit.org/show_bug.cgi?id=85446
Modified: trunk/Tools/Scripts/webkitpy/common/system/platforminfo.py (120737 => 120738)
--- trunk/Tools/Scripts/webkitpy/common/system/platforminfo.py 2012-06-19 18:05:58 UTC (rev 120737)
+++ trunk/Tools/Scripts/webkitpy/common/system/platforminfo.py 2012-06-19 18:21:16 UTC (rev 120738)
@@ -81,18 +81,9 @@
# Windows-2008ServerR2-6.1.7600
return self._platform_module.platform()
- def free_bytes_memory(self):
- if self.is_mac():
- vm_stat_output = self._executive.run_command(["vm_stat"])
- free_bytes = self._compute_bytes_from_vm_stat_output("Pages free", vm_stat_output)
- # Per https://bugs.webkit.org/show_bug.cgi?id=74650 include inactive memory since the OS is lazy about freeing memory.
- free_bytes += self._compute_bytes_from_vm_stat_output("Pages inactive", vm_stat_output)
- return free_bytes
- return None
-
def total_bytes_memory(self):
if self.is_mac():
- return int(self._executive.run_command(["sysctl", "-n", "hw.memsize"]))
+ return long(self._executive.run_command(["sysctl", "-n", "hw.memsize"]))
return None
def _determine_os_name(self, sys_platform):
@@ -141,14 +132,3 @@
match_object = re.search(r'(?P<major>\d)\.(?P<minor>\d)\.(?P<build>\d+)', ver_output)
assert match_object, 'cmd returned an unexpected version string: ' + ver_output
return tuple(map(int, match_object.groups()))
-
- def _compute_bytes_from_vm_stat_output(self, label_text, vm_stat_output):
- page_size_match = re.search(r"page size of (\d+) bytes", vm_stat_output)
- free_pages_match = re.search(r"%s:\s+(\d+)." % label_text, vm_stat_output)
-
- # Fail hard if vmstat's output isn't what we expect.
- assert(page_size_match and free_pages_match)
-
- free_page_count = int(free_pages_match.group(1))
- page_size = int(page_size_match.group(1))
- return free_page_count * page_size
Modified: trunk/Tools/Scripts/webkitpy/common/system/platforminfo_mock.py (120737 => 120738)
--- trunk/Tools/Scripts/webkitpy/common/system/platforminfo_mock.py 2012-06-19 18:05:58 UTC (rev 120737)
+++ trunk/Tools/Scripts/webkitpy/common/system/platforminfo_mock.py 2012-06-19 18:21:16 UTC (rev 120738)
@@ -51,7 +51,4 @@
return "MockPlatform 1.0"
def total_bytes_memory(self):
- return 2 * 1024 * 1024 * 1024 # 2GB is a reasonable amount of ram to mock.
-
- def free_bytes_memory(self):
- return 1 * 1024 * 1024 * 1024 # 1GB is a reasonable amount of ram to mock as free.
+ return 3 * 1024 * 1024 * 1024 # 3GB is a reasonable amount of ram to mock.
Modified: trunk/Tools/Scripts/webkitpy/common/system/platforminfo_unittest.py (120737 => 120738)
--- trunk/Tools/Scripts/webkitpy/common/system/platforminfo_unittest.py 2012-06-19 18:05:58 UTC (rev 120737)
+++ trunk/Tools/Scripts/webkitpy/common/system/platforminfo_unittest.py 2012-06-19 18:21:16 UTC (rev 120738)
@@ -82,10 +82,8 @@
if info.is_mac():
self.assertTrue(info.total_bytes_memory() > 0)
- self.assertTrue(info.free_bytes_memory() > 0)
else:
self.assertEquals(info.total_bytes_memory(), None)
- self.assertEquals(info.free_bytes_memory(), None)
def test_os_name_and_wrappers(self):
info = self.make_info(fake_sys('linux2'))
@@ -180,22 +178,6 @@
info = self.make_info(fake_sys('freebsd9'))
self.assertEquals(info.total_bytes_memory(), None)
- def test_free_bytes_memory(self):
- vmstat_output = ("Mach Virtual Memory Statistics: (page size of 4096 bytes)\n"
- "Pages free: 1.\n"
- "Pages inactive: 1.\n")
- info = self.make_info(fake_sys('darwin'), fake_platform('10.6.3'), fake_executive(vmstat_output))
- self.assertEquals(info.free_bytes_memory(), 8192)
- info = self.make_info(fake_sys('win32', tuple([6, 1, 7600])))
- self.assertEquals(info.free_bytes_memory(), None)
-
- info = self.make_info(fake_sys('linux2'))
- self.assertEquals(info.free_bytes_memory(), None)
-
- info = self.make_info(fake_sys('freebsd9'))
- self.assertEquals(info.free_bytes_memory(), None)
-
-
if __name__ == '__main__':
unittest.main()
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py (120737 => 120738)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py 2012-06-19 18:05:58 UTC (rev 120737)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py 2012-06-19 18:21:16 UTC (rev 120738)
@@ -154,17 +154,7 @@
def default_child_processes(self):
"""Return the number of DumpRenderTree instances to use for this port."""
- cpu_count = self._executive.cpu_count()
- # Make sure we have enough ram to support that many instances:
- free_memory = self.host.platform.free_bytes_memory()
- if free_memory:
- bytes_per_drt = 200 * 1024 * 1024 # Assume each DRT needs 200MB to run.
- supportable_instances = max(free_memory / bytes_per_drt, 1) # Always use one process, even if we don't have space for it.
- if supportable_instances < cpu_count:
- # FIXME: The Printer isn't initialized when this is called, so using _log would just show an unitialized logger error.
- print "This machine could support %s child processes, but only has enough memory for %s." % (cpu_count, supportable_instances)
- return min(supportable_instances, cpu_count)
- return cpu_count
+ return self._executive.cpu_count()
def worker_startup_delay_secs(self):
# FIXME: If we start workers up too quickly, DumpRenderTree appears
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py (120737 => 120738)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py 2012-06-19 18:05:58 UTC (rev 120737)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py 2012-06-19 18:21:16 UTC (rev 120738)
@@ -60,22 +60,8 @@
def test_default_child_processes(self):
port = self.make_port()
- # Even though the MockPlatformInfo shows 1GB free memory (enough for 5 DRT instances)
- # we're still limited by the 2 mock cores we have:
- self.assertEqual(port.default_child_processes(), 2)
- bytes_for_drt = 200 * 1024 * 1024
+ self.assertNotEquals(port.default_child_processes(), None)
- port.host.platform.free_bytes_memory = lambda: bytes_for_drt
- expected_stdout = "This machine could support 2 child processes, but only has enough memory for 1.\n"
- child_processes = OutputCapture().assert_outputs(self, port.default_child_processes, (), expected_stdout=expected_stdout)
- self.assertEqual(child_processes, 1)
-
- # Make sure that we always use one process, even if we don't have the memory for it.
- port.host.platform.free_bytes_memory = lambda: bytes_for_drt - 1
- expected_stdout = "This machine could support 2 child processes, but only has enough memory for 1.\n"
- child_processes = OutputCapture().assert_outputs(self, port.default_child_processes, (), expected_stdout=expected_stdout)
- self.assertEqual(child_processes, 1)
-
def test_format_wdiff_output_as_html(self):
output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END)
html = self.make_port()._format_wdiff_output_as_html(output)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py (120737 => 120738)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py 2012-06-19 18:05:58 UTC (rev 120737)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py 2012-06-19 18:21:16 UTC (rev 120738)
@@ -101,16 +101,6 @@
def operating_system(self):
return 'mac'
- def default_child_processes(self):
- # FIXME: As a temporary workaround while we figure out what's going
- # on with https://bugs.webkit.org/show_bug.cgi?id=83076, reduce by
- # half the # of workers we run by default on bigger machines.
- default_count = super(ChromiumMacPort, self).default_child_processes()
- if default_count >= 8:
- cpu_count = self._executive.cpu_count()
- return max(1, min(default_count, int(cpu_count / 2)))
- return default_count
-
#
# PROTECTED METHODS
#
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/mac.py (120737 => 120738)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/mac.py 2012-06-19 18:05:58 UTC (rev 120737)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/mac.py 2012-06-19 18:21:16 UTC (rev 120738)
@@ -31,6 +31,7 @@
import os
import re
import subprocess
+import sys
import time
from webkitpy.common.system.crashlogs import CrashLogs
@@ -117,19 +118,23 @@
return self._version == "lion"
def default_child_processes(self):
+ # FIXME: The Printer isn't initialized when this is called, so using _log would just show an unitialized logger error.
+
if self.is_snowleopard():
- _log.warn("Cannot run tests in parallel on Snow Leopard due to rdar://problem/10621525.")
+ print >> sys.stderr, "Cannot run tests in parallel on Snow Leopard due to rdar://problem/10621525."
return 1
- # FIXME: As a temporary workaround while we figure out what's going
- # on with https://bugs.webkit.org/show_bug.cgi?id=83076, reduce by
- # half the # of workers we run by default on bigger machines.
default_count = super(MacPort, self).default_child_processes()
- if default_count >= 8:
- cpu_count = self._executive.cpu_count()
- return max(1, min(default_count, int(cpu_count / 2)))
- return default_count
+ # Make sure we have enough ram to support that many instances:
+ total_memory = self.host.platform.total_bytes_memory()
+ bytes_per_drt = 256 * 1024 * 1024 # Assume each DRT needs 256MB to run.
+ overhead = 2048 * 1024 * 1024 # Assume we need 2GB free for the O/S
+ supportable_instances = max((total_memory - overhead) / bytes_per_drt, 1) # Always use one process, even if we don't have space for it.
+ if supportable_instances < default_count:
+ print >> sys.stderr, "This machine could support %s child processes, but only has enough memory for %s." % (default_count, supportable_instances)
+ return min(supportable_instances, default_count)
+
def _build_java_test_support(self):
java_tests_path = self._filesystem.join(self.layout_tests_dir(), "java")
build_java = ["/usr/bin/make", "-C", java_tests_path]
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py (120737 => 120738)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py 2012-06-19 18:05:58 UTC (rev 120737)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py 2012-06-19 18:21:16 UTC (rev 120738)
@@ -183,10 +183,22 @@
# MockPlatformInfo only has 2 mock cores. The important part is that 2 > 1.
self.assertEqual(port.default_child_processes(), 2)
+ bytes_for_drt = 200 * 1024 * 1024
+ port.host.platform.total_bytes_memory = lambda: bytes_for_drt
+ expected_stderr = "This machine could support 2 child processes, but only has enough memory for 1.\n"
+ child_processes = OutputCapture().assert_outputs(self, port.default_child_processes, (), expected_stderr=expected_stderr)
+ self.assertEqual(child_processes, 1)
+
+ # Make sure that we always use one process, even if we don't have the memory for it.
+ port.host.platform.total_bytes_memory = lambda: bytes_for_drt - 1
+ expected_stderr = "This machine could support 2 child processes, but only has enough memory for 1.\n"
+ child_processes = OutputCapture().assert_outputs(self, port.default_child_processes, (), expected_stderr=expected_stderr)
+ self.assertEqual(child_processes, 1)
+
# SnowLeopard has a CFNetwork bug which causes crashes if we execute more than one copy of DRT at once.
port = self.make_port(port_name='mac-snowleopard')
- expected_logs = "Cannot run tests in parallel on Snow Leopard due to rdar://problem/10621525.\n"
- child_processes = OutputCapture().assert_outputs(self, port.default_child_processes, (), expected_logs=expected_logs)
+ expected_stderr = "Cannot run tests in parallel on Snow Leopard due to rdar://problem/10621525.\n"
+ child_processes = OutputCapture().assert_outputs(self, port.default_child_processes, (), expected_stderr=expected_stderr)
self.assertEqual(child_processes, 1)
def test_get_crash_log(self):
