Modified: trunk/Tools/Scripts/webkitpy/performance_tests/perftest.py (144477 => 144478)
--- trunk/Tools/Scripts/webkitpy/performance_tests/perftest.py 2013-03-01 19:51:31 UTC (rev 144477)
+++ trunk/Tools/Scripts/webkitpy/performance_tests/perftest.py 2013-03-01 20:04:13 UTC (rev 144478)
@@ -54,9 +54,10 @@
class PerfTestMetric(object):
def __init__(self, metric, unit=None, iterations=None):
- self._metric = metric
+ # FIXME: Fix runner.js to report correct metric names
self._iterations = iterations or []
self._unit = unit or self.metric_to_unit(metric)
+ self._metric = self.time_unit_to_metric(self._unit) if metric == 'Time' else metric
def metric(self):
return self._metric
@@ -67,45 +68,27 @@
# FIXME: We don't need to support this anymore. Make outputs more human friendly.
def legacy_chromium_bot_compatible_test_name(self, test_name_with_extension):
test_name = re.sub(r'\.\w+$', '', test_name_with_extension)
- return test_name if self._metric == 'Time' else test_name + ':' + self._metric
+ return test_name + ':' + self._metric
def append(self, value):
self._iterations.append(value)
- def to_dict(self):
- assert self.has_values()
- statistics = self.compute_statistics(self._iterations)
- statistics['unit'] = self._unit
- statistics['values'] = self._iterations
- return statistics
+ def iteration_values(self):
+ return self._iterations
- @classmethod
- def metric_to_unit(cls, metric):
+ def unit(self):
+ return self._unit
+
+ @staticmethod
+ def metric_to_unit(metric):
assert metric in ('Time', 'Malloc', 'JSHeap')
return 'ms' if metric == 'Time' else 'bytes'
@staticmethod
- def compute_statistics(values):
- sorted_values = sorted(values)
+ def time_unit_to_metric(unit):
+ return {'fps': 'FrameRate', 'runs/s': 'Runs', 'ms': 'Time'}[unit]
- # Compute the mean and variance using Knuth's online algorithm (has good numerical stability).
- squareSum = 0
- mean = 0
- for i, time in enumerate(sorted_values):
- delta = time - mean
- sweep = i + 1.0
- mean += delta / sweep
- squareSum += delta * (time - mean)
- middle = int(len(sorted_values) / 2)
- result = {'avg': sum(sorted_values) / len(values),
- 'min': sorted_values[0],
- 'max': sorted_values[-1],
- 'median': sorted_values[middle] if len(sorted_values) % 2 else (sorted_values[middle - 1] + sorted_values[middle]) / 2,
- 'stdev': math.sqrt(squareSum / (len(sorted_values) - 1)) if len(sorted_values) > 1 else 0}
- return result
-
-
class PerfTest(object):
def __init__(self, port, test_name, test_path):
self._port = port
@@ -138,19 +121,42 @@
if not metrics:
return metrics
+ should_log = not self._port.get_option('profile')
+ if should_log and self._description:
+ _log.info('DESCRIPTION: %s' % self._description)
+
results = {}
for metric in metrics:
legacy_test_name = metric.legacy_chromium_bot_compatible_test_name(self.test_name())
- results[legacy_test_name] = metric.to_dict()
+ results[legacy_test_name] = metric.iteration_values()
+ if should_log:
+ self.log_statistics(legacy_test_name.replace(':', ': ').replace('/', ': '),
+ metric.iteration_values(), metric.unit())
- if not self._port.get_option('profile'):
- if self._description:
- _log.info('DESCRIPTION: %s' % self._description)
- for result_name in sorted(results.keys()):
- self.output_statistics(result_name, results[result_name])
-
return results
+ @staticmethod
+ def log_statistics(test_name, values, unit):
+ sorted_values = sorted(values)
+
+ # Compute the mean and variance using Knuth's online algorithm (has good numerical stability).
+ square_sum = 0
+ mean = 0
+ for i, time in enumerate(sorted_values):
+ delta = time - mean
+ sweep = i + 1.0
+ mean += delta / sweep
+ square_sum += delta * (time - mean)
+
+ middle = int(len(sorted_values) / 2)
+ mean = sum(sorted_values) / len(values)
+ median = sorted_values[middle] if len(sorted_values) % 2 else (sorted_values[middle - 1] + sorted_values[middle]) / 2
+ stdev = math.sqrt(square_sum / (len(sorted_values) - 1)) if len(sorted_values) > 1 else 0
+
+ _log.info('RESULT %s= %s %s' % (test_name, mean, unit))
+ _log.info('median= %s %s, stdev= %s %s, min= %s %s, max= %s %s' %
+ (median, unit, stdev, unit, sorted_values[0], unit, sorted_values[-1], unit))
+
_description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IGNORECASE)
_metrics_regex = re.compile(r'^(?P<metric>Time|Malloc|JS Heap):')
_statistics_keys = ['avg', 'median', 'stdev', 'min', 'max', 'unit', 'values']
@@ -242,12 +248,7 @@
if output.text:
output.text = '\n'.join([line for line in re.split('\n', output.text) if not self._should_ignore_line(self._lines_to_ignore_in_parser_result, line)])
- def output_statistics(self, test_name, results):
- unit = results['unit']
- _log.info('RESULT %s= %s %s' % (test_name.replace(':', ': ').replace('/', ': '), results['avg'], unit))
- _log.info(', '.join(['%s= %s %s' % (key, results[key], unit) for key in self._statistics_keys[1:5]]))
-
class ChromiumStylePerfTest(PerfTest):
_chromium_style_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$')
Modified: trunk/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py (144477 => 144478)
--- trunk/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py 2013-03-01 19:51:31 UTC (rev 144477)
+++ trunk/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py 2013-03-01 20:04:13 UTC (rev 144478)
@@ -50,12 +50,17 @@
class TestPerfTestMetric(unittest.TestCase):
def test_init_set_missing_unit(self):
- self.assertEqual(PerfTestMetric('Time', iterations=[1, 2, 3, 4, 5]).to_dict()['unit'], 'ms')
- self.assertEqual(PerfTestMetric('Malloc', iterations=[1, 2, 3, 4, 5]).to_dict()['unit'], 'bytes')
- self.assertEqual(PerfTestMetric('JSHeap', iterations=[1, 2, 3, 4, 5]).to_dict()['unit'], 'bytes')
+ self.assertEqual(PerfTestMetric('Time', iterations=[1, 2, 3, 4, 5]).unit(), 'ms')
+ self.assertEqual(PerfTestMetric('Malloc', iterations=[1, 2, 3, 4, 5]).unit(), 'bytes')
+ self.assertEqual(PerfTestMetric('JSHeap', iterations=[1, 2, 3, 4, 5]).unit(), 'bytes')
+ def test_init_set_time_metric(self):
+ self.assertEqual(PerfTestMetric('Time', 'ms').metric(), 'Time')
+ self.assertEqual(PerfTestMetric('Time', 'fps').metric(), 'FrameRate')
+ self.assertEqual(PerfTestMetric('Time', 'runs/s').metric(), 'Runs')
+
def test_legacy_chromium_bot_compatible_test_name(self):
- self.assertEqual(PerfTestMetric('Time').legacy_chromium_bot_compatible_test_name('test'), 'test')
+ self.assertEqual(PerfTestMetric('Time').legacy_chromium_bot_compatible_test_name('test'), 'test:Time')
self.assertEqual(PerfTestMetric('Malloc').legacy_chromium_bot_compatible_test_name('test'), 'test:Malloc')
self.assertEqual(PerfTestMetric('JSHeap').legacy_chromium_bot_compatible_test_name('test'), 'test:JSHeap')
self.assertEqual(PerfTestMetric('FontSize', unit='em').legacy_chromium_bot_compatible_test_name('test'), 'test:FontSize')
@@ -73,51 +78,22 @@
metric.append(1)
self.assertTrue(metric.has_values())
self.assertFalse(metric2.has_values())
- self.assertEqual(metric.to_dict()['values'], [1])
+ self.assertEqual(metric.iteration_values(), [1])
metric.append(2)
- self.assertEqual(metric.to_dict()['values'], [1, 2])
+ self.assertEqual(metric.iteration_values(), [1, 2])
metric2.append(3)
self.assertTrue(metric2.has_values())
- self.assertEqual(metric.to_dict()['values'], [1, 2])
- self.assertEqual(metric2.to_dict()['values'], [3])
+ self.assertEqual(metric.iteration_values(), [1, 2])
+ self.assertEqual(metric2.iteration_values(), [3])
- def test_compute_statistics(self):
- def compute_statistics(values):
- statistics = PerfTestMetric.compute_statistics(map(lambda x: float(x), values))
- return json.loads(json.dumps(statistics))
- statistics = compute_statistics([10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11])
- self.assertItemsEqual(statistics.keys(), ['avg', 'max', 'median', 'min', 'stdev'])
- self.assertEqual(statistics['avg'], 10.5)
- self.assertEqual(statistics['min'], 1)
- self.assertEqual(statistics['max'], 20)
- self.assertEqual(statistics['median'], 10.5)
- self.assertEqual(compute_statistics([8, 9, 10, 11, 12])['avg'], 10)
- self.assertEqual(compute_statistics([8, 9, 10, 11, 12] * 4)['avg'], 10)
- self.assertEqual(compute_statistics([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19])['avg'], 10)
- self.assertEqual(compute_statistics([1, 5, 2, 8, 7])['median'], 5)
- self.assertEqual(compute_statistics([1, 6, 2, 8, 7, 2])['median'], 4)
- self.assertAlmostEqual(statistics['stdev'], math.sqrt(35))
- self.assertAlmostEqual(compute_statistics([1])['stdev'], 0)
- self.assertAlmostEqual(compute_statistics([1, 2, 3, 4, 5, 6])['stdev'], math.sqrt(3.5))
- self.assertAlmostEqual(compute_statistics([4, 2, 5, 8, 6])['stdev'], math.sqrt(5))
-
-
class TestPerfTest(unittest.TestCase):
def _assert_results_are_correct(self, test, output):
test.run_single = lambda driver, path, time_out_ms: output
parsed_results = test._run_with_driver(None, None)
self.assertEqual(len(parsed_results), 1)
- some_test_results = parsed_results[0].to_dict()
- self.assertItemsEqual(some_test_results.keys(), ['avg', 'max', 'median', 'min', 'stdev', 'unit', 'values'])
- self.assertEqual(some_test_results['values'], [1080, 1120, 1095, 1101, 1104])
- self.assertEqual(some_test_results['min'], 1080)
- self.assertEqual(some_test_results['max'], 1120)
- self.assertEqual(some_test_results['avg'], 1100)
- self.assertEqual(some_test_results['median'], 1101)
- self.assertAlmostEqual(some_test_results['stdev'], 14.50862, places=5)
- self.assertEqual(some_test_results['unit'], 'ms')
+ self.assertEqual(parsed_results[0].iteration_values(), [1080, 1120, 1095, 1101, 1104])
def test_parse_output(self):
output = DriverOutput("""
@@ -341,8 +317,7 @@
self.assertEqual(len(metrics), 1)
self.assertEqual(metrics[0].metric(), 'Time')
- self.assertEqual(metrics[0].to_dict(), {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
- 'values': [float(i * 1000) for i in range(2, 21)]})
+ self.assertEqual(metrics[0].iteration_values(), [float(i * 1000) for i in range(2, 21)])
def test_run_with_driver_accumulates_memory_results(self):
port = MockPort()
@@ -368,14 +343,11 @@
self.assertEqual(len(metrics), 3)
self.assertEqual(metrics[0].metric(), 'Time')
- self.assertEqual(metrics[0].to_dict(), {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
- 'values': [float(i * 1000) for i in range(2, 21)]})
+ self.assertEqual(metrics[0].iteration_values(), [float(i * 1000) for i in range(2, 21)])
self.assertEqual(metrics[1].metric(), 'Malloc')
- self.assertEqual(metrics[1].to_dict(), {'max': 10, 'avg': 10.0, 'median': 10, 'min': 10, 'stdev': 0.0, 'unit': 'bytes',
- 'values': [float(10)] * 19})
+ self.assertEqual(metrics[1].iteration_values(), [float(10)] * 19)
self.assertEqual(metrics[2].metric(), 'JSHeap')
- self.assertEqual(metrics[2].to_dict(), {'max': 5, 'avg': 5.0, 'median': 5, 'min': 5, 'stdev': 0.0, 'unit': 'bytes',
- 'values': [float(5)] * 19})
+ self.assertEqual(metrics[2].iteration_values(), [float(5)] * 19)
def test_prepare_fails_when_wait_until_ready_fails(self):
output_capture = OutputCapture()
