Modified: trunk/Tools/Scripts/webkitpy/performance_tests/perftest.py (129157 => 129158)
--- trunk/Tools/Scripts/webkitpy/performance_tests/perftest.py 2012-09-20 19:54:28 UTC (rev 129157)
+++ trunk/Tools/Scripts/webkitpy/performance_tests/perftest.py 2012-09-20 20:04:31 UTC (rev 129158)
@@ -114,8 +114,8 @@
_description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IGNORECASE)
_result_classes = ['Time', 'JS Heap', 'Malloc']
_result_class_regex = re.compile(r'^(?P<resultclass>' + r'|'.join(_result_classes) + '):')
- _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max', 'unit']
- _score_regex = re.compile(r'^(?P<key>' + r'|'.join(_statistics_keys) + r')\s+(?P<value>[0-9\.]+)\s*(?P<unit>.*)')
+ _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max', 'unit', 'values']
+ _score_regex = re.compile(r'^(?P<key>' + r'|'.join(_statistics_keys) + r')\s+(?P<value>([0-9\.]+(,\s+)?)+)\s*(?P<unit>.*)')
def parse_output(self, output):
test_failed = False
@@ -138,7 +138,10 @@
score = self._score_regex.match(line)
if score:
key = score.group('key')
- value = float(score.group('value'))
+ if ', ' in score.group('value'):
+ value = [float(number) for number in score.group('value').split(', ')]
+ else:
+ value = float(score.group('value'))
unit = score.group('unit')
name = test_name
if result_class != 'Time':
@@ -154,9 +157,14 @@
test_failed = True
_log.error(line)
- if test_failed or set(self._statistics_keys) != set(results[test_name].keys()):
+ if test_failed:
return None
+ if set(self._statistics_keys) != set(results[test_name].keys() + ['values']):
+ # values is not provided by Dromaeo tests.
+ _log.error("The test didn't report all statistics.")
+ return None
+
for result_name in ordered_results_keys:
if result_name == test_name:
self.output_statistics(result_name, results[result_name], description_string)
@@ -208,23 +216,24 @@
continue
test_times.append(output.test_time * 1000)
- test_times = sorted(test_times)
+ sorted_test_times = sorted(test_times)
# Compute the mean and variance using a numerically stable algorithm.
squareSum = 0
mean = 0
- valueSum = sum(test_times)
- for i, time in enumerate(test_times):
+ valueSum = sum(sorted_test_times)
+ for i, time in enumerate(sorted_test_times):
delta = time - mean
sweep = i + 1.0
mean += delta / sweep
squareSum += delta * delta * (i / sweep)
middle = int(len(test_times) / 2)
- results = {'avg': mean,
- 'min': min(test_times),
- 'max': max(test_times),
- 'median': test_times[middle] if len(test_times) % 2 else (test_times[middle - 1] + test_times[middle]) / 2,
+ results = {'values': test_times,
+ 'avg': mean,
+ 'min': sorted_test_times[0],
+ 'max': sorted_test_times[-1],
+ 'median': sorted_test_times[middle] if len(sorted_test_times) % 2 else (sorted_test_times[middle - 1] + sorted_test_times[middle]) / 2,
'stdev': math.sqrt(squareSum),
'unit': 'ms'}
self.output_statistics(self.test_name(), results, '')
Modified: trunk/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py (129157 => 129158)
--- trunk/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py 2012-09-20 19:54:28 UTC (rev 129157)
+++ trunk/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py 2012-09-20 20:04:31 UTC (rev 129158)
@@ -50,6 +50,7 @@
'Ignoring warm-up run (1115)',
'',
'Time:',
+ 'values 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ms',
'avg 1100 ms',
'median 1101 ms',
'stdev 11 ms',
@@ -60,7 +61,8 @@
try:
test = PerfTest(None, 'some-test', '/path/some-dir/some-test')
self.assertEqual(test.parse_output(output),
- {'some-test': {'avg': 1100.0, 'median': 1101.0, 'min': 1080.0, 'max': 1120.0, 'stdev': 11.0, 'unit': 'ms'}})
+ {'some-test': {'avg': 1100.0, 'median': 1101.0, 'min': 1080.0, 'max': 1120.0, 'stdev': 11.0, 'unit': 'ms',
+ 'values': [i for i in range(1, 20)]}})
finally:
pass
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
@@ -76,6 +78,7 @@
'some-unrecognizable-line',
'',
'Time:'
+ 'values 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ms',
'avg 1100 ms',
'median 1101 ms',
'stdev 11 ms',
@@ -109,12 +112,13 @@
def test_run(self):
test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test')
- driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
+ driver = TestPageLoadingPerfTest.MockDriver(range(1, 21))
output_capture = OutputCapture()
output_capture.capture_output()
try:
self.assertEqual(test.run(driver, None),
- {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms'}})
+ {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms',
+ 'values': [i * 1000 for i in range(2, 21)]}})
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
Modified: trunk/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py (129157 => 129158)
--- trunk/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py 2012-09-20 19:54:28 UTC (rev 129157)
+++ trunk/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py 2012-09-20 20:04:31 UTC (rev 129158)
@@ -92,6 +92,7 @@
1471
Time:
+values 1504, 1505, 1510, 1504, 1507, 1509, 1510, 1487, 1488, 1472, 1472, 1488, 1473, 1472, 1475, 1487, 1486, 1486, 1475, 1471 ms
avg 1489.05 ms
median 1487 ms
stdev 14.46 ms
@@ -103,6 +104,7 @@
Ignoring warm-up run (1115)
Time:
+values 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ms
avg 1100 ms
median 1101 ms
stdev 11 ms
@@ -114,6 +116,7 @@
Ignoring warm-up run (1115)
Time:
+values 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ms
avg 1100 ms
median 1101 ms
stdev 11 ms
@@ -121,6 +124,7 @@
max 1120 ms
JS Heap:
+values 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 bytes
avg 832000 bytes
median 829000 bytes
stdev 15000 bytes
@@ -128,6 +132,7 @@
max 848000 bytes
Malloc:
+values 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 bytes
avg 532000 bytes
median 529000 bytes
stdev 13000 bytes
@@ -286,9 +291,10 @@
'Finished: 0.1 s',
'', '']))
results = runner.load_output_json()[0]['results']
- self.assertEqual(results['Parser/memory-test'], {'min': 1080.0, 'max': 1120.0, 'median': 1101.0, 'stdev': 11.0, 'avg': 1100.0, 'unit': 'ms'})
- self.assertEqual(results['Parser/memory-test:JSHeap'], {'min': 811000.0, 'max': 848000.0, 'median': 829000.0, 'stdev': 15000.0, 'avg': 832000.0, 'unit': 'bytes'})
- self.assertEqual(results['Parser/memory-test:Malloc'], {'min': 511000.0, 'max': 548000.0, 'median': 529000.0, 'stdev': 13000.0, 'avg': 532000.0, 'unit': 'bytes'})
+ values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+ self.assertEqual(results['Parser/memory-test'], {'min': 1080.0, 'max': 1120.0, 'median': 1101.0, 'stdev': 11.0, 'avg': 1100.0, 'unit': 'ms', 'values': values})
+ self.assertEqual(results['Parser/memory-test:JSHeap'], {'min': 811000.0, 'max': 848000.0, 'median': 829000.0, 'stdev': 15000.0, 'avg': 832000.0, 'unit': 'bytes', 'values': values})
+ self.assertEqual(results['Parser/memory-test:Malloc'], {'min': 511000.0, 'max': 548000.0, 'median': 529000.0, 'stdev': 13000.0, 'avg': 532000.0, 'unit': 'bytes', 'values': values})
def _test_run_with_json_output(self, runner, filesystem, upload_suceeds=False, expected_exit_code=0):
filesystem.write_text_file(runner._base_path + '/inspector/pass.html', 'some content')
@@ -330,6 +336,12 @@
return logs
_event_target_wrapper_and_inspector_results = {
+ "Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms",
+ "values": [1504, 1505, 1510, 1504, 1507, 1509, 1510, 1487, 1488, 1472, 1472, 1488, 1473, 1472, 1475, 1487, 1486, 1486, 1475, 1471]},
+ "inspector/pass.html:group_name:test_name": 42}
+
+ # FIXME: Remove this variance once perf-o-matic supported "values".
+ _event_target_wrapper_and_inspector_results_without_values = {
"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"},
"inspector/pass.html:group_name:test_name": 42}
@@ -338,7 +350,7 @@
'--test-results-server=some.host'])
self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True)
self.assertEqual(runner.load_output_json(), {
- "timestamp": 123456789, "results": self._event_target_wrapper_and_inspector_results,
+ "timestamp": 123456789, "results": self._event_target_wrapper_and_inspector_results_without_values,
"webkit-revision": "5678", "branch": "webkit-trunk"})
def test_run_with_description(self):
@@ -347,7 +359,7 @@
self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True)
self.assertEqual(runner.load_output_json(), {
"timestamp": 123456789, "description": "some description",
- "results": self._event_target_wrapper_and_inspector_results,
+ "results": self._event_target_wrapper_and_inspector_results_without_values,
"webkit-revision": "5678", "branch": "webkit-trunk"})
def create_runner_and_setup_results_template(self, args=[]):
@@ -437,7 +449,7 @@
port.host.filesystem.write_text_file('/mock-checkout/slave-config.json', '{"key": "value"}')
self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True)
self.assertEqual(runner.load_output_json(), {
- "timestamp": 123456789, "results": self._event_target_wrapper_and_inspector_results,
+ "timestamp": 123456789, "results": self._event_target_wrapper_and_inspector_results_without_values,
"webkit-revision": "5678", "branch": "webkit-trunk", "key": "value"})
def test_run_with_bad_slave_config_json(self):
@@ -456,7 +468,7 @@
port.repository_paths = lambda: [('webkit', '/mock-checkout'), ('some', '/mock-checkout/some')]
self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True)
self.assertEqual(runner.load_output_json(), {
- "timestamp": 123456789, "results": self._event_target_wrapper_and_inspector_results,
+ "timestamp": 123456789, "results": self._event_target_wrapper_and_inspector_results_without_values,
"webkit-revision": "5678", "some-revision": "5678", "branch": "webkit-trunk"})
def test_run_with_upload_json(self):
