Reviewers: Benedikt Meurer, tandrii(chromium),
Message:
PTAL
Description:
Make perf test runner more robust.
Convert all results to floats and log errors on failing
conversions. Otherwise these results bubble up into
buildbot.
[email protected]
BUG=chromium:374740
LOG=n
NOTRY=true
Please review this at https://codereview.chromium.org/811483008/
Base URL: https://chromium.googlesource.com/v8/v8.git@master
Affected files (+45, -28 lines):
M tools/run_perf.py
M tools/unittests/run_perf_test.py
Index: tools/run_perf.py
diff --git a/tools/run_perf.py b/tools/run_perf.py
index
a4fc8b3463d14856fdc0d5b0935f1aa24b4fefe0..14d67c56b046332496a9fcf53206a24f46fc19a9
100755
--- a/tools/run_perf.py
+++ b/tools/run_perf.py
@@ -265,8 +265,11 @@ class Trace(Graph):
def ConsumeOutput(self, stdout):
try:
- self.results.append(
- re.search(self.results_regexp, stdout, re.M).group(1))
+ result = re.search(self.results_regexp, stdout, re.M).group(1)
+ self.results.append(str(float(result)))
+ except ValueError:
+ self.errors.append("Regexp \"%s\" returned a non-numeric for
test %s."
+ % (self.results_regexp, self.graphs[-1]))
except:
self.errors.append("Regexp \"%s\" didn't match for test %s."
% (self.results_regexp, self.graphs[-1]))
@@ -374,6 +377,7 @@ class RunnableGeneric(Runnable):
units = match.group(4)
match_stddev = RESULT_STDDEV_RE.match(body)
match_list = RESULT_LIST_RE.match(body)
+ errors = []
if match_stddev:
result, stddev = map(str.strip,
match_stddev.group(1).split(","))
results = [result]
@@ -382,12 +386,19 @@ class RunnableGeneric(Runnable):
else:
results = [body.strip()]
+ try:
+ results = map(lambda r: str(float(r)), results)
+ except ValueError:
+ results = []
+ errors = ["Found non-numeric in %s" %
+ "/".join(self.graphs + [graph, trace])]
+
trace_result = traces.setdefault(trace, Results([{
"graphs": self.graphs + [graph, trace],
"units": (units or self.units).strip(),
"results": [],
"stddev": "",
- }], []))
+ }], errors))
trace_result.traces[0]["results"].extend(results)
trace_result.traces[0]["stddev"] = stddev
Index: tools/unittests/run_perf_test.py
diff --git a/tools/unittests/run_perf_test.py
b/tools/unittests/run_perf_test.py
index
5ac0789ae86438923576c07b88830281dc2a9a81..f9ea0c09c990e1084de0afb27aa9df262e9ca8dc
100644
--- a/tools/unittests/run_perf_test.py
+++ b/tools/unittests/run_perf_test.py
@@ -174,7 +174,7 @@ class PerfTest(unittest.TestCase):
self.assertEquals(0, self._CallMain())
self._VerifyResults("test", "score", [
{"name": "Richards", "results": ["1.234"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+ {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
])
self._VerifyErrors([])
self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
@@ -187,7 +187,7 @@ class PerfTest(unittest.TestCase):
self.assertEquals(0, self._CallMain())
self._VerifyResults("test", "score", [
{"name": "Richards", "results": ["1.234"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+ {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
])
self._VerifyErrors([])
self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js",
@@ -204,8 +204,8 @@ class PerfTest(unittest.TestCase):
"Richards: 50\nDeltaBlue: 300\n"])
self.assertEquals(0, self._CallMain())
self._VerifyResults("v8", "ms", [
- {"name": "Richards", "results": ["50", "100"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
+ {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""},
+ {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""},
])
self._VerifyErrors([])
self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
@@ -222,8 +222,8 @@ class PerfTest(unittest.TestCase):
"Richards: 50\nDeltaBlue: 300\n"])
self.assertEquals(0, self._CallMain())
self._VerifyResults("test", "score", [
- {"name": "Richards", "results": ["50", "100"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
+ {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""},
+ {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""},
])
self._VerifyErrors([])
self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
@@ -241,15 +241,15 @@ class PerfTest(unittest.TestCase):
self.assertEquals([
{"units": "score",
"graphs": ["test", "Richards"],
- "results": ["50", "100"],
+ "results": ["50.0", "100.0"],
"stddev": ""},
{"units": "ms",
"graphs": ["test", "Sub", "Leaf"],
- "results": ["3", "2", "1"],
+ "results": ["3.0", "2.0", "1.0"],
"stddev": ""},
{"units": "score",
"graphs": ["test", "DeltaBlue"],
- "results": ["200"],
+ "results": ["200.0"],
"stddev": ""},
], self._LoadResults()["traces"])
self._VerifyErrors([])
@@ -270,7 +270,7 @@ class PerfTest(unittest.TestCase):
self.assertEquals(0, self._CallMain())
self._VerifyResults("test", "score", [
{"name": "Richards", "results": ["1.234"], "stddev": "0.23"},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": "106"},
+ {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": "106"},
])
self._VerifyErrors([])
self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
@@ -286,8 +286,8 @@ class PerfTest(unittest.TestCase):
"DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"])
self.assertEquals(1, self._CallMain())
self._VerifyResults("test", "score", [
- {"name": "Richards", "results": ["2", "3"], "stddev": "0.7"},
- {"name": "DeltaBlue", "results": ["5", "6"], "stddev": "0.8"},
+ {"name": "Richards", "results": ["2.0", "3.0"], "stddev": "0.7"},
+ {"name": "DeltaBlue", "results": ["5.0", "6.0"], "stddev": "0.8"},
])
self._VerifyErrors(
["Test Richards should only run once since a stddev is provided "
@@ -304,7 +304,7 @@ class PerfTest(unittest.TestCase):
self.assertEquals(0, self._CallMain("--buildbot"))
self._VerifyResults("test", "score", [
{"name": "Richards", "results": ["1.234"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+ {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
])
self._VerifyErrors([])
self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
@@ -317,7 +317,7 @@ class PerfTest(unittest.TestCase):
self.assertEquals(0, self._CallMain("--buildbot"))
self._VerifyResults("test", "score", [
{"name": "Richards", "results": ["1.234"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+ {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
{"name": "Total", "results": ["3626.49109719"], "stddev": ""},
])
self._VerifyErrors([])
@@ -327,14 +327,15 @@ class PerfTest(unittest.TestCase):
test_input = dict(V8_JSON)
test_input["total"] = True
self._WriteTestInput(test_input)
- self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue:
10657567\ny\n"])
+ self._MockCommand(["."], ["x\nRichards: bla\nDeltaBlue:
10657567\ny\n"])
self.assertEquals(1, self._CallMain("--buildbot"))
self._VerifyResults("test", "score", [
{"name": "Richards", "results": [], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+ {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
])
self._VerifyErrors(
- ["Regexp \"^Richards: (.+)$\" didn't match for test Richards.",
+ ["Regexp \"^Richards: (.+)$\" "
+ "returned a non-numeric for test Richards.",
"Not all traces have the same number of results."])
self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
@@ -344,7 +345,7 @@ class PerfTest(unittest.TestCase):
self.assertEquals(1, self._CallMain())
self._VerifyResults("test", "score", [
{"name": "Richards", "results": [], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+ {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
])
self._VerifyErrors(
["Regexp \"^Richards: (.+)$\" didn't match for test Richards."])
@@ -356,23 +357,28 @@ class PerfTest(unittest.TestCase):
self._MockCommand(["."], [
"RESULT Infra: Constant1= 11 count\n"
"RESULT Infra: Constant2= [10,5,10,15] count\n"
- "RESULT Infra: Constant3= {12,1.2} count\n"])
- self.assertEquals(0, self._CallMain())
+ "RESULT Infra: Constant3= {12,1.2} count\n"
+ "RESULT Infra: Constant4= [10,5,error,15] count\n"])
+ self.assertEquals(1, self._CallMain())
self.assertEquals([
{"units": "count",
"graphs": ["test", "Infra", "Constant1"],
- "results": ["11"],
+ "results": ["11.0"],
"stddev": ""},
{"units": "count",
"graphs": ["test", "Infra", "Constant2"],
- "results": ["10", "5", "10", "15"],
+ "results": ["10.0", "5.0", "10.0", "15.0"],
"stddev": ""},
{"units": "count",
"graphs": ["test", "Infra", "Constant3"],
- "results": ["12"],
+ "results": ["12.0"],
"stddev": "1.2"},
+ {"units": "count",
+ "graphs": ["test", "Infra", "Constant4"],
+ "results": [],
+ "stddev": ""},
], self._LoadResults()["traces"])
- self._VerifyErrors([])
+ self._VerifyErrors(["Found non-numeric in test/Infra/Constant4"])
self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "")
def testOneRunTimingOut(self):
@@ -408,5 +414,5 @@ class PerfTest(unittest.TestCase):
"--arch", "android_arm"))
self._VerifyResults("test", "score", [
{"name": "Richards", "results": ["1.234"], "stddev": ""},
- {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
+ {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
])
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
