Reviewers: Benedikt Meurer,
Message:
PTAL
https://codereview.chromium.org/502473002/diff/20001/tools/run_benchmarks.py
File tools/run_benchmarks.py (right):
https://codereview.chromium.org/502473002/diff/20001/tools/run_benchmarks.py#newcode316
tools/run_benchmarks.py:316: trace_result.traces[0]["stddev"] = stddev
Note: Having multiple reruns by the driver and a stddev from the
executable makes no sense. But this error case is not tested (instead,
the provided stddev would be overwritten in the end).
Description:
Teach benchmark runner to understand generic traces.
BUG=406405
Please review this at https://codereview.chromium.org/502473002/
SVN Base: https://v8.googlecode.com/svn/branches/bleeding_edge
Affected files (+57, -1 lines):
M tools/run_benchmarks.py
M tools/unittests/run_benchmarks_test.py
Index: tools/run_benchmarks.py
diff --git a/tools/run_benchmarks.py b/tools/run_benchmarks.py
index
d6e9145dacea3bdf73062d5fa88954d5b807bdd8..b74f3588f360e2ed869ab76cbf2fcb543df69559
100755
--- a/tools/run_benchmarks.py
+++ b/tools/run_benchmarks.py
@@ -113,6 +113,8 @@ SUPPORTED_ARCHS = ["android_arm",
"x64",
"arm64"]
+GENERIC_RESULTS_RE = re.compile(
+ r"^Trace\(([^\)]+)\), Result\(([^\)]+)\), StdDev\(([^\)]+)\)$")
class Results(object):
"""Place holder for result traces."""
@@ -249,7 +251,7 @@ class Runnable(Graph):
"""
@property
def main(self):
- return self._suite["main"]
+ return self._suite.get("main", "")
def ChangeCWD(self, suite_path):
"""Changes the cwd to to path defined in the current graph.
@@ -289,6 +291,33 @@ class RunnableTrace(Trace, Runnable):
return self.GetResults()
+class RunnableGeneric(Runnable):
+ """Represents a runnable benchmark suite definition with generic
traces."""
+ def __init__(self, suite, parent, arch):
+ super(RunnableGeneric, self).__init__(suite, parent, arch)
+
+ def Run(self, runner):
+ """Iterates over several runs and handles the output."""
+ traces = {}
+ for stdout in runner():
+ for line in stdout.strip().splitlines():
+ match = GENERIC_RESULTS_RE.match(line)
+ if match:
+ trace = match.group(1)
+ result = match.group(2)
+ stddev = match.group(3)
+ trace_result = traces.setdefault(trace, Results([{
+ "graphs": self.graphs + [trace],
+ "units": self.units,
+ "results": [],
+ "stddev": "",
+ }], []))
+ trace_result.traces[0]["results"].append(result)
+ trace_result.traces[0]["stddev"] = stddev
+
+ return reduce(lambda r, t: r + t, traces.itervalues(), Results())
+
+
def MakeGraph(suite, arch, parent):
"""Factory method for making graph objects."""
if isinstance(parent, Runnable):
@@ -302,6 +331,10 @@ def MakeGraph(suite, arch, parent):
else:
# This graph has no subbenchmarks, it's a leaf.
return RunnableTrace(suite, parent, arch)
+ elif suite.get("generic"):
+ # This is a generic suite definition. It is either a runnable
executable
+ # or has a main js file.
+ return RunnableGeneric(suite, parent, arch)
elif suite.get("benchmarks"):
# This is neither a leaf nor a runnable.
return Graph(suite, parent, arch)
Index: tools/unittests/run_benchmarks_test.py
diff --git a/tools/unittests/run_benchmarks_test.py
b/tools/unittests/run_benchmarks_test.py
index
37a816e760d2a59d5107f86a54efa2eeddfe2828..054522091ed1738143ac30cb686b808eed0af2aa
100644
--- a/tools/unittests/run_benchmarks_test.py
+++ b/tools/unittests/run_benchmarks_test.py
@@ -68,6 +68,15 @@ V8_NESTED_SUITES_JSON = {
]
}
+V8_GENERIC_JSON = {
+ "path": ["."],
+ "binary": "cc",
+ "flags": ["--flag"],
+ "generic": True,
+ "run_count": 1,
+ "units": "ms",
+}
+
Output = namedtuple("Output", "stdout, stderr")
class BenchmarksTest(unittest.TestCase):
@@ -295,3 +304,17 @@ class BenchmarksTest(unittest.TestCase):
self._VerifyErrors(
["Regexp \"^Richards: (.+)$\" didn't match for benchmark
Richards."])
self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
+
+ def testOneRunGeneric(self):
+ test_input = dict(V8_GENERIC_JSON)
+ self._WriteTestInput(test_input)
+ self._MockCommand(["."], [
+ "Trace(Test1), Result(1.234), StdDev(0.23)\n"
+ "Trace(Test2), Result(10657567), StdDev(106)\n"])
+ self.assertEquals(0, self._CallMain())
+ self._VerifyResults("test", "ms", [
+ {"name": "Test1", "results": ["1.234"], "stddev": "0.23"},
+ {"name": "Test2", "results": ["10657567"], "stddev": "106"},
+ ])
+ self._VerifyErrors([])
+ self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "")
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
