Reviewers: Jakob,
Message:
PTAL
Description:
Let test driver export json results.
BUG=374134
LOG=n
Please review this at https://codereview.chromium.org/285193009/
SVN Base: https://v8.googlecode.com/svn/branches/bleeding_edge
Affected files (+52, -28 lines):
M tools/run-deopt-fuzzer.py
M tools/run-tests.py
M tools/testrunner/local/execution.py
M tools/testrunner/local/progress.py
M tools/testrunner/network/network_execution.py
Index: tools/run-deopt-fuzzer.py
diff --git a/tools/run-deopt-fuzzer.py b/tools/run-deopt-fuzzer.py
index
44a4ab5e01e594719ac43d783572860322e479cf..6995d30b21f84586e59c0bd59a3b3bd407c4edf3
100755
--- a/tools/run-deopt-fuzzer.py
+++ b/tools/run-deopt-fuzzer.py
@@ -416,7 +416,7 @@ def Execute(arch, mode, args, options, suites,
workspace):
progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
runner = execution.Runner(suites, progress_indicator, ctx)
- exit_code = runner.Run(options.j)
+ exit_code, _ = runner.Run(options.j)
print(">>> Analysis phase")
num_tests = 0
@@ -463,7 +463,7 @@ def Execute(arch, mode, args, options, suites,
workspace):
progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
runner = execution.Runner(suites, progress_indicator, ctx)
- code = runner.Run(options.j)
+ code, _ = runner.Run(options.j)
return exit_code or code
Index: tools/run-tests.py
diff --git a/tools/run-tests.py b/tools/run-tests.py
index
6b07574139264778cf116a89bcb43bf583e72841..af5febc457d315f1047883810178544469e17b86
100755
--- a/tools/run-tests.py
+++ b/tools/run-tests.py
@@ -29,6 +29,7 @@
import itertools
+import json
import multiprocessing
import optparse
import os
@@ -173,6 +174,8 @@ def BuildOptions():
help=("Quick check mode (skip slow/flaky tests)"))
result.add_option("--report", help="Print a summary of the tests to be
run",
default=False, action="store_true")
+ result.add_option("--results-file",
+ help="Path to a file for storing json results.")
result.add_option("--shard-count",
help="Split testsuites into this number of shards",
default=1, type="int")
@@ -367,12 +370,17 @@ def Main():
for s in suites:
s.DownloadData()
+ results = []
for (arch, mode) in options.arch_and_mode:
try:
- code = Execute(arch, mode, args, options, suites, workspace)
+ code, result = Execute(arch, mode, args, options, suites, workspace)
+ results.append({"arch": arch, "mode": mode, "results": result})
except KeyboardInterrupt:
return 2
exit_code = exit_code or code
+ if options.results_file:
+ with open(options.results_file, "w") as f:
+ f.write(json.dumps(results))
return exit_code
@@ -453,14 +461,14 @@ def Execute(arch, mode, args, options, suites,
workspace):
test_id += 1
if options.cat:
- return 0 # We're done here.
+ return 0, [] # We're done here.
if options.report:
verbose.PrintReport(all_tests)
if num_tests == 0:
print "No tests to run."
- return 0
+ return 0, []
# Run the tests, either locally or distributed on the network.
start_time = time.time()
@@ -494,12 +502,12 @@ def Execute(arch, mode, args, options, suites,
workspace):
else:
runner = execution.Runner(suites, progress_indicator, ctx)
- exit_code = runner.Run(options.j)
+ exit_code, results = runner.Run(options.j)
overall_duration = time.time() - start_time
if options.time:
verbose.PrintTestDurations(suites, overall_duration)
- return exit_code
+ return exit_code, results
if __name__ == "__main__":
Index: tools/testrunner/local/execution.py
diff --git a/tools/testrunner/local/execution.py
b/tools/testrunner/local/execution.py
index
4dc62c2e90066032c2dd3412987a729786c14b9c..81f0da3d568a845e983d542f7be3cb730e97b8c7
100644
--- a/tools/testrunner/local/execution.py
+++ b/tools/testrunner/local/execution.py
@@ -35,6 +35,8 @@ from . import perfdata
from . import utils
+ABS_PATH_PREFIX = os.getcwd() + os.sep
+
class Job(object):
def __init__(self, command, dep_command, test_id, timeout, verbose):
self.command = command
@@ -57,6 +59,7 @@ def RunTest(job):
output = commands.Execute(job.command, job.verbose, job.timeout)
return (job.id, output, time.time() - start_time)
+
class Runner(object):
def __init__(self, suites, progress_indicator, context):
@@ -64,6 +67,7 @@ class Runner(object):
self.perf_data_manager = perfdata.PerfDataManager(datapath)
self.perfdata = self.perf_data_manager.GetStore(context.arch,
context.mode)
self.tests = [ t for s in suites for t in s.tests ]
+ self.results = []
for t in self.tests:
t.duration = self.perfdata.FetchPerfData(t) or 1.0
self._CommonInit(len(self.tests), progress_indicator, context)
@@ -83,8 +87,20 @@ class Runner(object):
self._RunInternal(jobs)
self.indicator.Done()
if self.failed or self.remaining:
- return 1
- return 0
+ return 1, self.results
+ return 0, self.results
+
+ def _AddResult(self, test, has_crashed):
+ self.results.append({
+ "suite": test.suite.name,
+ "path": test.path,
+ "flags": test.flags,
+ "command": self.GetEscapedCommand(test).replace(ABS_PATH_PREFIX, ""),
+ "stdout": test.output.stdout,
+ "stderr": test.output.stderr,
+ "exit_code": test.output.exit_code,
+ "result": "CRASH" if has_crashed else "FAIL",
+ })
def _RunInternal(self, jobs):
pool = Pool(jobs)
@@ -125,8 +141,9 @@ class Runner(object):
has_unexpected_output = test.suite.HasUnexpectedOutput(test)
if has_unexpected_output:
self.failed.append(test)
- if test.output.HasCrashed():
- self.crashed += 1
+ has_crashed = test.output.HasCrashed()
+ self.crashed += has_crashed
+ self._AddResult(test, has_crashed)
else:
self.succeeded += 1
self.remaining -= 1
@@ -142,7 +159,6 @@ class Runner(object):
if queued_exception:
raise queued_exception
-
def GetCommand(self, test):
d8testflag = []
shell = test.suite.shell()
@@ -158,6 +174,17 @@ class Runner(object):
self.context.extra_flags)
return cmd
+ def GetEscapedCommand(self, test):
+ parts = []
+ for part in self.GetCommand(test):
+ if ' ' in part:
+ # Escape spaces. We may need to escape more characters for this
+ # to work properly.
+ parts.append('"%s"' % part)
+ else:
+ parts.append(part)
+ return " ".join(parts)
+
class BreakNowException(Exception):
def __init__(self, value):
Index: tools/testrunner/local/progress.py
diff --git a/tools/testrunner/local/progress.py
b/tools/testrunner/local/progress.py
index
03116ee768d758cb1d0d6b3d6f8ab94dc08a7a0c..aee7c7f64f112ae0079f81d97786018eeb70022b
100644
--- a/tools/testrunner/local/progress.py
+++ b/tools/testrunner/local/progress.py
@@ -31,17 +31,6 @@ import time
from . import junit_output
-def EscapeCommand(command):
- parts = []
- for part in command:
- if ' ' in part:
- # Escape spaces. We may need to escape more characters for this
- # to work properly.
- parts.append('"%s"' % part)
- else:
- parts.append(part)
- return " ".join(parts)
-
class ProgressIndicator(object):
@@ -87,7 +76,7 @@ class SimpleProgressIndicator(ProgressIndicator):
if failed.output.stdout:
print "--- stdout ---"
print failed.output.stdout.strip()
- print "Command: %s" % EscapeCommand(self.runner.GetCommand(failed))
+ print "Command: %s" % self.runner.GetEscapedCommand(failed)
if failed.output.HasCrashed():
print "exit code: %d" % failed.output.exit_code
print "--- CRASHED ---"
@@ -170,7 +159,7 @@ class CompactProgressIndicator(ProgressIndicator):
stderr = test.output.stderr.strip()
if len(stderr):
print self.templates['stderr'] % stderr
- print "Command: %s" % EscapeCommand(self.runner.GetCommand(test))
+ print "Command: %s" % self.runner.GetEscapedCommand(test)
if test.output.HasCrashed():
print "exit code: %d" % test.output.exit_code
print "--- CRASHED ---"
@@ -266,7 +255,7 @@ class JUnitTestProgressIndicator(ProgressIndicator):
stderr = test.output.stderr.strip()
if len(stderr):
fail_text += "stderr:\n%s\n" % stderr
- fail_text += "Command: %s" %
EscapeCommand(self.runner.GetCommand(test))
+ fail_text += "Command: %s" % self.runner.GetEscapedCommand(test)
if test.output.HasCrashed():
fail_text += "exit code: %d\n--- CRASHED ---" %
test.output.exit_code
if test.output.HasTimedOut():
Index: tools/testrunner/network/network_execution.py
diff --git a/tools/testrunner/network/network_execution.py
b/tools/testrunner/network/network_execution.py
index
a43a6cfdedfc1b282165ba53eb1991006333e339..025b3afaa2641b61acfa7d407a7f4a4eff284dfb
100644
--- a/tools/testrunner/network/network_execution.py
+++ b/tools/testrunner/network/network_execution.py
@@ -134,7 +134,7 @@ class NetworkedRunner(execution.Runner):
if binary[0] is None:
print("Error: Failed to create signature.")
assert binary[1] != 0
- return binary[1]
+ return binary[1], []
binary.append(binary_needs_libv8)
self.binaries[shell] = binary
if need_libv8:
@@ -165,7 +165,7 @@ class NetworkedRunner(execution.Runner):
if self.tests:
self._RunInternal(jobs)
self.indicator.Done()
- return not self.failed
+ return not self.failed, []
def _TalkToPeer(self, peer):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
