Revision: 3240 Author: [email protected] Date: Mon Nov 9 01:31:57 2009 Log: Added test runner for sputnik tests.
Review URL: http://codereview.chromium.org/372053 http://code.google.com/p/v8/source/detail?r=3240 Added: /branches/bleeding_edge/test/sputnik /branches/bleeding_edge/test/sputnik/README /branches/bleeding_edge/test/sputnik/sputnik.status /branches/bleeding_edge/test/sputnik/testcfg.py Modified: /branches/bleeding_edge/tools/test.py ======================================= --- /dev/null +++ /branches/bleeding_edge/test/sputnik/README Mon Nov 9 01:31:57 2009 @@ -0,0 +1,6 @@ +To run the sputniktests you must check out the test suite from +googlecode.com. The test expectations are currently relative to +version 21. To get the tests run the following command within +v8/tests/sputnik/ + + svn co http://sputniktests.googlecode.com/svn/trunk/ -r21 sputniktests ======================================= --- /dev/null +++ /branches/bleeding_edge/test/sputnik/sputnik.status Mon Nov 9 01:31:57 2009 @@ -0,0 +1,325 @@ +# Copyright 2009 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +prefix sputnik +def FAIL_OK = FAIL, OKAY + +##################### DELIBERATE INCOMPATIBILITIES ##################### + +# 900066: Deleting elements in .arguments should disconnect the +# element from the actual arguments. Implementing this is nontrivial +# and we have no indication that anything on the web depends on this +# feature. +S13_A13_T1: FAIL_OK +S13_A13_T2: FAIL_OK +S13_A13_T3: FAIL_OK + +# This tests precision of trignometric functions. We're slightly off +# from the implementation in libc (~ 1e-17) but it's not clear if we +# or they are closer to the right answer, or if it even matters. +S15.8.2.16_A7: PASS || FAIL_OK +S15.8.2.18_A7: PASS || FAIL_OK +S15.8.2.13_A23: PASS || FAIL_OK + +# We allow calls to regexp exec() with no arguments to fail for +# compatibility reasons. +S15.10.6.2_A1_T16: FAIL_OK +S15.10.6.3_A1_T16: FAIL_OK + +# We allow regexps to be called as functions for compatibility reasons. +S15.10.7_A1_T1: FAIL_OK +S15.10.7_A1_T2: FAIL_OK + +# We allow construct calls to built-in functions, and we allow built-in +# functions to have prototypes. +S15.1.2.1_A4.6: FAIL_OK +S15.1.2.1_A4.7: FAIL_OK +S15.1.2.2_A9.6: FAIL_OK +S15.1.2.2_A9.7: FAIL_OK +S15.1.2.3_A7.6: FAIL_OK +S15.1.2.3_A7.7: FAIL_OK +S15.1.2.4_A2.6: FAIL_OK +S15.1.2.4_A2.7: FAIL_OK +S15.1.2.5_A2.6: FAIL_OK +S15.1.2.5_A2.7: FAIL_OK +S15.1.3.1_A5.6: FAIL_OK +S15.1.3.1_A5.7: FAIL_OK +S15.1.3.2_A5.6: FAIL_OK +S15.1.3.2_A5.7: FAIL_OK +S15.1.3.3_A5.6: FAIL_OK +S15.1.3.3_A5.7: FAIL_OK +S15.1.3.4_A5.6: FAIL_OK +S15.1.3.4_A5.7: FAIL_OK +S15.10.6.2_A6: FAIL_OK +S15.10.6.3_A6: FAIL_OK +S15.10.6.4_A6: FAIL_OK +S15.10.6.4_A7: FAIL_OK +S15.2.4.2_A6: FAIL_OK +S15.2.4.3_A6: FAIL_OK +S15.2.4.4_A6: FAIL_OK +S15.2.4.5_A6: FAIL_OK +S15.2.4.6_A6: FAIL_OK +S15.2.4.7_A6: FAIL_OK +S15.3.4.2_A6: FAIL_OK +S15.4.4.10_A5.6: FAIL_OK +S15.4.4.10_A5.7: FAIL_OK +S15.4.4.11_A7.6: FAIL_OK +S15.4.4.11_A7.7: FAIL_OK +S15.4.4.12_A5.6: FAIL_OK +S15.4.4.12_A5.7: FAIL_OK +S15.4.4.13_A5.6: FAIL_OK +S15.4.4.13_A5.7: FAIL_OK +S15.4.4.2_A4.6: FAIL_OK +S15.4.4.3_A4.6: FAIL_OK +S15.4.4.3_A4.6: FAIL_OK +S15.4.4.4_A4.6: FAIL_OK +S15.4.4.4_A4.7: FAIL_OK +S15.4.4.5_A6.6: FAIL_OK +S15.4.4.5_A6.7: FAIL_OK +S15.4.4.6_A5.6: FAIL_OK +S15.4.4.6_A5.7: FAIL_OK +S15.4.4.7_A6.6: FAIL_OK +S15.4.4.7_A6.7: FAIL_OK +S15.4.4.8_A5.6: FAIL_OK +S15.4.4.8_A5.7: FAIL_OK +S15.4.4.9_A5.6: FAIL_OK +S15.4.4.9_A5.7: FAIL_OK +S15.5.4.10_A6: FAIL_OK +S15.5.4.11_A6: FAIL_OK +S15.5.4.12_A6: FAIL_OK +S15.5.4.13_A6: FAIL_OK +S15.5.4.14_A6: FAIL_OK +S15.5.4.15_A6: FAIL_OK +S15.5.4.16_A6: FAIL_OK +S15.5.4.17_A6: FAIL_OK +S15.5.4.18_A6: FAIL_OK +S15.5.4.19_A6: FAIL_OK +S15.5.4.4_A6: FAIL_OK +S15.5.4.5_A6: FAIL_OK +S15.5.4.6_A6: FAIL_OK +S15.5.4.7_A6: FAIL_OK +S15.5.4.9_A6: FAIL_OK +S15.3.4.3_A12: FAIL_OK +S15.3.4.4_A12: FAIL_OK +S15.5.4.8_A6: FAIL_OK + +# We are silent in some regexp cases where the spec wants us to give +# errors, for compatibility. +S15.10.2.11_A1_T2: FAIL +S15.10.2.11_A1_T3: FAIL +S15.10.4.1_A5_T1: FAIL +S15.10.4.1_A5_T2: FAIL +S15.10.4.1_A5_T3: FAIL +S15.10.4.1_A5_T4: FAIL +S15.10.4.1_A5_T5: FAIL +S15.10.4.1_A5_T6: FAIL +S15.10.4.1_A5_T7: FAIL +S15.10.4.1_A5_T8: FAIL +S15.10.4.1_A5_T9: FAIL + +# We are more lenient in which string character escapes we allow than +# the spec (7.8.4 p. 19) wants us to be. This is for compatibility. +S7.8.4_A4.3_T2: FAIL_OK +S7.8.4_A4.3_T2: FAIL_OK +S7.8.4_A6.2_T2: FAIL_OK +S7.8.4_A6.1_T4: FAIL_OK +S7.8.4_A4.3_T4: FAIL_OK +S7.8.4_A7.2_T2: FAIL_OK +S7.8.4_A7.1_T4: FAIL_OK +S7.8.4_A6.4_T2: FAIL_OK +S7.8.4_A7.4_T2: FAIL_OK +S7.8.4_A7.2_T4: FAIL_OK +S7.8.4_A4.3_T6: FAIL_OK +S7.8.4_A7.2_T6: FAIL_OK +S7.8.4_A4.3_T1: FAIL_OK +S7.8.4_A6.2_T1: FAIL_OK +S7.8.4_A4.3_T3: FAIL_OK +S7.8.4_A7.2_T1: FAIL_OK +S7.8.4_A6.4_T1: FAIL_OK +S7.8.4_A7.2_T3: FAIL_OK +S7.8.4_A7.4_T1: FAIL_OK +S7.8.4_A4.3_T5: FAIL_OK +S7.8.4_A7.2_T5: FAIL_OK +S7.8.4_A4.3_T1: FAIL_OK +S7.8.4_A6.2_T1: FAIL_OK +S7.8.4_A4.3_T3: FAIL_OK +S7.8.4_A7.2_T1: FAIL_OK +S7.8.4_A6.4_T1: FAIL_OK +S7.8.4_A7.2_T3: FAIL_OK +S7.8.4_A7.4_T1: FAIL_OK +S7.8.4_A4.3_T5: FAIL_OK +S7.8.4_A7.2_T5: FAIL_OK + +# We allow some keywords to be used as identifiers +S7.5.3_A1.17: FAIL_OK +S7.5.3_A1.26: FAIL_OK +S7.5.3_A1.18: FAIL_OK +S7.5.3_A1.27: FAIL_OK +S7.5.3_A1.28: FAIL_OK +S7.5.3_A1.19: FAIL_OK +S7.5.3_A1.29: FAIL_OK +S7.5.3_A1.1: FAIL_OK +S7.5.3_A1.2: FAIL_OK +S7.5.3_A1.3: FAIL_OK +S7.5.3_A1.4: FAIL_OK +S7.5.3_A1.5: FAIL_OK +S7.5.3_A1.8: FAIL_OK +S7.5.3_A1.9: FAIL_OK +S7.5.3_A1.10: FAIL_OK +S7.5.3_A1.11: FAIL_OK +S7.5.3_A1.21: FAIL_OK +S7.5.3_A1.12: FAIL_OK +S7.5.3_A1.30: FAIL_OK +S7.5.3_A1.31: FAIL_OK +S7.5.3_A1.13: FAIL_OK +S7.5.3_A1.22: FAIL_OK +S7.5.3_A1.23: FAIL_OK +S7.5.3_A1.14: FAIL_OK +S7.5.3_A1.15: FAIL_OK +S7.5.3_A1.24: FAIL_OK +S7.5.3_A1.25: FAIL_OK +S7.5.3_A1.16: FAIL_OK + +# This checks for non-262 behavior +S12.6.4_A14_T1: PASS || FAIL_OK +S12.6.4_R1: PASS || FAIL_OK +S12.6.4_R2: PASS || FAIL_OK +S8.4_D2.1: PASS || FAIL_OK +S8.4_D2.2: PASS || FAIL_OK +S8.4_D2.3: PASS || FAIL_OK +S8.4_D2.4: PASS || FAIL_OK +S8.4_D2.5: PASS || FAIL_OK +S8.4_D2.6: PASS || FAIL_OK +S8.4_D2.7: PASS || FAIL_OK +S8.4_D1.1: PASS || FAIL_OK +S13.2_D1.2: PASS || FAIL_OK +S11.4.3_D1.2: PASS || FAIL_OK +S7.6_D1: PASS || FAIL_OK +S7.6_D2: PASS || FAIL_OK +S15.1.2.2_D1.2: PASS || FAIL_OK +S13_D1_T1: PASS || FAIL_OK +S14_D4_T3: PASS || FAIL_OK +S14_D7: PASS || FAIL_OK +S15.5.4.11_D1.1_T2: PASS || FAIL_OK +S15.5.4.11_D1.1_T4: PASS || FAIL_OK +S15.5.2_D2: PASS || FAIL_OK +S15.5.4.11_D1.1_T1: PASS || FAIL_OK +S15.5.4.11_D1.1_T3: PASS || FAIL_OK +S12.6.4_D1: PASS || FAIL_OK + +# We deliberately don't throw type errors when iterating through the +# undefined object +S9.9_A1: FAIL_OK +S9.9_A2: FAIL_OK + +# We allow function declarations within statements +S12.5_A9_T1: FAIL_OK +S12.5_A9_T2: FAIL_OK +# S12.6.2_A13_T3: FAIL_OK +# S12.5_A9_T3: FAIL_OK +# S12.6.1_A13_T3: FAIL_OK +S12.1_A1: FAIL_OK +S12.6.2_A13_T1: FAIL_OK +S12.6.2_A13_T2: FAIL_OK +S12.6.1_A13_T1: FAIL_OK +S12.6.1_A13_T2: FAIL_OK +S12.6.4_A13_T1: FAIL_OK +S12.6.4_A13_T2: FAIL_OK +#S12.6.4_A13_T3: FAIL_OK +S15.3.4.2_A1_T1: FAIL_OK + +# Linux and Mac defaults to extended 80 bit floating point format in the FPU. +# We follow the other major JS engines by keeping this default. +S8.5_A2.2: PASS, FAIL if $system == linux, FAIL if $system == macos +S8.5_A2.1: PASS, FAIL if $system == linux, FAIL if $system == macos + +##################### SKIPPED TESTS ##################### + +# These tests take a looong time to run in debug mode. +S15.1.3.2_A2.5_T1: PASS, SKIP if $mode == debug +S15.1.3.1_A2.5_T1: PASS, SKIP if $mode == debug + + +# These tests fail because we had to add bugs to be compatible with JSC. See +# http://code.google.com/p/chromium/issues/detail?id=1717 +S15.4.4_A1.1_T2: FAIL_OK +S15.5.4.1_A1_T2: FAIL_OK +S15.5.4_A1: FAIL_OK +S15.5.4_A3: FAIL_OK +S15.9.3.1_A3_T1.1: FAIL_OK +S15.9.3.1_A3_T2.1: FAIL_OK +S15.9.3.1_A3_T3.1: FAIL_OK +S15.9.3.1_A3_T4.1: FAIL_OK +S15.9.3.1_A3_T5.1: FAIL_OK +S15.9.3.1_A3_T6.1: FAIL_OK +S15.9.3.2_A3_T1.1: FAIL_OK +S15.9.5.10_A1_T2: FAIL_OK +S15.9.5.11_A1_T2: FAIL_OK +S15.9.5.12_A1_T2: FAIL_OK +S15.9.5.13_A1_T2: FAIL_OK +S15.9.5.14_A1_T2: FAIL_OK +S15.9.5.15_A1_T2: FAIL_OK +S15.9.5.16_A1_T2: FAIL_OK +S15.9.5.17_A1_T2: FAIL_OK +S15.9.5.18_A1_T2: FAIL_OK +S15.9.5.19_A1_T2: FAIL_OK +S15.9.5.20_A1_T2: FAIL_OK +S15.9.5.21_A1_T2: FAIL_OK +S15.9.5.22_A1_T2: FAIL_OK +S15.9.5.23_A1_T2: FAIL_OK +S15.9.5.24_A1_T2: FAIL_OK +S15.9.5.25_A1_T2: FAIL_OK +S15.9.5.26_A1_T2: FAIL_OK +S15.9.5.27_A1_T2: FAIL_OK +S15.9.5.28_A1_T2: FAIL_OK +S15.9.5.29_A1_T2: FAIL_OK +S15.9.5.2_A1_T2: FAIL_OK +S15.9.5.30_A1_T2: FAIL_OK +S15.9.5.31_A1_T2: FAIL_OK +S15.9.5.32_A1_T2: FAIL_OK +S15.9.5.33_A1_T2: FAIL_OK +S15.9.5.34_A1_T2: FAIL_OK +S15.9.5.35_A1_T2: FAIL_OK +S15.9.5.36_A1_T2: FAIL_OK +S15.9.5.37_A1_T2: FAIL_OK +S15.9.5.38_A1_T2: FAIL_OK +S15.9.5.39_A1_T2: FAIL_OK +S15.9.5.3_A1_T2: FAIL_OK +S15.9.5.40_A1_T2: FAIL_OK +S15.9.5.41_A1_T2: FAIL_OK +S15.9.5.42_A1_T2: FAIL_OK +S15.9.5.4_A1_T2: FAIL_OK +S15.9.5.5_A1_T2: FAIL_OK +S15.9.5.6_A1_T2: FAIL_OK +S15.9.5.7_A1_T2: FAIL_OK +S15.9.5.8_A1_T2: FAIL_OK +S15.9.5.9_A1_T2: FAIL_OK + +# Regexps have type "function", not "object". +S11.4.3_A3.6: FAIL_OK +S15.10.7_A3_T2: FAIL_OK +S15.10.7_A3_T1: FAIL_OK ======================================= --- /dev/null +++ /branches/bleeding_edge/test/sputnik/testcfg.py Mon Nov 9 01:31:57 2009 @@ -0,0 +1,118 @@ +# Copyright 2009 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +from os.path import join, exists +import sys +import test +import time + + +class SputnikTestCase(test.TestCase): + + def __init__(self, case, path, context, mode): + super(SputnikTestCase, self).__init__(context, path) + self.case = case + self.mode = mode + self.tmpfile = None + self.source = None + + def IsNegative(self): + return '@negative' in self.GetSource() + + def IsFailureOutput(self, output): + if output.exit_code != 0: + return True + out = output.stdout + return "SputnikError" in out + + def BeforeRun(self): + self.tmpfile = sputnik.TempFile(suffix='.js', prefix='sputnik-', text=True) + self.tmpfile.Write(self.GetSource()) + self.tmpfile.Close() + + def AfterRun(self): + self.tmpfile.Dispose() + self.tmpfile = None + + def GetCommand(self): + result = [self.context.GetVm(self.mode)] + result.append(self.tmpfile.name) + return result + + def GetLabel(self): + return "%s sputnik %s" % (self.mode, self.GetName()) + + def GetName(self): + return self.path[-1] + + def GetSource(self): + if not self.source: + self.source = self.case.GetSource() + return self.source + +class SputnikTestConfiguration(test.TestConfiguration): + + def __init__(self, context, root): + super(SputnikTestConfiguration, self).__init__(context, root) + + def ListTests(self, current_path, path, mode): + # Import the sputnik test runner script as a module + testroot = join(self.root, 'sputniktests') + modroot = join(testroot, 'tools') + sys.path.append(modroot) + import sputnik + globals()['sputnik'] = sputnik + test_suite = sputnik.TestSuite(testroot) + test_suite.Validate() + tests = test_suite.EnumerateTests([]) + result = [] + for test in tests: + full_path = current_path + [test.GetPath()[-1]] + if self.Contains(path, full_path): + case = SputnikTestCase(test, full_path, self.context, mode) + result.append(case) + return result + + def GetBuildRequirements(self): + return ['sample', 'sample=shell'] + + def GetTestStatus(self, sections, defs): + status_file = join(self.root, 'sputnik.status') + if exists(status_file): + test.ReadConfigurationInto(status_file, sections, defs) + + +def GetConfiguration(context, root): + checkout = join(root, 'sputniktests') + if not exists(checkout): + print "No checkout of sputniktests found. Check out the tests under" + print "v8/test/sputnik using:" + print " svn co http://sputniktests.googlecode.com/svn/trunk/ sputniktests" + sys.exit(0) + return SputnikTestConfiguration(context, root) ======================================= --- /branches/bleeding_edge/tools/test.py Wed Oct 28 02:38:09 2009 +++ /branches/bleeding_edge/tools/test.py Mon Nov 9 01:31:57 2009 @@ -359,8 +359,19 @@ self.Cleanup() return TestOutput(self, full_command, output) + def BeforeRun(self): + pass + + def AfterRun(self): + pass + def Run(self): - return self.RunCommand(self.GetCommand()) + self.BeforeRun() + try: + result = self.RunCommand(self.GetCommand()) + finally: + self.AfterRun() + return result def Cleanup(self): return --~--~---------~--~----~------------~-------~--~----~ v8-dev mailing list [email protected] http://groups.google.com/group/v8-dev -~----------~----~----~----~------~----~------~--~---
