Repository : ssh://darcs.haskell.org//srv/darcs/testsuite On branch : master
http://hackage.haskell.org/trac/ghc/changeset/d183af39b5e1f74f28ebfe833151d42c50814f26 >--------------------------------------------------------------- commit d183af39b5e1f74f28ebfe833151d42c50814f26 Author: Simon Marlow <[email protected]> Date: Wed Feb 15 14:33:36 2012 +0000 Allow perf bounds to be specified as base + percentage deviation This makes it much easier to update the bounds. Instead of coming up with a suitable (min,max) pair, you just give e.g. (base, 10) to allow 10% deviation from the base figure, which can be pasted from the error. e.g. previously: - # expected value: 458700632 (amd64/Linux): - if_wordsize(64, - compiler_stats_num_field('bytes allocated', 440000000, - 480000000)), now: + if_wordsize(64, # sample from amd64/Linux 15/2/2012 + compiler_stats_range_field('bytes allocated', 360243576, 10)), Note: use stats_range_field rather than stats_num_field. I left support for the old way for now so that we can do a gradual migration. (next I suppose we should make it so that 'make accept' works for perf tests, but that's for another day) >--------------------------------------------------------------- driver/testglobals.py | 12 ++++++++++++ driver/testlib.py | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/driver/testglobals.py b/driver/testglobals.py index 6e079a7..9a4bfba 100644 --- a/driver/testglobals.py +++ b/driver/testglobals.py @@ -201,6 +201,18 @@ class TestOptions: # Elements of these lists should be things like # ('bytes allocated', # 9300000000, + # 10) + # To allow a 10% deviation from 9300000000. + self.compiler_stats_range_fields = {} + self.stats_range_fields = {} + + # TODO: deprecate this in favour of compiler_stats_range_fields + # + # which -t numeric fields do we want to look at, and what bounds must + # they fall within? + # Elements of these lists should be things like + # ('bytes allocated', + # 9300000000, # 9400000000) self.compiler_stats_num_fields = {} self.stats_num_fields = {} diff --git a/driver/testlib.py b/driver/testlib.py index f877edb..cb4a765 100644 --- a/driver/testlib.py +++ b/driver/testlib.py @@ -253,6 +253,24 @@ def _compiler_stats_num_field( opts, f, x, y ): # ----- +def stats_range_field( field, min, max ): + return lambda opts, f=field, x=min, y=max: _stats_range_field(opts, f, x, y); + +def _stats_range_field( opts, f, x, y ): + # copy the dictionary, as the config gets shared between all tests + opts.stats_range_fields = opts.stats_range_fields.copy() + opts.stats_range_fields[f] = (x, y) + +def compiler_stats_range_field( field, min, max ): + return lambda opts, f=field, x=min, y=max: _compiler_stats_range_field(opts, f, x, y); + +def _compiler_stats_range_field( opts, f, x, y ): + # copy the dictionary, as the config gets shared between all tests + opts.compiler_stats_range_fields = opts.compiler_stats_range_fields.copy() + opts.compiler_stats_range_fields[f] = (x, y) + +# ----- + def skip_if_no_ghci(opts): if not ('ghci' in config.run_ways): opts.skip = 1 @@ -949,13 +967,34 @@ def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ): # ----------------------------------------------------------------------------- # Check -t stats info -def checkStats(stats_file, num_fields): +def checkStats(stats_file, range_fields, num_fields): result = passed() if len(num_fields) > 0: f = open(in_testdir(stats_file)) contents = f.read() f.close() + for (field, (expected, dev)) in range_fields.items(): + m = re.search('\("' + field + '", "([0-9]+)"\)', contents) + if m == None: + print 'Failed to find field: ', field + result = failBecause('no such stats field') + val = int(m.group(1)) + + min = expected * ((100 - float(dev))/100); + max = expected * ((100 + float(dev))/100); + + if val < min: + print field, val, 'is more than ' + repr(dev) + '%' + print 'less than the exepected value', expected + print 'If this is because you have improved GHC, please' + print 'update the test so that GHC doesn\'t regress again' + result = failBecause('stat too good') + if val > max: + print field, val, 'is more than ' + repr(dev) + '% greater than the expected value,', expected, max + result = failBecause('stat not good enough') + + # ToDo: remove all uses of this, and delete it for (field, (min, max)) in num_fields.items(): m = re.search('\("' + field + '", "([0-9]+)"\)', contents) if m == None: @@ -1058,7 +1097,8 @@ def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, # ToDo: if the sub-shell was killed by ^C, then exit - statsResult = checkStats(stats_file, opts.compiler_stats_num_fields) + statsResult = checkStats(stats_file, opts.compiler_stats_range_fields + , opts.compiler_stats_num_fields) if badResult(statsResult): return statsResult @@ -1149,7 +1189,8 @@ def simple_run( name, way, prog, args ): if check_prof and not check_prof_ok(name): return failBecause('bad profile') - return checkStats(stats_file, opts.stats_num_fields) + return checkStats(stats_file, opts.stats_range_fields + , opts.stats_num_fields) def rts_flags(way): if (way == ''): _______________________________________________ Cvs-ghc mailing list [email protected] http://www.haskell.org/mailman/listinfo/cvs-ghc
