On Monday, February 17, 2014 03:25:22 PM Ilia Mirkin wrote: > On Mon, Feb 17, 2014 at 2:45 PM, Dylan Baker <[email protected]> wrote: > > It would be an unwelcome surprise if some test returns 'fails' or 'pas', > > so rather than allowing such a thing to happen, assert that the result > > is actually viable. > > > > Signed-off-by: Dylan Baker <[email protected]> > > --- > > > > framework/core.py | 2 +- > > framework/log.py | 10 ++++-- > > framework/tests/log_tests.py | 74 > > +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 77 > > insertions(+), 9 deletions(-) > > > > diff --git a/framework/core.py b/framework/core.py > > index fc59e3c..f6ae80f 100644 > > --- a/framework/core.py > > +++ b/framework/core.py > > > > @@ -435,7 +435,6 @@ class Test(object): > > ''' > > > > log_current = log.get_current() > > > > - test_result = None > > > > # Run the test > > > > if env.execute: > > try: > > @@ -471,6 +470,7 @@ class Test(object): > > else: > > json_writer.write_dict_item(path, result) > > > > else: > > + test_result = 'dry-run' > > > > log.log() > > > > log.mark_complete(log_current, test_result) > > > > diff --git a/framework/log.py b/framework/log.py > > index 25ecdf5..7cdc940 100644 > > --- a/framework/log.py > > +++ b/framework/log.py > > @@ -21,6 +21,7 @@ > > > > # > > > > import sys > > > > +import collections > > > > from .threads import synchronized_self > > > > @@ -37,7 +38,9 @@ class Log(object): > > self.__running = [] > > self.__generator = (x for x in xrange(self.__total)) > > self.__pad = len(str(self.__total)) > > > > - self.__summary = {} > > + self.__summary_keys = set(['pass', 'fail', 'warn', 'crash', > > 'skip', + 'dmesg-warn', 'dmesg-fail', > > 'dry-run']) + self.__summary = collections.defaultdict(lambda: 0) > > > > def _summary(self): > > return ", ".join("{0}: {1}".format(k, self.__summary[k]) > > > > @@ -67,8 +70,8 @@ class Log(object): > > # increment the number of completed tests > > self.__complete += 1 > > > > - if result: > > - self.__summary[result] = self.__summary.get(result, 0) + 1 > > + assert result in self.__summary_keys > > + self.__summary[result] += 1 > > > > @synchronized_self > > > > def log(self): > > @@ -80,6 +83,7 @@ class Log(object): > > """ > > sys.stdout.write("{0} {1} {2}\r".format( > > > > self._percent(), self._summary(), self._running())) > > > > + > > > > # Need to flush explicitly, otherwise it all gets buffered > > without a > > # newline. > > sys.stdout.flush() > > > > diff --git a/framework/tests/log_tests.py b/framework/tests/log_tests.py > > index 5f0640f..09d99e4 100644 > > --- a/framework/tests/log_tests.py > > +++ b/framework/tests/log_tests.py > > @@ -20,10 +20,14 @@ > > > > """ Module provides tests for log.py module """ > > > > +import sys > > +import itertools > > > > from types import * # This is a special * safe module > > import nose.tools as nt > > from framework.log import Log > > > > +valid_statuses = ('pass', 'fail', 'crash', 'warn', 'dmesg-warn', > > + 'dmesg-fail', 'skip', 'dry-run') > > > > def test_initialize_log(): > > """ Test that Log initializes with """ > > > > @@ -63,11 +67,6 @@ def check_mark_complete_increment_summary(stat): > > def test_mark_complete_increment_summary(): > > """ Generator that creates tests for self.__summary """ > > > > - > > - > > - valid_statuses = ('pass', 'fail', 'crash', 'warn', 'dmesg-warn', > > - 'dmesg-fail', 'skip') > > - > > > > yieldable = check_mark_complete_increment_summary > > > > for stat in valid_statuses: > > @@ -83,3 +82,68 @@ def test_mark_complete_removes_complete(): > > log.mark_complete(ret, 'pass') > > nt.assert_not_in(ret, log._Log__running, > > > > msg="Running tests not removed from running list") > > > > + > > + > > [email protected](AssertionError) > > +def check_mark_complete_incrment_summary_bad(): > > + log = Log(100) > > + ret = log.get_current() > > + log.mark_complete(ret, 'fails') > > + > > + > > +def check_output(stat): > > + """ Test that dry output returns the expected values """ > > + log = Log(100) > > + > > + for x in stat: > > + ret = log.get_current() > > + log.mark_complete(ret, x) > > + > > + # Print to stdout, and then capture it, the capture absolutely needs > > to + # imediatly after log.log() > > + log.log() > > + output = sys.stdout.getvalue().strip() # sys.stdout is a StringIO in > > nose + > > + # Create a count of the statuses in alphabetical order. What we need > > to end + # up with is something like: 'crash': 2, 'skip': 1 > > + s = [] > > + for x in sorted(set(stat)): > > + s.append("{0}: {1}".format(x, stat.count(x))) > > + > > + expected = '[{0}/100] {1} Running Test(s):'.format( > > + str(len(stat)).zfill(3), > > + ", ".join(s)) > > This is a dangerous sort of test. Any change in the output will make > it fail, and I can pretty much guarantee that people won't run the > tests. I wonder if it'd be better to test for the bits you expect, > e.g. n/y and that each status: val is in there?
Hmm, that is a good point. I'll have to think about that. > > > + > > + print expected > > + nt.assert_equal(output, expected, > > + msg="Did not return execpted output: " + expected) > > expected. (dyslexics, untie!) Ah yes, I really need to just leave spell checking on in vim... > > > + > > + > > +def gen_test_output(): > > + """ Generate a set of tests for checking __summary output > > + > > + This generates severl hundered tests > > + > > + """ > > + yieldable = check_output > > + > > + # test dry run seperately, we don't want to test it mixed in with > > other + # statuses, since it shouldn't be mixed > > + yieldable.description = "Test output for status combinations: > > dry-run" > > + yield yieldable, ['dry-run'] > > + > > + yieldable.description = "Test output for status combinations: dry-run > > dry-run" + yield yieldable, ['dry-run', 'dry-run'] > > + > > + yieldable.description = "Test output for status combinations: dry-run > > dry-run dry-run" + yield yieldable, ['dry-run', 'dry-run', 'dry-run'] > > + > > + # All combinations except dry-run up to 3 in length. We could > > generate > > + # more, but it quickly becomes overwhelming, and the usefulness is > > + # uncertain > > + for r in xrange(1, 4): > > + for stat in itertools.product(valid_statuses[:-1], > > + repeat=r): > > + yieldable.description = "Test output for status: " + \ > > + " ".join([str(x) for x in stat]) > > The [ ] are unnecessary. Generator expressions have been around since > 2.4, iirc. (With the additional benefit that x won't leak out into the > function scope... not that one would really care.) I'll make that change. I didn't think that list comprehensions leaked, since they're supposed to be usable anywhere map() and filter() is used without any additional side effects. Maybe I'm wrong though. > > > + yield yieldable, stat > > -- > > 1.8.5.5
signature.asc
Description: This is a digitally signed message part.
_______________________________________________ Piglit mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/piglit
