Re: [Qemu-devel] [PATCH v3 3/8] iotests: Allow skipping test cases
On 19/08/2019 23:18, Max Reitz wrote: > case_notrun() does not actually skip the current test case. It just > adds a "notrun" note and then returns to the caller, who manually has to > skip the test. Generally, skipping a test case is as simple as > returning from the current function, but not always: For example, this > model does not allow skipping tests already in the setUp() function. > > Thus, add a QMPTestCase.case_skip() function that invokes case_notrun() > and then self.skipTest(). To make this work, we need to filter the > information on how many test cases were skipped from the unittest > output. > > Signed-off-by: Max Reitz > --- > tests/qemu-iotests/iotests.py | 21 ++--- > 1 file changed, 18 insertions(+), 3 deletions(-) > > diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py > index 84438e837c..2f53baf633 100644 > --- a/tests/qemu-iotests/iotests.py > +++ b/tests/qemu-iotests/iotests.py > @@ -802,6 +802,11 @@ class QMPTestCase(unittest.TestCase): > return self.pause_wait(job_id) > return result > > +def case_skip(self, reason): > +'''Skip this test case''' > +case_notrun(reason) > +self.skipTest(reason) > + > > def notrun(reason): > '''Skip this test suite''' > @@ -813,7 +818,10 @@ def notrun(reason): > sys.exit(0) > > def case_notrun(reason): > -'''Skip this test case''' > +'''Mark this test case as not having been run, but do not actually > +skip it; that is left to the caller. See QMPTestCase.case_skip() The clause "do not actually skip it" sounds like a prescription. I would like the comment to be clearer for a reader that the method is a notifier only. Andrey > +for a variant that actually skips the current test case.''' > + > # Each test in qemu-iotests has a number ("seq") > seq = os.path.basename(sys.argv[0]) > > @@ -904,8 +912,15 @@ def execute_unittest(output, verbosity, debug): > unittest.main(testRunner=runner) > finally: > if not debug: > -sys.stderr.write(re.sub(r'Ran (\d+) tests? in [\d.]+s', > -r'Ran \1 tests', output.getvalue())) > +out = output.getvalue() > +out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', > out) > + > +# Hide skipped tests from the reference output > +out = re.sub(r'OK \(skipped=\d+\)', 'OK', out) > +out_first_line, out_rest = out.split('\n', 1) > +out = out_first_line.replace('s', '.') + '\n' + out_rest > + > +sys.stderr.write(out) > > def execute_test(test_function=None, >supported_fmts=[], supported_oses=['linux'], > Reviewed-by: Andrey Shinkevich -- With the best regards, Andrey Shinkevich
Re: [Qemu-devel] [PATCH v3 3/8] iotests: Allow skipping test cases
On 8/19/19 4:18 PM, Max Reitz wrote: > case_notrun() does not actually skip the current test case. It just > adds a "notrun" note and then returns to the caller, who manually has to > skip the test. Generally, skipping a test case is as simple as > returning from the current function, but not always: For example, this > model does not allow skipping tests already in the setUp() function. > > Thus, add a QMPTestCase.case_skip() function that invokes case_notrun() > and then self.skipTest(). To make this work, we need to filter the > information on how many test cases were skipped from the unittest > output. > > Signed-off-by: Max Reitz Hm, didn't someone else send a patch like this recently? Ah, yes: [Qemu-block] [PATCH v5 3/6] iotests: Add casenotrun report to bash tests Oh, theirs is for bash. Moving along. > --- > tests/qemu-iotests/iotests.py | 21 ++--- > 1 file changed, 18 insertions(+), 3 deletions(-) > > diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py > index 84438e837c..2f53baf633 100644 > --- a/tests/qemu-iotests/iotests.py > +++ b/tests/qemu-iotests/iotests.py > @@ -802,6 +802,11 @@ class QMPTestCase(unittest.TestCase): > return self.pause_wait(job_id) > return result > > +def case_skip(self, reason): > +'''Skip this test case''' > +case_notrun(reason) > +self.skipTest(reason) > + > > def notrun(reason): > '''Skip this test suite''' > @@ -813,7 +818,10 @@ def notrun(reason): > sys.exit(0) > > def case_notrun(reason): > -'''Skip this test case''' > +'''Mark this test case as not having been run, but do not actually > +skip it; that is left to the caller. See QMPTestCase.case_skip() > +for a variant that actually skips the current test case.''' > + > # Each test in qemu-iotests has a number ("seq") > seq = os.path.basename(sys.argv[0]) > > @@ -904,8 +912,15 @@ def execute_unittest(output, verbosity, debug): > unittest.main(testRunner=runner) > finally: > if not debug: > -sys.stderr.write(re.sub(r'Ran (\d+) tests? in [\d.]+s', > -r'Ran \1 tests', output.getvalue())) > +out = output.getvalue() > +out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', > out) > + > +# Hide skipped tests from the reference output > +out = re.sub(r'OK \(skipped=\d+\)', 'OK', out) > +out_first_line, out_rest = out.split('\n', 1) > +out = out_first_line.replace('s', '.') + '\n' + out_rest > + > +sys.stderr.write(out) > > def execute_test(test_function=None, > supported_fmts=[], supported_oses=['linux'], > okey dokey. Reviewed-by: John Snow
[Qemu-devel] [PATCH v3 3/8] iotests: Allow skipping test cases
case_notrun() does not actually skip the current test case. It just adds a "notrun" note and then returns to the caller, who manually has to skip the test. Generally, skipping a test case is as simple as returning from the current function, but not always: For example, this model does not allow skipping tests already in the setUp() function. Thus, add a QMPTestCase.case_skip() function that invokes case_notrun() and then self.skipTest(). To make this work, we need to filter the information on how many test cases were skipped from the unittest output. Signed-off-by: Max Reitz --- tests/qemu-iotests/iotests.py | 21 ++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 84438e837c..2f53baf633 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -802,6 +802,11 @@ class QMPTestCase(unittest.TestCase): return self.pause_wait(job_id) return result +def case_skip(self, reason): +'''Skip this test case''' +case_notrun(reason) +self.skipTest(reason) + def notrun(reason): '''Skip this test suite''' @@ -813,7 +818,10 @@ def notrun(reason): sys.exit(0) def case_notrun(reason): -'''Skip this test case''' +'''Mark this test case as not having been run, but do not actually +skip it; that is left to the caller. See QMPTestCase.case_skip() +for a variant that actually skips the current test case.''' + # Each test in qemu-iotests has a number ("seq") seq = os.path.basename(sys.argv[0]) @@ -904,8 +912,15 @@ def execute_unittest(output, verbosity, debug): unittest.main(testRunner=runner) finally: if not debug: -sys.stderr.write(re.sub(r'Ran (\d+) tests? in [\d.]+s', -r'Ran \1 tests', output.getvalue())) +out = output.getvalue() +out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', out) + +# Hide skipped tests from the reference output +out = re.sub(r'OK \(skipped=\d+\)', 'OK', out) +out_first_line, out_rest = out.split('\n', 1) +out = out_first_line.replace('s', '.') + '\n' + out_rest + +sys.stderr.write(out) def execute_test(test_function=None, supported_fmts=[], supported_oses=['linux'], -- 2.21.0