-1 to this change. It is not right to arbitrarily change test data this way.
Changes to test data should only be made if there is a genuine reason to do so e.g. due to a mistake in the original data or a planned change in the API. If a test starts failing, the most likely reason is a bug due to a change in the code, not a bug in the test data. (There are of course exceptions, e.g. where a test is dependent on timing or other environmental factors) On Wed, 2 Jun 2021 at 13:23, <[email protected]> wrote: > > This is an automated email from the ASF dual-hosted git repository. > > humbedooh pushed a commit to branch master > in repository > https://gitbox.apache.org/repos/asf/incubator-ponymail-unit-tests.git > > commit fb8025f06013811eed9e2ea3e4f7946dde1eea16 > Author: Daniel Gruno <[email protected]> > AuthorDate: Wed Jun 2 14:02:14 2021 +0200 > > --dropin for quickly reconfiguring unit tests on the fly > --- > runall.py | 7 ++++++- > tests/test-generators.py | 11 +++++++++++ > 2 files changed, 17 insertions(+), 1 deletion(-) > > diff --git a/runall.py b/runall.py > index 556eec8..4b735cb 100755 > --- a/runall.py > +++ b/runall.py > @@ -20,7 +20,10 @@ if __name__ == '__main__': > parser.add_argument('--gtype', dest='gtype', type=str, nargs='+', > help="Run only specified generators (medium, > cluster, dkim, full, etc)") > parser.add_argument('--yamldir', dest='yamldir', type=str, > action='store', > - help="Load yaml specs from alternate directory") > + help="Load yaml specs from alternate directory") > + parser.add_argument('--dropin', dest='dropin', type=str, action='store', > + help="If set to a specific generator name, replaces > its unit test results with the current " > + "output in the yaml tests") > parser.add_argument('--nomboxo', dest = 'nomboxo', action='store_true', > help = 'Skip Mboxo processing') > parser.add_argument('--fof', dest='failonfail', action='store_true', > @@ -69,6 +72,8 @@ if __name__ == '__main__': > if args.gtype and test_type == 'generators': > cliargs.append('--generators') > cliargs.extend(args.gtype) > + if args.dropin: > + cliargs.extend(['--dropin', args.dropin]) > if args.skipnodate and test_type == 'generators': > cliargs.append('--skipnodate') > rv = subprocess.check_output(cliargs, env=env) > diff --git a/tests/test-generators.py b/tests/test-generators.py > index aaa3daa..670d732 100755 > --- a/tests/test-generators.py > +++ b/tests/test-generators.py > @@ -10,6 +10,7 @@ import yaml > import argparse > import collections > import interfacer > +import time > import email.utils > > parse_html = False > @@ -150,9 +151,17 @@ def run_tests(args): > errors += 1 > sys.stderr.write("""[FAIL] %s, index %2u: Expected > '%s', got '%s'!\n""" % > (gen_type, key, expected, actual)) > + if args.dropin and gen_type == args.dropin: > + if expected != actual: > + test['generated'] = actual > + else: > + test['alternate'] = actual > else: > print("[PASS] %s index %u" % (gen_type, key)) > mboxfiles = [] # reset for the next set of tests > + if args.dropin and errors: > + sys.stderr.write("Writing replacement yaml as --dropin was > specified\n") > + yaml.safe_dump(yml, open(args.load, "w"), sort_keys=False) > print("[DONE] %u tests run, %u failed." % (tests_run, errors)) > if errors: > sys.exit(-1) > @@ -174,6 +183,8 @@ def main(): > help="Root directory of Apache Pony Mail") > parser.add_argument('--nomboxo', dest = 'nomboxo', action='store_true', > help = 'Skip Mboxo processing') > + parser.add_argument('--dropin', dest = 'dropin', type=str, > + help = 'Perform drop-in replacement of unit test > results for the specified generator type [devs only!]') > parser.add_argument('--skipnodate', dest = 'skipnodate', > action='store_true', > help = 'Skip emails with no Date: header (useful for > medium generator tests)') > args = parser.parse_args()
