A Dilluns, 12 de setembre de 2011, Carlos Garcia Campos vàreu escriure:
> Hi all,
Hi
> as you all probably know, our current regression test suite are some
> custom shell scripts and a lot of pdf files that Albert has collected
> for some years. During the last few days I have been working on
> improving the regression test, converting Albert's scripts into a
> small python program to allow everybody to run their own tests. We are
> not going to upload pdfs to the repo, so we still depend on Albert
> to make sure a change doesn't actually introduce regressions.
>
> SNIP
>
> There are still things to do, like adding an option to provide documents
> to skip,
Yeah, the documents to skip is important, i have a few files here that cause
infinite loops because of errors in things we do not control (like openjpeg)
and i need to be able of ignoring those.
It would be good to have some sense of how the test is progressing, the
problem at the moment is that files are tested in "random" order, so can not
use the current file to know how much is left. It would be good either to sort
the files or to get a "1 of 1000" text to know how much is left
I am not sure the "Test run in X seconds" works, see here
$ time ~/devel/poppler/regtest/poppler-regtest --utils-
dir=/home/tsdgeos/devel/poppler/build-old/utils/ --backends=splash run-tests
--refs-dir=./refs --out-dir=./out .
Testing './alumnes_normativa.pdf' using splash backend: PASS
Total 1 tests
1 tests passed (100.00%)
Tests run in 0 seconds
real 0m1.101s
user 0m1.690s
sys 0m0.097s
That 0 seconds seem wrong either to the 1.1 real or 1.6 user seconds
> html backend support, and I'm sure there are a lot of bugs
> too. So, feel free to comment, or even better provide patches :-)
Sending in a patch to make it work with Python 3, seems to work fine with
Python 2 too. Basically uses the "new" syntax for print and exceptions and
opens a few files in binary mode as they are not plaintext.
Albert
>
> --
> Carlos Garcia Campos
> PGP key: http://pgp.mit.edu:11371/pks/lookup?op=get&search=0x523E6462
diff -rub regtest/backends/__init__.py regtestmio/backends/__init__.py
--- regtest/backends/__init__.py 2011-09-13 12:25:14.000000000 +0200
+++ regtestmio/backends/__init__.py 2011-09-13 12:20:12.000000000 +0200
@@ -46,7 +46,7 @@
if not entry.startswith(self._name) or entry.endswith('.md5'):
continue
ref_path = os.path.join(refs_path, entry)
- f = open(ref_path, 'r')
+ f = open(ref_path, 'rb')
md5_file.write("%s %s\n" % (md5(f.read()).hexdigest(), ref_path))
f.close()
if delete_refs:
@@ -66,21 +66,21 @@
basename = os.path.basename(ref_path)
if not basename in tests:
retval = False
- print "%s found in md5 ref file but missing in output dir %s" % (basename, out_path)
+ print("%s found in md5 ref file but missing in output dir %s" % (basename, out_path))
continue
result_path = os.path.join(out_path, basename)
- f = open(result_path, 'r')
+ f = open(result_path, 'rb')
matched = md5sum == md5(f.read()).hexdigest()
f.close()
if matched:
if remove_results:
os.remove(result_path)
else:
- print "Differences found in %s" % (basename)
+ print("Differences found in %s" % (basename))
if create_diffs:
if not os.path.exists(ref_path):
- print "Reference file %s not found, skipping diff for %s" % (ref_path, result_path)
+ print("Reference file %s not found, skipping diff for %s" % (ref_path, result_path))
else:
try:
self._create_diff(ref_path, result_path)
@@ -115,7 +115,7 @@
def __create_stderr_file(self, stderr, out_path):
if not stderr:
return
- stderr_file = open(out_path + '.stderr', 'w')
+ stderr_file = open(out_path + '.stderr', 'wb')
stderr_file.write(stderr)
stderr_file.close()
diff -rub regtest/commands/create-refs.py regtestmio/commands/create-refs.py
--- regtest/commands/create-refs.py 2011-09-13 12:25:14.000000000 +0200
+++ regtestmio/commands/create-refs.py 2011-09-13 11:58:46.000000000 +0200
@@ -60,6 +60,6 @@
refs.create_refs()
else:
refs.create_refs_for_file(os.path.basename(doc))
- print "Refs created in %s" % (t.elapsed_str())
+ print("Refs created in %s" % (t.elapsed_str()))
register_command('create-refs', CreateRefs)
diff -rub regtest/commands/__init__.py regtestmio/commands/__init__.py
--- regtest/commands/__init__.py 2011-09-13 12:25:14.000000000 +0200
+++ regtestmio/commands/__init__.py 2011-09-13 11:47:35.000000000 +0200
@@ -83,11 +83,11 @@
except ImportError:
pass
- print "Commands are:"
+ print("Commands are:")
commands = [(x.name, x.description) for x in _commands.values()]
commands.sort()
for name, description in commands:
- print " %-15s %s" % (name, description)
+ print(" %-15s %s" % (name, description))
print
- print "For more information run 'poppler-regtest --help-command <command>'"
+ print("For more information run 'poppler-regtest --help-command <command>'")
diff -rub regtest/commands/run-tests.py regtestmio/commands/run-tests.py
--- regtest/commands/run-tests.py 2011-09-13 12:25:14.000000000 +0200
+++ regtestmio/commands/run-tests.py 2011-09-13 12:18:06.000000000 +0200
@@ -64,6 +64,6 @@
else:
tests.run_test(os.path.basename(doc))
tests.summary()
- print "Tests run in %s" % (t.elapsed_str())
+ print("Tests run in %s" % (t.elapsed_str()))
register_command('run-tests', RunTests)
diff -rub regtest/Config.py regtestmio/Config.py
--- regtest/Config.py 2011-09-13 12:25:14.000000000 +0200
+++ regtestmio/Config.py 2011-09-13 11:48:28.000000000 +0200
@@ -27,6 +27,6 @@
if __name__ == '__main__':
c = Config({'foo' : 25})
- print c.foo
+ print(c.foo)
cc = Config()
- print cc.foo
+ print(cc.foo)
diff -rub regtest/main.py regtestmio/main.py
--- regtest/main.py 2011-09-13 12:25:14.000000000 +0200
+++ regtestmio/main.py 2011-09-13 11:46:35.000000000 +0200
@@ -67,7 +67,7 @@
sys.stderr.write("Unknown command: %s\n" % (args[0]))
commands.print_help()
sys.exit(1)
- except backends.UnknownBackendError, e:
+ except backends.UnknownBackendError as e:
sys.stderr.write(str(e) + "\n")
sys.stdout.write("Backends are: %s\n" % (", ".join([backend.get_name() for backend in backends.get_all_backends()])))
sys.exit(1)
diff -rub regtest/TestReferences.py regtestmio/TestReferences.py
--- regtest/TestReferences.py 2011-09-13 12:25:14.000000000 +0200
+++ regtestmio/TestReferences.py 2011-09-13 11:59:31.000000000 +0200
@@ -30,7 +30,7 @@
try:
os.makedirs(self._refsdir)
- except OSError, e:
+ except OSError as e:
if e.errno != errno.EEXIST:
raise
except:
@@ -40,7 +40,7 @@
refs_path = os.path.join(self._refsdir, filename)
try:
os.makedirs(refs_path)
- except OSError, e:
+ except OSError as e:
if e.errno != errno.EEXIST:
raise
except:
@@ -54,9 +54,9 @@
for backend in backends:
if not self.config.force and backend.has_md5(refs_path):
- print "Checksum file found, skipping '%s' for %s backend" % (doc_path, backend.get_name())
+ print("Checksum file found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
continue
- print "Creating refs for '%s' using %s backend" % (doc_path, backend.get_name())
+ print("Creating refs for '%s' using %s backend" % (doc_path, backend.get_name()))
if backend.create_refs(doc_path, refs_path):
backend.create_checksums(refs_path, self.config.checksums_only)
diff -rub regtest/TestRun.py regtestmio/TestRun.py
--- regtest/TestRun.py 2011-09-13 12:25:14.000000000 +0200
+++ regtestmio/TestRun.py 2011-09-13 12:19:57.000000000 +0200
@@ -40,7 +40,7 @@
try:
os.makedirs(self._outdir);
- except OSError, e:
+ except OSError as e:
if e.errno != errno.EEXIST:
raise
except:
@@ -52,7 +52,7 @@
ref_is_crashed = backend.is_crashed(refs_path)
ref_is_failed = backend.is_failed(refs_path)
if not ref_has_md5 and not ref_is_crashed and not ref_is_failed:
- print "Reference files not found, skipping '%s' for %s backend" % (doc_path, backend.get_name())
+ print("Reference files not found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
return
self._n_tests += 1
@@ -66,40 +66,40 @@
if ref_has_md5 and test_has_md5:
if backend.compare_checksums(refs_path, test_path, not self.config.keep_results, self.config.create_diffs):
# FIXME: remove dir if it's empty?
- print "PASS"
+ print("PASS")
self._n_passed += 1
else:
- print "FAIL"
+ print("FAIL")
self._failed.append("%s (%s)" % (doc_path, backend.get_name()))
return
elif test_has_md5:
if ref_is_crashed:
- print "DOES NOT CRASH"
+ print("DOES NOT CRASH")
elif ref_is_failed:
- print "DOES NOT FAIL"
+ print("DOES NOT FAIL")
return
test_is_crashed = backend.is_crashed(test_path)
if ref_is_crashed and test_is_crashed:
- print "PASS (Expected crash)"
+ print("PASS (Expected crash)")
self._n_passed += 1
return
test_is_failed = backend.is_failed(test_path)
if ref_is_failed and test_is_failed:
# FIXME: compare status errors
- print "PASS (Expected fail with status error %d)" % (test_is_failed)
+ print("PASS (Expected fail with status error %d)" % (test_is_failed))
self._n_passed += 1
return
if test_is_crashed:
- print "CRASH"
+ print("CRASH")
self._crashed.append("%s (%s)" % (doc_path, backend.get_name()))
return
if test_is_failed:
- print "FAIL (status error %d)" % (test_is_failed)
+ print("FAIL (status error %d)" % (test_is_failed))
self._failed_status_error("%s (%s)" % (doc_path, backend.get_name()))
return
@@ -107,7 +107,7 @@
out_path = os.path.join(self._outdir, filename)
try:
os.makedirs(out_path)
- except OSError, e:
+ except OSError as e:
if e.errno != errno.EEXIST:
raise
except:
@@ -116,7 +116,7 @@
refs_path = os.path.join(self._refsdir, filename)
if not os.path.isdir(refs_path):
- print "Reference dir not found for %s, skipping" % (doc_path)
+ print("Reference dir not found for %s, skipping" % (doc_path))
return
if self.config.backends:
@@ -138,16 +138,16 @@
def summary(self):
if not self._n_tests:
- print "No tests run"
+ print("No tests run")
return
- print "Total %d tests" % (self._n_tests)
- print "%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_tests)
+ print("Total %d tests" % (self._n_tests))
+ print("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_tests))
def report_tests(test_list, test_type):
n_tests = len(test_list)
if not n_tests:
return
- print "%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_tests, ", ".join(test_list))
+ print("%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_tests, ", ".join(test_list)))
report_tests(self._failed, "failed")
report_tests(self._crashed, "crashed")
report_tests(self._failed_status_error, "failed to run")
diff -rub regtest/Timer.py regtestmio/Timer.py
--- regtest/Timer.py 2011-09-13 12:25:14.000000000 +0200
+++ regtestmio/Timer.py 2011-09-13 12:00:11.000000000 +0200
@@ -61,13 +61,13 @@
t = Timer()
sleep(5)
- print "Elapsed: %s" % (t.elapsed_str())
+ print("Elapsed: %s" % (t.elapsed_str()))
sleep(1)
- print "Elapsed: %s" % (t.elapsed_str())
+ print("Elapsed: %s" % (t.elapsed_str()))
t.start()
sleep(2)
t.stop()
- print "Elapsed: %s" % (t.elapsed_str())
+ print("Elapsed: %s" % (t.elapsed_str()))
sleep(2)
- print "Elapsed: %s" % (t.elapsed_str())
+ print("Elapsed: %s" % (t.elapsed_str()))
_______________________________________________
poppler mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/poppler
