Re: [PATCH v2 1/4] test: dynamically generate parser tests
Apologies for the noise, I sent the entire series to the wrong list. :(
BR,
Jani.
On Mon, 28 Jan 2019, Jani Nikula wrote:
> It's impossible to have expected failures or other unittest decorators
> at subtest granularity. They only work at the test method level. On the
> other hand we don't want to be manually adding test methods when all of
> the tests are defined in terms of input files and expected results.
>
> Generate test methods dynamically from the input files, and assign to
> the test class. Running code at import time to do this is less than
> stellar, but it needs to be done early to have unittest test discovery
> find the methods.
>
> The alternative would be to add a load_tests protocol function [1], but
> that seems like more boilerplate. Can be added later as needed.
>
> Finally, one massive upside to this is the ability to run individual
> named testcases. For example, to test enum.c and typedef-enum.c, use:
>
> $ test/test_hawkmoth.py ParserTest.test_enum ParserTest.test_typedef_enum
>
> [1] https://docs.python.org/3/library/unittest.html#load-tests-protocol
> ---
> test/test_hawkmoth.py | 26 +++---
> test/testenv.py | 29 +
> 2 files changed, 36 insertions(+), 19 deletions(-)
>
> diff --git a/test/test_hawkmoth.py b/test/test_hawkmoth.py
> index 1fe02efc004d..75eebbe35eef 100755
> --- a/test/test_hawkmoth.py
> +++ b/test/test_hawkmoth.py
> @@ -8,28 +8,16 @@ import unittest
> import testenv
> from hawkmoth import hawkmoth
>
> -class ParserTest(unittest.TestCase):
> -def _run_test(self, input_filename):
> -# sanity check
> -self.assertTrue(os.path.isfile(input_filename))
> -
> -options = testenv.get_testcase_options(input_filename)
> -output = hawkmoth.parse_to_string(input_filename, False, **options)
> -expected = testenv.read_file(input_filename, ext='stdout')
> +def _get_output(input_filename, **options):
> +return hawkmoth.parse_to_string(input_filename, False, **options)
>
> -self.assertEqual(expected, output)
> +def _get_expected(input_filename, **options):
> +return testenv.read_file(input_filename, ext='stdout')
>
> -def _run_dir(self, path):
> -# sanity check
> -self.assertTrue(os.path.isdir(path))
> -
> -with self.subTest(path=path):
> -for f in testenv.get_testcases(path):
> -with self.subTest(source=os.path.basename(f)):
> -self._run_test(f)
> +class ParserTest(unittest.TestCase):
> +pass
>
> -def test_parser(self):
> -self._run_dir(testenv.testdir)
> +testenv.assign_test_methods(ParserTest, _get_output, _get_expected)
>
> if __name__ == '__main__':
> unittest.main()
> diff --git a/test/testenv.py b/test/testenv.py
> index f026aead8c07..cc80ef2218ed 100644
> --- a/test/testenv.py
> +++ b/test/testenv.py
> @@ -3,6 +3,7 @@
>
> import sys
> import os
> +import unittest
>
> testext = '.c'
> testdir = os.path.dirname(os.path.abspath(__file__))
> @@ -10,6 +11,16 @@ rootdir = os.path.dirname(testdir)
>
> sys.path.insert(0, rootdir)
>
> +def _testcase_name(testcase):
> +"""Convert a testcase filename into a test case identifier."""
> +name = os.path.splitext(os.path.basename(testcase))[0]
> +name = name.replace('-', '_')
> +name = 'test_{name}'.format(name=name)
> +
> +assert name.isidentifier()
> +
> +return name
> +
> def get_testcases(path):
> for f in sorted(os.listdir(path)):
> if f.endswith(testext):
> @@ -52,3 +63,21 @@ def read_file(filename, **kwargs):
> expected = file.read()
>
> return expected
> +
> +def _test_generator(get_output, get_expected, input_filename, **options):
> +"""Return a function that compares output/expected results on
> input_filename."""
> +def test(self):
> +output = get_output(input_filename, **options)
> +expected = get_expected(input_filename, **options)
> +
> +self.assertEqual(expected, output)
> +
> +return test
> +
> +def assign_test_methods(cls, get_output, get_expected):
> +"""Assign test case functions to the given class."""
> +for f in get_testcases(testdir):
> +options = get_testcase_options(f)
> +method = _test_generator(get_output, get_expected, f, **options)
> +
> +setattr(cls, _testcase_name(f), method)
> --
> 2.20.1
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch
[PATCH v2 4/4] test: use html builder for directive tests
Slower but does not lose information as much, providing more accurate results. Switch to the basic template for speed. --- test/conf.py | 2 +- test/test_cautodoc.py | 8 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/conf.py b/test/conf.py index 6d36327df085..d4c3d34c8b3b 100644 --- a/test/conf.py +++ b/test/conf.py @@ -75,7 +75,7 @@ pygments_style = None # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'basic' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/test/test_cautodoc.py b/test/test_cautodoc.py index 0a8e5bb2815f..ad54cae2e98e 100755 --- a/test/test_cautodoc.py +++ b/test/test_cautodoc.py @@ -9,7 +9,7 @@ import unittest import testenv from sphinx_testing import with_app -@with_app(confdir=testenv.testdir, create_new_srcdir=True, buildername='text') +@with_app(confdir=testenv.testdir, create_new_srcdir=True, buildername='html') def _get_output(input_filename, app, status, warning, **options): shutil.copyfile(input_filename, testenv.modify_filename(input_filename, dir=app.srcdir)) @@ -23,16 +23,16 @@ def _get_output(input_filename, app, status, warning, **options): app.build() -return testenv.read_file(os.path.join(app.outdir, 'index.txt')) +return testenv.read_file(os.path.join(app.outdir, 'index.html')) -@with_app(confdir=testenv.testdir, create_new_srcdir=True, buildername='text') +@with_app(confdir=testenv.testdir, create_new_srcdir=True, buildername='html') def _get_expected(input_filename, app, status, warning, **options): shutil.copyfile(testenv.modify_filename(input_filename, ext='stdout'), os.path.join(app.srcdir, 'index.rst')) app.build() -return testenv.read_file(os.path.join(app.outdir, 'index.txt')) +return testenv.read_file(os.path.join(app.outdir, 'index.html')) class DirectiveTest(unittest.TestCase): pass -- 2.20.1 ___ notmuch mailing list notmuch@notmuchmail.org https://notmuchmail.org/mailman/listinfo/notmuch
[PATCH v2 3/4] test: add support for flagging expected failures in testcase options
Since our tests are dynamically created, we also need to decorate
expected failures dynamically. Use the testcase options file as the
source. Only pass known directive options to the directive in the
cautodoc test.
Add a meta test to verify this works, with result "OK (expected
failures=2)".
---
test/meta-expected-failure.c | 3 +++
test/meta-expected-failure.options | 1 +
test/meta-expected-failure.stdout | 3 +++
test/test_cautodoc.py | 2 +-
test/testenv.py| 8
5 files changed, 16 insertions(+), 1 deletion(-)
create mode 100644 test/meta-expected-failure.c
create mode 100644 test/meta-expected-failure.options
create mode 100644 test/meta-expected-failure.stdout
diff --git a/test/meta-expected-failure.c b/test/meta-expected-failure.c
new file mode 100644
index ..18065136572f
--- /dev/null
+++ b/test/meta-expected-failure.c
@@ -0,0 +1,3 @@
+/**
+ * Meta test: This fails. Always.
+ */
diff --git a/test/meta-expected-failure.options
b/test/meta-expected-failure.options
new file mode 100644
index ..68c905946fce
--- /dev/null
+++ b/test/meta-expected-failure.options
@@ -0,0 +1 @@
+test-expected-failure
diff --git a/test/meta-expected-failure.stdout
b/test/meta-expected-failure.stdout
new file mode 100644
index ..dbd71dcb308f
--- /dev/null
+++ b/test/meta-expected-failure.stdout
@@ -0,0 +1,3 @@
+
+Meta test: This fails.
+
diff --git a/test/test_cautodoc.py b/test/test_cautodoc.py
index 848f2105a1da..0a8e5bb2815f 100755
--- a/test/test_cautodoc.py
+++ b/test/test_cautodoc.py
@@ -17,7 +17,7 @@ def _get_output(input_filename, app, status, warning,
**options):
with open(os.path.join(app.srcdir, 'index.rst'), 'w') as file:
fmt = '.. c:autodoc:: {source}\n'
file.write(fmt.format(source=os.path.basename(input_filename)))
-for key in options.keys():
+for key in [k for k in options.keys() if k in
testenv.directive_options]:
fmt = ' :{key}: {value}\n'
file.write(fmt.format(key=key, value=options[key]))
diff --git a/test/testenv.py b/test/testenv.py
index cc80ef2218ed..b6842a81b375 100644
--- a/test/testenv.py
+++ b/test/testenv.py
@@ -26,6 +26,11 @@ def get_testcases(path):
if f.endswith(testext):
yield os.path.join(path, f)
+directive_options = [
+'compat',
+'clang',
+]
+
def get_testcase_options(testcase):
options_filename = modify_filename(testcase, ext='options')
@@ -80,4 +85,7 @@ def assign_test_methods(cls, get_output, get_expected):
options = get_testcase_options(f)
method = _test_generator(get_output, get_expected, f, **options)
+if options.get('test-expected-failure') is not None:
+method = unittest.expectedFailure(method)
+
setattr(cls, _testcase_name(f), method)
--
2.20.1
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch
[PATCH v2 2/4] test: dynamically generate directive tests
Similar to the parser test transition to dynamically generate test
methods, but with more test separation added. After this, the Sphinx
build is done independently for each test case, and separately for the
actual directive output and the expected output. This removes any
potential C domain interactions between test cases and expected/actual
inputs.
The change does mean the Sphinx build is run roughly 50x times per full
test run, at the current number of test cases. This is somewhat offset
by the ability to run individual directive test cases:
$ test/test_cautodoc.py DirectiveTest.test_example_10_macro
---
test/{sphinx => }/conf.py | 0
test/sphinx/index.rst | 20 -
test/test_cautodoc.py | 61 ++-
3 files changed, 21 insertions(+), 60 deletions(-)
rename test/{sphinx => }/conf.py (100%)
delete mode 100644 test/sphinx/index.rst
diff --git a/test/sphinx/conf.py b/test/conf.py
similarity index 100%
rename from test/sphinx/conf.py
rename to test/conf.py
diff --git a/test/sphinx/index.rst b/test/sphinx/index.rst
deleted file mode 100644
index 20404ce8ee5e..
--- a/test/sphinx/index.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. Hawkmoth Test documentation master file, created by
- sphinx-quickstart on Wed Dec 12 13:11:26 2018.
- You can adapt this file completely to your liking, but it should at least
- contain the root `toctree` directive.
-
-Welcome to Hawkmoth Test's documentation!
-=
-
-.. toctree::
- :maxdepth: 2
- :caption: Contents:
-
-
-
-Indices and tables
-==
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
diff --git a/test/test_cautodoc.py b/test/test_cautodoc.py
index 3fb98af8629d..848f2105a1da 100755
--- a/test/test_cautodoc.py
+++ b/test/test_cautodoc.py
@@ -9,54 +9,35 @@ import unittest
import testenv
from sphinx_testing import with_app
-class DirectiveTest(unittest.TestCase):
-
-def _setup_src(self, srcdir, testcase_in):
-testcase_out = testenv.modify_filename(testcase_in, dir=srcdir)
-
-# use the pre-generated rst as comparison data
-shutil.copyfile(testenv.modify_filename(testcase_in, ext='stdout'),
-testenv.modify_filename(testcase_out,
ext='expected.rst'))
-
-# set up an rst file to run the extension
-shutil.copyfile(testcase_in, testcase_out)
-options = testenv.get_testcase_options(testcase_in)
+@with_app(confdir=testenv.testdir, create_new_srcdir=True, buildername='text')
+def _get_output(input_filename, app, status, warning, **options):
+shutil.copyfile(input_filename,
+testenv.modify_filename(input_filename, dir=app.srcdir))
-with open(testenv.modify_filename(testcase_out, ext='output.rst'),
'w') as file:
-fmt = '.. c:autodoc:: {source}\n'
-file.write(fmt.format(source=os.path.basename(testcase_out)))
-for key in options.keys():
-fmt = ' :{key}: {value}\n'
-file.write(fmt.format(key=key, value=options[key]))
+with open(os.path.join(app.srcdir, 'index.rst'), 'w') as file:
+fmt = '.. c:autodoc:: {source}\n'
+file.write(fmt.format(source=os.path.basename(input_filename)))
+for key in options.keys():
+fmt = ' :{key}: {value}\n'
+file.write(fmt.format(key=key, value=options[key]))
-def _check_out(self, outdir, testcase_in):
-testcase_out = testenv.modify_filename(testcase_in, dir=outdir)
+app.build()
-# compare output from the pre-generated rst against the output
generated
-# by the extension
+return testenv.read_file(os.path.join(app.outdir, 'index.txt'))
-output = testenv.read_file(testenv.modify_filename(testcase_out,
- ext='output.txt'))
+@with_app(confdir=testenv.testdir, create_new_srcdir=True, buildername='text')
+def _get_expected(input_filename, app, status, warning, **options):
+shutil.copyfile(testenv.modify_filename(input_filename, ext='stdout'),
+os.path.join(app.srcdir, 'index.rst'))
-expected = testenv.read_file(testenv.modify_filename(testcase_out,
-
ext='expected.txt'))
+app.build()
-self.assertEqual(expected, output)
+return testenv.read_file(os.path.join(app.outdir, 'index.txt'))
-# Use copy_srcdir_to_tmpdir=False and outdir='some-dir' for debugging
-@with_app(srcdir=os.path.join(testenv.testdir, 'sphinx'),
- buildername='text', copy_srcdir_to_tmpdir=True)
-def test_directive(self, app, status, warning):
-testcases = list(testenv.get_testcases(testenv.testdir))
-
-for f in testcases:
-self._setup_src(app.srcdir, f)
-
-app.build()
+class DirectiveTest(unittest.TestCase):
+pass
-for f in testcases:
-
[PATCH v2 1/4] test: dynamically generate parser tests
It's impossible to have expected failures or other unittest decorators
at subtest granularity. They only work at the test method level. On the
other hand we don't want to be manually adding test methods when all of
the tests are defined in terms of input files and expected results.
Generate test methods dynamically from the input files, and assign to
the test class. Running code at import time to do this is less than
stellar, but it needs to be done early to have unittest test discovery
find the methods.
The alternative would be to add a load_tests protocol function [1], but
that seems like more boilerplate. Can be added later as needed.
Finally, one massive upside to this is the ability to run individual
named testcases. For example, to test enum.c and typedef-enum.c, use:
$ test/test_hawkmoth.py ParserTest.test_enum ParserTest.test_typedef_enum
[1] https://docs.python.org/3/library/unittest.html#load-tests-protocol
---
test/test_hawkmoth.py | 26 +++---
test/testenv.py | 29 +
2 files changed, 36 insertions(+), 19 deletions(-)
diff --git a/test/test_hawkmoth.py b/test/test_hawkmoth.py
index 1fe02efc004d..75eebbe35eef 100755
--- a/test/test_hawkmoth.py
+++ b/test/test_hawkmoth.py
@@ -8,28 +8,16 @@ import unittest
import testenv
from hawkmoth import hawkmoth
-class ParserTest(unittest.TestCase):
-def _run_test(self, input_filename):
-# sanity check
-self.assertTrue(os.path.isfile(input_filename))
-
-options = testenv.get_testcase_options(input_filename)
-output = hawkmoth.parse_to_string(input_filename, False, **options)
-expected = testenv.read_file(input_filename, ext='stdout')
+def _get_output(input_filename, **options):
+return hawkmoth.parse_to_string(input_filename, False, **options)
-self.assertEqual(expected, output)
+def _get_expected(input_filename, **options):
+return testenv.read_file(input_filename, ext='stdout')
-def _run_dir(self, path):
-# sanity check
-self.assertTrue(os.path.isdir(path))
-
-with self.subTest(path=path):
-for f in testenv.get_testcases(path):
-with self.subTest(source=os.path.basename(f)):
-self._run_test(f)
+class ParserTest(unittest.TestCase):
+pass
-def test_parser(self):
-self._run_dir(testenv.testdir)
+testenv.assign_test_methods(ParserTest, _get_output, _get_expected)
if __name__ == '__main__':
unittest.main()
diff --git a/test/testenv.py b/test/testenv.py
index f026aead8c07..cc80ef2218ed 100644
--- a/test/testenv.py
+++ b/test/testenv.py
@@ -3,6 +3,7 @@
import sys
import os
+import unittest
testext = '.c'
testdir = os.path.dirname(os.path.abspath(__file__))
@@ -10,6 +11,16 @@ rootdir = os.path.dirname(testdir)
sys.path.insert(0, rootdir)
+def _testcase_name(testcase):
+"""Convert a testcase filename into a test case identifier."""
+name = os.path.splitext(os.path.basename(testcase))[0]
+name = name.replace('-', '_')
+name = 'test_{name}'.format(name=name)
+
+assert name.isidentifier()
+
+return name
+
def get_testcases(path):
for f in sorted(os.listdir(path)):
if f.endswith(testext):
@@ -52,3 +63,21 @@ def read_file(filename, **kwargs):
expected = file.read()
return expected
+
+def _test_generator(get_output, get_expected, input_filename, **options):
+"""Return a function that compares output/expected results on
input_filename."""
+def test(self):
+output = get_output(input_filename, **options)
+expected = get_expected(input_filename, **options)
+
+self.assertEqual(expected, output)
+
+return test
+
+def assign_test_methods(cls, get_output, get_expected):
+"""Assign test case functions to the given class."""
+for f in get_testcases(testdir):
+options = get_testcase_options(f)
+method = _test_generator(get_output, get_expected, f, **options)
+
+setattr(cls, _testcase_name(f), method)
--
2.20.1
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch
[RFC PATCH] forward: Add References header to forwarded messages
Include the message-id of forwarded messages in the new message. This ensures that the new message is linked to the same thread. --- emacs/notmuch-mua.el | 14 +++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/emacs/notmuch-mua.el b/emacs/notmuch-mua.el index e205fa4c..1ba9317e 100644 --- a/emacs/notmuch-mua.el +++ b/emacs/notmuch-mua.el @@ -433,8 +433,9 @@ the From: address." (let* ((other-headers (when (or prompt-for-sender notmuch-always-prompt-for-sender) (list (cons 'From (notmuch-mua-prompt-for-sender) -forward-subject) ;; Comes from the first message and is +forward-subject ;; Comes from the first message and is ;; applied later. +forward-references) ;; Accumulated message-ids of forwarded messages ;; Generate the template for the outgoing message. (notmuch-mua-mail nil "" other-headers nil (notmuch-mua-get-switch-function)) @@ -452,7 +453,12 @@ the From: address." ;; Because we process the messages in reverse order, ;; always generate a forwarded subject, then use the ;; last (i.e. first) one. - (setq forward-subject (message-make-forward-subject))) + (setq forward-subject (message-make-forward-subject)) + (if forward-references + (setq forward-references + (concat forward-references ", " + (message-fetch-field "Message-ID"))) + (setq forward-references (message-fetch-field "Message-ID" ;; Make a copy ready to be forwarded in the ;; composition buffer. (message-forward-make-body temp-buffer) @@ -466,7 +472,9 @@ the From: address." (save-restriction (message-narrow-to-headers) (message-remove-header "Subject") - (message-add-header (concat "Subject: " forward-subject))) + (message-add-header (concat "Subject: " forward-subject)) + (message-remove-header "References") + (message-add-header (concat "References: " forward-references))) ;; `message-forward-make-body' shows the User-agent header. Hide ;; it again. -- 2.20.1 ___ notmuch mailing list notmuch@notmuchmail.org https://notmuchmail.org/mailman/listinfo/notmuch
Re: [PATCH] doc: fix references to search.exclude_tags
Peter Wang writes: > The documentation incorrectly referred to a configuration item > "search.tag_exclude" in some places, instead of "search.exclude_tags". pushed, thanks. d ___ notmuch mailing list notmuch@notmuchmail.org https://notmuchmail.org/mailman/listinfo/notmuch
