Apologies for the noise, I sent the entire series to the wrong list. :( BR, Jani. On Mon, 28 Jan 2019, Jani Nikula <jani@nikula.org> wrote: > It's impossible to have expected failures or other unittest decorators > at subtest granularity. They only work at the test method level. On the > other hand we don't want to be manually adding test methods when all of > the tests are defined in terms of input files and expected results. > > Generate test methods dynamically from the input files, and assign to > the test class. Running code at import time to do this is less than > stellar, but it needs to be done early to have unittest test discovery > find the methods. > > The alternative would be to add a load_tests protocol function [1], but > that seems like more boilerplate. Can be added later as needed. > > Finally, one massive upside to this is the ability to run individual > named testcases. For example, to test enum.c and typedef-enum.c, use: > > $ test/test_hawkmoth.py ParserTest.test_enum ParserTest.test_typedef_enum > > [1] https://docs.python.org/3/library/unittest.html#load-tests-protocol > --- > test/test_hawkmoth.py | 26 +++++++------------------- > test/testenv.py | 29 +++++++++++++++++++++++++++++ > 2 files changed, 36 insertions(+), 19 deletions(-) > > diff --git a/test/test_hawkmoth.py b/test/test_hawkmoth.py > index 1fe02efc004d..75eebbe35eef 100755 > --- a/test/test_hawkmoth.py > +++ b/test/test_hawkmoth.py > @@ -8,28 +8,16 @@ import unittest > import testenv > from hawkmoth import hawkmoth > > -class ParserTest(unittest.TestCase): > - def _run_test(self, input_filename): > - # sanity check > - self.assertTrue(os.path.isfile(input_filename)) > - > - options = testenv.get_testcase_options(input_filename) > - output = hawkmoth.parse_to_string(input_filename, False, **options) > - expected = testenv.read_file(input_filename, ext='stdout') > +def _get_output(input_filename, **options): > + return hawkmoth.parse_to_string(input_filename, False, **options) > > - self.assertEqual(expected, output) > +def _get_expected(input_filename, **options): > + return testenv.read_file(input_filename, ext='stdout') > > - def _run_dir(self, path): > - # sanity check > - self.assertTrue(os.path.isdir(path)) > - > - with self.subTest(path=path): > - for f in testenv.get_testcases(path): > - with self.subTest(source=os.path.basename(f)): > - self._run_test(f) > +class ParserTest(unittest.TestCase): > + pass > > - def test_parser(self): > - self._run_dir(testenv.testdir) > +testenv.assign_test_methods(ParserTest, _get_output, _get_expected) > > if __name__ == '__main__': > unittest.main() > diff --git a/test/testenv.py b/test/testenv.py > index f026aead8c07..cc80ef2218ed 100644 > --- a/test/testenv.py > +++ b/test/testenv.py > @@ -3,6 +3,7 @@ > > import sys > import os > +import unittest > > testext = '.c' > testdir = os.path.dirname(os.path.abspath(__file__)) > @@ -10,6 +11,16 @@ rootdir = os.path.dirname(testdir) > > sys.path.insert(0, rootdir) > > +def _testcase_name(testcase): > + """Convert a testcase filename into a test case identifier.""" > + name = os.path.splitext(os.path.basename(testcase))[0] > + name = name.replace('-', '_') > + name = 'test_{name}'.format(name=name) > + > + assert name.isidentifier() > + > + return name > + > def get_testcases(path): > for f in sorted(os.listdir(path)): > if f.endswith(testext): > @@ -52,3 +63,21 @@ def read_file(filename, **kwargs): > expected = file.read() > > return expected > + > +def _test_generator(get_output, get_expected, input_filename, **options): > + """Return a function that compares output/expected results on input_filename.""" > + def test(self): > + output = get_output(input_filename, **options) > + expected = get_expected(input_filename, **options) > + > + self.assertEqual(expected, output) > + > + return test > + > +def assign_test_methods(cls, get_output, get_expected): > + """Assign test case functions to the given class.""" > + for f in get_testcases(testdir): > + options = get_testcase_options(f) > + method = _test_generator(get_output, get_expected, f, **options) > + > + setattr(cls, _testcase_name(f), method) > -- > 2.20.1 _______________________________________________ notmuch mailing list notmuch@notmuchmail.org https://notmuchmail.org/mailman/listinfo/notmuch