This adds a mostly compliant JUnit backend. It currently passes all of its tests and jenkins happily consumes the xml we produce.
This needs to have some refactoring done in profile to make it 100% compliant with the junit-7.xsd from upstream jenkins, (although I would be comfortable for now pushing with the known non-optimal behavior if Jenkins will accept it), because JUnit expects to be given the number of tests for the initial metadata block, but we have no way to calculate that number until after the run has started. This is because of the flattening pass in profile that flattens the nested directory structure into a flat dictionary. There are two options to solve this problem: 1) Flatten all.py and other modules. This is a lot of work and I have many work-in-progress branches to do just hat 2) Push the pass out to a public method and call it ahead of time. This seems really hacky to me, and I'd rather not do something that ugly. Currently this patch just passes 0 for the test count unconditionally, jenkins does not seem to have a problem with this. This includes JUnit.xsd from the jenkins svn repository for piglit framework unit testing. This is only used in the piglit python framework unit tests. Signed-off-by: Dylan Baker <[email protected]> --- framework/programs/run.py | 4 ++ framework/results.py | 65 ++++++++++++++++++++++- framework/tests/results_tests.py | 91 ++++++++++++++++++++++++++++++++ framework/tests/schema/junit-7.xsd | 104 +++++++++++++++++++++++++++++++++++++ framework/tests/utils.py | 22 ++++++++ 5 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 framework/tests/schema/junit-7.xsd diff --git a/framework/programs/run.py b/framework/programs/run.py index 8b7045d..82fc797 100644 --- a/framework/programs/run.py +++ b/framework/programs/run.py @@ -221,6 +221,10 @@ def run(input_): options['platform'] = args.platform options['name'] = results.name options['env'] = core.collect_system_info() + # FIXME: this should be the actual count, but profile needs to be + # refactored to make that possible because of the flattening pass that is + # part of profile.run + options['test_count'] = 0 # Begin json. backend = framework.results.get_backend(args.backend)( diff --git a/framework/results.py b/framework/results.py index 741adc9..eabeb4d 100644 --- a/framework/results.py +++ b/framework/results.py @@ -26,11 +26,16 @@ import os import sys import abc import threading +import posixpath from cStringIO import StringIO try: import simplejson as json except ImportError: import json +try: + from lxml import etree +except ImportError: + import xml.etree.cElementTree as etree import framework.status as status @@ -43,7 +48,7 @@ __all__ = [ ] # A list of available backends -BACKENDS = ['json'] +BACKENDS = ['json', 'junit'] # The current version of the JSON results CURRENT_JSON_VERSION = 1 @@ -349,6 +354,63 @@ class JSONBackend(Backend): self._write_dict_item(name, data) +class JUnitBackend(Backend): + """ Backend that produces ANT JUnit XML + + Based on the following schema: + https://svn.jenkins-ci.org/trunk/hudson/dtkit/dtkit-format/dtkit-junit-model/src/main/resources/com/thalesgroup/dtkit/junit/model/xsd/junit-7.xsd + + """ + # TODO: add fsync support + + def __init__(self, dest, metadata, **options): + self._file = open(os.path.join(dest, 'results.xml'), 'w') + + # Write initial headers and other data that etree cannot write for us + self._file.write('<?xml version="1.0" encoding="UTF-8" ?>\n') + self._file.write('<testsuites>\n') + self._file.write( + '<testsuite name="piglit" tests="{}">\n'.format( + metadata['test_count'])) + + def finalize(self, metadata=None): + self._file.write('</testsuite>\n') + self._file.write('</testsuites>\n') + self._file.close() + + def write_test(self, name, data): + # Split the name of the test and the group (what junit refers to as + # classname), and replace piglits '/' seperated groups with '.', after + # replacing any '.' with '_' (so we don't get false groups) + classname, testname = posixpath.split(name) + assert classname + assert testname + classname = classname.replace('.', '_').replace('/', '.') + element = etree.Element('testcase', name=testname, classname=classname, + time=str(data['time']), + status=str(data['result'])) + + # Add stdout + out = etree.SubElement(element, 'system-out') + out.text = data['out'] + + # Add stderr + err = etree.SubElement(element, 'system-err') + err.text = data['err'] + + # Add relavent result value, if the result is pass then it doesn't need + # one of these statuses + if data['result'] == 'skip': + etree.SubElement(element, 'skipped') + elif data['result'] in ['warn', 'fail', 'dmesg-warn', 'dmesg-fail']: + etree.SubElement(element, 'failure') + elif data['result'] == 'crash': + etree.SubElement(element, 'error') + + self._file.write(etree.tostring(element)) + self._file.write('\n') + + class TestResult(dict): def __init__(self, *args): super(TestResult, self).__init__(*args) @@ -587,6 +649,7 @@ def get_backend(backend): """ Returns a BackendInstance based on the string passed """ backends = { 'json': JSONBackend, + 'junit': JUnitBackend, } # Be sure that we're exporting the same list of backends that we actually diff --git a/framework/tests/results_tests.py b/framework/tests/results_tests.py index f11a86c..df89126 100644 --- a/framework/tests/results_tests.py +++ b/framework/tests/results_tests.py @@ -23,6 +23,10 @@ import os import json +try: + from lxml import etree +except ImportError: + import xml.etree.cElementTree as etree import nose.tools as nt import framework.tests.utils as utils import framework.results as results @@ -32,8 +36,11 @@ import framework.status as status BACKEND_INITIAL_META = { 'name': 'name', 'env': {}, + 'test_count': 0, } +JUNIT_SCHEMA = 'framework/tests/schema/junit-7.xsd' + def check_initialize(target): """ Check that a class initializes without error """ @@ -181,3 +188,87 @@ def test_get_backend(): for name, inst in backends.iteritems(): check.description = 'get_backend({0}) returns {0} backend'.format(name) yield check, name, inst + + +class TestJunitNoTests(utils.StaticDirectory): + @classmethod + def setup_class(cls): + super(TestJunitNoTests, cls).setup_class() + test = results.JUnitBackend(cls.tdir, BACKEND_INITIAL_META) + test.finalize() + cls.test_file = os.path.join(cls.tdir, 'results.xml') + + def test_xml_well_formed(self): + """ JUnitBackend.__init__ and finalize produce well formed xml + + While it will produce valid XML, it cannot produc valid JUnit, since + JUnit requires at least one test to be valid + + """ + try: + etree.parse(self.test_file) + except Exception as e: + raise AssertionError(e) + + +class TestJUnitSingleTest(TestJunitNoTests): + @classmethod + def setup_class(cls): + super(TestJUnitSingleTest, cls).setup_class() + cls.test_file = os.path.join(cls.tdir, 'results.xml') + test = results.JUnitBackend(cls.tdir, BACKEND_INITIAL_META) + test.write_test( + 'a/test/group/test1', + results.TestResult({ + 'time': 1.2345, + 'result': 'pass', + 'out': 'this is stdout', + 'err': 'this is stderr', + }) + ) + test.finalize() + + def test_xml_well_formed(self): + """ JUnitBackend.write_test() (once) produces well formed xml """ + super(TestJUnitSingleTest, self).test_xml_well_formed() + + def test_xml_valid(self): + """ JUnitBackend.write_test() (once) produces valid xml """ + schema = etree.XMLSchema(file=JUNIT_SCHEMA) + with open(self.test_file, 'r') as f: + assert schema.validate(etree.parse(f)), 'xml is not valid' + + +class TestJUnitMultiTest(TestJUnitSingleTest): + @classmethod + def setup_class(cls): + super(TestJUnitMultiTest, cls).setup_class() + cls.test_file = os.path.join(cls.tdir, 'results.xml') + test = results.JUnitBackend(cls.tdir, BACKEND_INITIAL_META) + test.write_test( + 'a/test/group/test1', + results.TestResult({ + 'time': 1.2345, + 'result': 'pass', + 'out': 'this is stdout', + 'err': 'this is stderr', + }) + ) + test.write_test( + 'a/different/test/group/test2', + results.TestResult({ + 'time': 1.2345, + 'result': 'fail', + 'out': 'this is stdout', + 'err': 'this is stderr', + }) + ) + test.finalize() + + def test_xml_well_formed(self): + """ JUnitBackend.write_test() (twice) produces well formed xml """ + super(TestJUnitMultiTest, self).test_xml_well_formed() + + def test_xml_valid(self): + """ JUnitBackend.write_test() (twice) produces valid xml """ + super(TestJUnitMultiTest, self).test_xml_valid() diff --git a/framework/tests/schema/junit-7.xsd b/framework/tests/schema/junit-7.xsd new file mode 100644 index 0000000..bc07b52 --- /dev/null +++ b/framework/tests/schema/junit-7.xsd @@ -0,0 +1,104 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema";> + + <xs:element name="failure"> + <xs:complexType mixed="true"> + <xs:attribute name="type" type="xs:string" use="optional"/> + <xs:attribute name="message" type="xs:string" use="optional"/> + </xs:complexType> + </xs:element> + + <xs:element name="error"> + <xs:complexType mixed="true"> + <xs:attribute name="type" type="xs:string" use="optional"/> + <xs:attribute name="message" type="xs:string" use="optional"/> + </xs:complexType> + </xs:element> + + <xs:element name="skipped"> + <xs:complexType mixed="true"> + <xs:attribute name="type" type="xs:string" use="optional"/> + <xs:attribute name="message" type="xs:string" use="optional"/> + </xs:complexType> + </xs:element> + + <xs:element name="properties"> + <xs:complexType> + <xs:sequence> + <xs:element ref="property" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + </xs:element> + + <xs:element name="property"> + <xs:complexType> + <xs:attribute name="name" type="xs:string" use="required"/> + <xs:attribute name="value" type="xs:string" use="required"/> + </xs:complexType> + </xs:element> + + <xs:element name="system-err" type="xs:string"/> + <xs:element name="system-out" type="xs:string"/> + + <xs:element name="testcase"> + <xs:complexType> + <xs:sequence> + <xs:choice minOccurs="0" maxOccurs="unbounded"> + <xs:element ref="skipped"/> + <xs:element ref="error"/> + <xs:element ref="failure"/> + <xs:element ref="system-out"/> + <xs:element ref="system-err"/> + </xs:choice> + </xs:sequence> + <xs:attribute name="name" type="xs:string" use="required"/> + <xs:attribute name="assertions" type="xs:string" use="optional"/> + <xs:attribute name="time" type="xs:string" use="optional"/> + <xs:attribute name="classname" type="xs:string" use="optional"/> + <xs:attribute name="status" type="xs:string" use="optional"/> + <xs:attribute name="class" type="xs:string" use="optional"/> + <xs:attribute name="file" type="xs:string" use="optional"/> + <xs:attribute name="line" type="xs:string" use="optional"/> + </xs:complexType> + </xs:element> + + <xs:element name="testsuite"> + <xs:complexType> + <xs:choice minOccurs="0" maxOccurs="unbounded"> + <xs:element ref="testsuite"/> + <xs:element ref="properties"/> + <xs:element ref="testcase"/> + <xs:element ref="system-out"/> + <xs:element ref="system-err"/> + </xs:choice> + <xs:attribute name="name" type="xs:string" use="optional"/> + <xs:attribute name="tests" type="xs:string" use="required"/> + <xs:attribute name="failures" type="xs:string" use="optional"/> + <xs:attribute name="errors" type="xs:string" use="optional"/> + <xs:attribute name="time" type="xs:string" use="optional"/> + <xs:attribute name="disabled" type="xs:string" use="optional"/> + <xs:attribute name="skipped" type="xs:string" use="optional"/> + <xs:attribute name="timestamp" type="xs:string" use="optional"/> + <xs:attribute name="hostname" type="xs:string" use="optional"/> + <xs:attribute name="id" type="xs:string" use="optional"/> + <xs:attribute name="package" type="xs:string" use="optional"/> + <xs:attribute name="assertions" type="xs:string" use="optional"/> + <xs:attribute name="file" type="xs:string" use="optional"/> + </xs:complexType> + </xs:element> + + <xs:element name="testsuites"> + <xs:complexType> + <xs:sequence> + <xs:element ref="testsuite" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + <xs:attribute name="name" type="xs:string" use="optional"/> + <xs:attribute name="time" type="xs:string" use="optional"/> + <xs:attribute name="tests" type="xs:string" use="optional"/> + <xs:attribute name="failures" type="xs:string" use="optional"/> + <xs:attribute name="disabled" type="xs:string" use="optional"/> + <xs:attribute name="errors" type="xs:string" use="optional"/> + </xs:complexType> + </xs:element> + +</xs:schema> \ No newline at end of file diff --git a/framework/tests/utils.py b/framework/tests/utils.py index b2fe86e..2694fab 100644 --- a/framework/tests/utils.py +++ b/framework/tests/utils.py @@ -230,3 +230,25 @@ class TestWithEnvClean(object): # reversed order to make any sense for call, args in reversed(self._teardown_calls): call(*args) + + +class StaticDirectory(object): + """ Helper class providing shared files creation and cleanup + + One should override the setup_class method in a child class, call super(), + and then add files to cls.dir. + + Tests in this class should NOT modify the contents of tidr, if you want + that functionality you want a different class + + """ + @classmethod + def setup_class(cls): + """ Create a temperary directory that will be removed in teardown_class + """ + cls.tdir = tempfile.mkdtemp() + + @classmethod + def teardown_class(cls): + """ Remove the temporary directory """ + shutil.rmtree(cls.tdir) -- 2.1.0 _______________________________________________ Piglit mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/piglit
