This helps to make the results format more portable, since it no longer requires a separate image directory in a hardcoded location.
Signed-off-by: Dylan Baker <[email protected]> --- framework/backends/json.py | 36 ++- framework/results.py | 7 +- templates/test_result.mako | 8 +- tests/xts.py | 24 +- unittests/framework/backends/images/LICENSE | 6 +- unittests/framework/backends/images/openlogo-nd-25.png | Bin 0 -> 409 bytes unittests/framework/backends/images/ref.png | 1 +- unittests/framework/backends/images/render.png | 1 +- unittests/framework/backends/schema/piglit-10.json | 135 ++++++++++- unittests/framework/backends/shared.py | 2 +- unittests/framework/backends/test_json_update.py | 101 +++++++- 11 files changed, 301 insertions(+), 20 deletions(-) create mode 100644 unittests/framework/backends/images/LICENSE create mode 100644 unittests/framework/backends/images/openlogo-nd-25.png create mode 120000 unittests/framework/backends/images/ref.png create mode 120000 unittests/framework/backends/images/render.png create mode 100644 unittests/framework/backends/schema/piglit-10.json diff --git a/framework/backends/json.py b/framework/backends/json.py index 17002ed..bf64ad9 100644 --- a/framework/backends/json.py +++ b/framework/backends/json.py @@ -23,10 +23,10 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals ) +import base64 import collections import functools import os -import posixpath import shutil import sys @@ -53,7 +53,7 @@ __all__ = [ ] # The current version of the JSON results -CURRENT_JSON_VERSION = 9 +CURRENT_JSON_VERSION = 10 # The minimum JSON format supported MINIMUM_SUPPORTED_VERSION = 7 @@ -312,6 +312,7 @@ def _update_results(results, filepath): updates = { 7: _update_seven_to_eight, 8: _update_eight_to_nine, + 9: _update_nine_to_ten, } while results['results_version'] < CURRENT_JSON_VERSION: @@ -387,6 +388,37 @@ def _update_eight_to_nine(result): return result +def _update_nine_to_ten(result): + """Update json results from version 8 to 9. + + This changes the PID feild of the TestResult object to alist of Integers or + null rather than a single integer or null. + + """ + for test in six.itervalues(result['tests']): + if 'images' in test: + new_list = [] + + for image in test['images']: + new = {'description': image['image_desc']} + + if 'image_ref' in image: + with open(image['image_ref'], 'rb') as f: + new['reference'] = base64.b64encode(f.read()).decode('utf-8') + + if 'image_render' in image: + with open(image['image_render'], 'rb') as f: + new['rendered'] = base64.b64encode(f.read()).decode('utf-8') + + new_list.append(new) + + test['images'] = new_list + + result['results_version'] = 10 + + return result + + REGISTRY = Registry( extensions=['.json'], backend=JSONBackend, diff --git a/framework/results.py b/framework/results.py index 3c2bb2a..fc1011d 100644 --- a/framework/results.py +++ b/framework/results.py @@ -207,6 +207,11 @@ class TestResult(object): 'dmesg': self.dmesg, 'pid': self.pid, } + + # Images are not required, only add them if they are not empty + if self.images: + obj['images'] = self.images + return obj @classmethod @@ -225,7 +230,7 @@ class TestResult(object): inst = cls() for each in ['returncode', 'command', 'exception', 'environment', - 'traceback', 'dmesg', 'pid', 'result']: + 'traceback', 'dmesg', 'pid', 'result', 'images']: if each in dict_: setattr(inst, each, dict_[each]) diff --git a/templates/test_result.mako b/templates/test_result.mako index ff08797..ab2b2b7 100644 --- a/templates/test_result.mako +++ b/templates/test_result.mako @@ -38,11 +38,11 @@ <td>reference</td> <td>rendered</td> </tr> - % for image in images: + % for image in value.images: <tr> - <td>${image['image_desc']}</td> - <td><img src="file://${image['image_ref']}" /></td> - <td><img src="file://${image['image_render']}" /></td> + <td>${image['description']}</td> + <td><img alt="Reference Image" src="data:image/png;base64,${image['reference']}" /></td> + <td><img alt="Rendered Image" src="data:image/png;base64,${image['rendered']}" /></td> </tr> % endfor </table> diff --git a/tests/xts.py b/tests/xts.py index 715ecfa..d2f1e30 100644 --- a/tests/xts.py +++ b/tests/xts.py @@ -26,6 +26,7 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals ) +import base64 import os import re import subprocess @@ -50,14 +51,6 @@ class XTSProfile(TestProfile): # pylint: disable=too-few-public-methods """ XTSTest.RESULTS_PATH = self.results_dir - try: - os.mkdir(os.path.join(self.results_dir, 'images')) - except OSError as e: - # If the exception is not 'directory already exists', raise the - # exception - if e.errno != 17: - raise - class XTSTest(Test): # pylint: disable=too-few-public-methods """ X Test Suite class @@ -123,7 +116,9 @@ class XTSTest(Test): # pylint: disable=too-few-public-methods try: out = subprocess.check_output(command, cwd=self.cwd) except OSError: - images.append({'image_desc': 'image processing failed'}) + images.append({'description': 'image processing failed', + 'reference': None, + 'rendered': None}) continue # Each Err*.err log contains a rendered image, and a reference @@ -139,13 +134,18 @@ class XTSTest(Test): # pylint: disable=too-few-public-methods self.RESULTS_PATH, 'images', '{1}-{2}-render.png'.format( self.testname, match.group(1))) + with open(ref_path, 'rb') as f: + ref = base64.b64encode(f.read()).decode('utf-8') + with open(render_path, 'rb') as f: + render = base64.b64encode(f.read()).decode('utf-8') + split = out.splitlines() os.rename(os.path.join(self.cwd, split[0]), render_path) os.rename(os.path.join(self.cwd, split[1]), ref_path) - images.append({'image_desc': desc, - 'image_ref': ref_path, - 'image_render': render_path}) + images.append({'description': desc, + 'reference': ref, + 'rendered': render}) return images diff --git a/unittests/framework/backends/images/LICENSE b/unittests/framework/backends/images/LICENSE new file mode 100644 index 0000000..96286c0 --- /dev/null +++ b/unittests/framework/backends/images/LICENSE @@ -0,0 +1,6 @@ +The Debian Open Use Logo(s) are Copyright (c) 1999 Software in the Public +Interest, Inc., and are released under the terms of the GNU Lesser General +Public License, version 3 or any later version, or, at your option, of the +Creative Commons Attribution-ShareAlike 3.0 Unported License. + +openlogo-nd-25.png and it's links are covered by this license. diff --git a/unittests/framework/backends/images/openlogo-nd-25.png b/unittests/framework/backends/images/openlogo-nd-25.png new file mode 100644 index 0000000..66f0ba0 Binary files /dev/null and b/unittests/framework/backends/images/openlogo-nd-25.png differ diff --git a/unittests/framework/backends/images/ref.png b/unittests/framework/backends/images/ref.png new file mode 120000 index 0000000..c7a1526 --- /dev/null +++ b/unittests/framework/backends/images/ref.png @@ -0,0 +1 @@ +openlogo-nd-25.png \ No newline at end of file diff --git a/unittests/framework/backends/images/render.png b/unittests/framework/backends/images/render.png new file mode 120000 index 0000000..c7a1526 --- /dev/null +++ b/unittests/framework/backends/images/render.png @@ -0,0 +1 @@ +openlogo-nd-25.png \ No newline at end of file diff --git a/unittests/framework/backends/schema/piglit-10.json b/unittests/framework/backends/schema/piglit-10.json new file mode 100644 index 0000000..440db14 --- /dev/null +++ b/unittests/framework/backends/schema/piglit-10.json @@ -0,0 +1,135 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#";, + "title": "TestrunResult", + "description": "The collection of all results", + "type": "object", + "properties": { + "__type__": { "type": "string" }, + "clinfo": { "type": ["string", "null"] }, + "glxinfo": { "type": ["string", "null"] }, + "lspci": { "type": ["string", "null"] }, + "wglinfo": { "type": ["string", "null"] }, + "name": { "type": "string" }, + "results_version": { "type": "number" }, + "uname": { "type": [ "string", "null" ] }, + "time_elapsed": { "$ref": "#/definitions/timeAttribute" }, + "options": { + "descrption": "The options that were invoked with this run. These are implementation specific and not required.", + "type": "object", + "properties": { + "exclude_tests": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true + }, + "include_filter": { + "type": "array", + "items": { "type": "string" } + }, + "exclude_filter": { + "type": "array", + "items": { "type": "string" } + }, + "sync": { "type": "boolean" }, + "valgrind": { "type": "boolean" }, + "monitored": { "type": "boolean" }, + "dmesg": { "type": "boolean" }, + "execute": { "type": "boolean" }, + "concurrent": { "enum": ["none", "all", "some"] }, + "platform": { "type": "string" }, + "log_level": { "type": "string" }, + "env": { + "description": "Environment variables that must be specified", + "type": "object", + "additionalProperties": { "type": "string" } + }, + "profile": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "totals": { + "type": "object", + "description": "A calculation of the group totals.", + "additionalProperties": { + "type": "object", + "properties": { + "crash": { "type": "number" }, + "dmesg-fail": { "type": "number" }, + "dmesg-warn": { "type": "number" }, + "fail": { "type": "number" }, + "incomplete": { "type": "number" }, + "notrun": { "type": "number" }, + "pass": { "type": "number" }, + "skip": { "type": "number" }, + "timeout": { "type": "number" }, + "warn": { "type": "number" } + }, + "additionalProperties": false, + "required": [ "crash", "dmesg-fail", "dmesg-warn", "fail", "incomplete", "notrun", "pass", "skip", "timeout", "warn" ] + } + }, + "tests": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "__type__": { "type": "string" }, + "err": { "type": "string" }, + "exception": { "type": ["string", "null"] }, + "result": { + "type": "string", + "enum": [ "pass", "fail", "crash", "warn", "incomplete", "notrun", "skip", "dmesg-warn", "dmesg-fail" ] + }, + "environment": { "type": "string" }, + "command": { "type": "string" }, + "traceback": { "type": ["string", "null"] }, + "out": { "type": "string" }, + "dmesg": { "type": "string" }, + "pid": { + "type": "array", + "items": { "type": "number" } + }, + "returncode": { "type": [ "number", "null" ] }, + "time": { "$ref": "#/definitions/timeAttribute" }, + "subtests": { + "type": "object", + "properties": { "__type__": { "type": "string" } }, + "additionalProperties": { "type": "string" }, + "required": [ "__type__" ] + }, + "images": { + "type": "array", + "items": { + "type": "object", + "properties": { + "description": { "type": "string" }, + "reference": { "type": "string" }, + "rendered": { "type": "string" } + }, + "required": [ "description" ], + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false, + "required": [ "__type__", "clinfo", "glxinfo", "lspci", "wglinfo", "name", "results_version", "uname", "time_elapsed", "tests" ], + "definitions": { + "timeAttribute": { + "type": "object", + "description": "An element containing a start and end time", + "properties": { + "__type__": { "type": "string" }, + "start": { "type": "number" }, + "end": { "type": "number" } + }, + "additionalProperties": false, + "required": [ "__type__", "start", "end" ] + } + } +} diff --git a/unittests/framework/backends/shared.py b/unittests/framework/backends/shared.py index d9f5790..5232053 100644 --- a/unittests/framework/backends/shared.py +++ b/unittests/framework/backends/shared.py @@ -42,7 +42,7 @@ INITIAL_METADATA = { # changes. This does not contain piglit specifc objects, only strings, floats, # ints, and Nones (instead of JSON's null) JSON = { - "results_version": 9, + "results_version": 10, "time_elapsed": { "start": 1469638791.2351687, "__type__": "TimeAttribute", diff --git a/unittests/framework/backends/test_json_update.py b/unittests/framework/backends/test_json_update.py index c8e3ee6..b09413d 100644 --- a/unittests/framework/backends/test_json_update.py +++ b/unittests/framework/backends/test_json_update.py @@ -23,6 +23,7 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals ) +import base64 import os try: import simplejson as json @@ -201,3 +202,103 @@ class TestV8toV9(object): jsonschema.validate( json.loads(json.dumps(result, default=backends.json.piglit_encoder)), schema) + + +class TestV9toV10(object): + """Tests for Version 9 to version 10.""" + + ref = os.path.join(os.path.dirname(__file__), 'images', 'ref.png') + render = os.path.join(os.path.dirname(__file__), 'images', 'render.png') + + data = { + "results_version": 9, + "name": "test", + "options": { + "profile": ['quick'], + "dmesg": False, + "verbose": False, + "platform": "gbm", + "sync": False, + "valgrind": False, + "filter": [], + "concurrent": "all", + "test_count": 0, + "exclude_tests": [], + "exclude_filter": [], + "env": {}, + }, + "lspci": "stuff", + "uname": "more stuff", + "glxinfo": "and stuff", + "wglinfo": "stuff", + "clinfo": "stuff", + "tests": { + 'a@test': { + "time": { + 'start': 1.2, + 'end': 1.8, + '__type__': 'TimeAttribute' + }, + 'dmesg': '', + 'result': 'fail', + '__type__': 'TestResult', + 'command': '/a/command', + 'traceback': None, + 'out': '', + 'environment': 'A=variable', + 'returncode': 0, + 'err': '', + 'pid': [5], + 'subtests': { + '__type__': 'Subtests', + }, + 'exception': None, + 'images': [ + { + 'image_desc': "Test image", + 'image_ref': os.path.join(os.path.dirname(__file__), + 'images', 'ref.png'), + 'image_render': os.path.join(os.path.dirname(__file__), + 'images', 'render.png'), + }, + ], + } + }, + "time_elapsed": { + 'start': 1.2, + 'end': 1.8, + '__type__': 'TimeAttribute' + }, + '__type__': 'TestrunResult', + } + + @pytest.fixture + def result(self, tmpdir): + p = tmpdir.join('result.json') + p.write(json.dumps(self.data, default=backends.json.piglit_encoder)) + with p.open('r') as f: + return backends.json._update_nine_to_ten(backends.json._load(f)) + + def test_images(self, result): + """Converts the images attribute correctly.""" + test = result['tests']['a@test']['images'][0] + + with open(self.ref, 'rb') as f: + ref = base64.b64encode(f.read()).decode('utf-8') + assert test['reference'] == ref + + with open(self.render, 'rb') as f: + rend = base64.b64encode(f.read()).decode('utf-8') + assert test['rendered'] == rend + + assert test['description'] == \ + self.data['tests']['a@test']['images'][0]['image_desc'] + + def test_valid(self, result): + with open(os.path.join(os.path.dirname(__file__), 'schema', + 'piglit-10.json'), + 'r') as f: + schema = json.load(f) + jsonschema.validate( + json.loads(json.dumps(result, default=backends.json.piglit_encoder)), + schema) -- git-series 0.8.10 _______________________________________________ Piglit mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/piglit
