Commit: 0f3f093d3b3d1b2fb9593233f23665d3135bea2d Author: Brecht Van Lommel Date: Thu Aug 3 16:41:50 2017 +0200 Branches: master https://developer.blender.org/rB0f3f093d3b3d1b2fb9593233f23665d3135bea2d
Cycles: add HTML report to inspect failed test images. Shows new, reference and diff renders, with mouse hover to flip between new and ref for easy comparison. This generates a report.html in build_dir/tests/cycles, stored along with the new and diff images. Differential Revision: https://developer.blender.org/D2770 =================================================================== M tests/python/CMakeLists.txt M tests/python/cycles_render_tests.py =================================================================== diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index f2fea48d9f4..4ba89c2805d 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -518,6 +518,7 @@ if(WITH_CYCLES) -blender "$<TARGET_FILE:blender>" -testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}" -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/cycles" ) else() add_test( @@ -526,6 +527,7 @@ if(WITH_CYCLES) -blender "$<TARGET_FILE:blender>" -testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}" -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/cycles" ) endif() endmacro() diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index a030cc5e0de..ea84f27ab7e 100755 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -2,7 +2,9 @@ # Apache License, Version 2.0 import argparse +import glob import os +import pathlib import shutil import subprocess import sys @@ -24,7 +26,7 @@ class COLORS_DUMMY: COLORS = COLORS_DUMMY -def printMessage(type, status, message): +def print_message(message, type=None, status=''): if type == 'SUCCESS': print(COLORS.GREEN, end="") elif type == 'FAILURE': @@ -109,20 +111,126 @@ def test_get_name(filepath): filename = os.path.basename(filepath) return os.path.splitext(filename)[0] - -def verify_output(filepath): +def test_get_images(filepath): testname = test_get_name(filepath) dirpath = os.path.dirname(filepath) - reference_dirpath = os.path.join(dirpath, "reference_renders") - reference_image = os.path.join(reference_dirpath, testname + ".png") - failed_image = os.path.join(reference_dirpath, testname + ".fail.png") - if not os.path.exists(reference_image): + ref_dirpath = os.path.join(dirpath, "reference_renders") + ref_img = os.path.join(ref_dirpath, testname + ".png") + new_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath)) + if not os.path.exists(new_dirpath): + os.makedirs(new_dirpath) + new_img = os.path.join(new_dirpath, testname + ".png") + diff_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "diff") + if not os.path.exists(diff_dirpath): + os.makedirs(diff_dirpath) + diff_img = os.path.join(diff_dirpath, testname + ".diff.png") + return ref_img, new_img, diff_img + + +class Report: + def __init__(self, testname): + self.failed_tests = "" + self.passed_tests = "" + self.testname = testname + + def output(self): + # write intermediate data for single test + outdir = os.path.join(OUTDIR, self.testname) + f = open(os.path.join(outdir, "failed.data"), "w") + f.write(self.failed_tests) + f.close() + + f = open(os.path.join(outdir, "passed.data"), "w") + f.write(self.passed_tests) + f.close() + + # gather intermediate data for all tests + failed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/failed.data"))) + passed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/passed.data"))) + + failed_tests = "" + passed_tests = "" + + for filename in failed_data: + failed_tests += open(os.path.join(OUTDIR, filename), "r").read() + for filename in passed_data: + passed_tests += open(os.path.join(OUTDIR, filename), "r").read() + + # write html for all tests + self.html = """ +<html> +<head> + <title>Cycles Test Report</title> + <style> + img {{ image-rendering: pixelated; width: 256; background-color: #000; }} + table td:first-child {{ width: 100%; }} + </style> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css"> +</head> +<body> + <div class="container"> + <br/> + <h1>Cycles Test Report</h1> + <br/> + <table class="table table-striped"> + <thead class="thead-default"> + <tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th> + </thead> + {}{} + </table> + <br/> + </div> +</body> +</html> + """ . format(failed_tests, passed_tests) + + filepath = os.path.join(OUTDIR, "report.html") + f = open(filepath, "w") + f.write(self.html) + f.close() + + print_message("Report saved to: " + pathlib.Path(filepath).as_uri()) + + def add_test(self, filepath, error): + name = test_get_name(filepath) + + ref_img, new_img, diff_img = test_get_images(filepath) + + status = error if error else "" + style = """ style="background-color: #f99;" """ if error else "" + + new_url = pathlib.Path(new_img).as_uri() + ref_url = pathlib.Path(ref_img).as_uri() + diff_url = pathlib.Path(diff_img).as_uri() + + test_html = """ + <tr{}> + <td><b>{}</b><br/>{}<br/>{}</td> + <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';"></td> + <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';"></td> + <td><img src="{}"></td> + </tr>""" . format(style, name, self.testname, status, + new_url, ref_url, new_url, + ref_url, new_url, ref_url, + diff_url) + + if error: + self.failed_tests += test_html + else: + self.passed_tests += test_html + + +def verify_output(report, filepath): + ref_img, new_img, diff_img = test_get_images(filepath) + if not os.path.exists(ref_img): return False + + # diff test with threshold command = ( IDIFF, - "-fail", "0.015", + "-fail", "0.016", "-failpercent", "1", - reference_image, + ref_img, TEMP_FILE, ) try: @@ -130,47 +238,66 @@ def verify_output(filepath): failed = False except subprocess.CalledProcessError as e: if VERBOSE: - print(e.output.decode("utf-8")) + print_message(e.output.decode("utf-8")) failed = e.returncode != 1 - if failed: - shutil.copy(TEMP_FILE, failed_image) - elif os.path.exists(failed_image): - os.remove(failed_image) + + # generate diff image + command = ( + IDIFF, + "-o", diff_img, + "-abs", "-scale", "16", + ref_img, + TEMP_FILE + ) + + try: + subprocess.check_output(command) + except subprocess.CalledProcessError as e: + if VERBOSE: + print_message(e.output.decode("utf-8")) + + # copy new image + if os.path.exists(new_img): + os.remove(new_img) + if os.path.exists(TEMP_FILE): + shutil.copy(TEMP_FILE, new_img) + return not failed -def run_test(filepath): +def run_test(report, filepath): testname = test_get_name(filepath) spacer = "." * (32 - len(testname)) - printMessage('SUCCESS', 'RUN', testname) + print_message(testname, 'SUCCESS', 'RUN') time_start = time.time() error = render_file(filepath) status = "FAIL" if not error: - if not verify_output(filepath): + if not verify_output(report, filepath): error = "VERIFY" time_end = time.time() elapsed_ms = int((time_end - time_start) * 1000) if not error: - printMessage('SUCCESS', 'OK', "{} ({} ms)" . - format(testname, elapsed_ms)) + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'SUCCESS', 'OK') else: if error == "NO_CYCLES": - print("Can't perform tests because Cycles failed to load!") - return False + print_message("Can't perform tests because Cycles failed to load!") + return error elif error == "NO_START": - print('Can not perform tests because blender fails to start.', + print_message('Can not perform tests because blender fails to start.', 'Make sure INSTALL target was run.') - return False + return error elif error == 'VERIFY': - print("Rendered result is different from reference image") + print_message("Rendered result is different from reference image") else: - print("Unknown error %r" % error) - printMessage('FAILURE', 'FAILED', "{} ({} ms)" . - format(testname, elapsed_ms)) + print_message("Unknown error %r" % error) + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'FAILURE', 'FAILED') return error + def blend_list(path): for dirpath, dirnames, filenames in os.walk(path): for filename in filenames: @@ -178,17 +305,18 @@ def blend_list(path): filepath = os.path.join(dirpath, filename) yield filepath - def run_all_tests(dirpath): passed_tests = [] failed_tests = [] all_files = list(blend_list(dirpath)) all_files.sort() - printMessage('SUCCESS', "==========", - "Running {} tests from 1 test case." . format(len(all_files))) + report = Report(os.path.basename(dirpath)) + print_message("Running {} tests from 1 test case." . + format(len(all_files)), + 'SUCCESS', "==========") time_start = time.time() for filepath in all_files: - error = run_test(filepath) + error = run_test(report, filepath) testname = test_get_name(filepath) if error: if error == "NO_CYCLES": @@ -198,28 +326,33 @@ def run_all_tests(dirpath): failed_tests.append(testname) else: passed_tests.append(testname) + report.add_test(filepath, error) time_end = time.time() elapsed_ms = int((time_end - time_start) * 1000) - print("") - printMessage('SUCCESS', "==========", - "{} tests from 1 test case ran. ({} ms total)" . - format(len(all_files), elapsed_ms)) - printMessage('SUCCESS', 'PASSED', "{} tes @@ Diff output truncated at 10240 characters. @@ _______________________________________________ Bf-blender-cvs mailing list Bf-blender-cvs@blender.org https://lists.blender.org/mailman/listinfo/bf-blender-cvs