Commit: 93901e7f0a05ba471f3b4c0201500d9dfcd68c2c Author: Brecht Van Lommel Date: Fri May 10 23:00:35 2019 +0200 Branches: master https://developer.blender.org/rB93901e7f0a05ba471f3b4c0201500d9dfcd68c2c
Tests: speed up render tests by running multiple in the same process Blender startup time and shader compilation is a big factor when running hundreds of tests, so now all renders in the same ctest run in the same process. If a test crashes, the remaining tests in the same category will be marked as skipped. Benchmarked on a quad core with ctest -j8. cycles: 118.1s -> 94.3s eevee: 66.2s -> 29.2s workbench: 31.7s -> 8.6s =================================================================== M tests/python/cycles_render_tests.py M tests/python/eevee_render_tests.py M tests/python/modules/render_report.py M tests/python/opengl_draw_tests.py M tests/python/workbench_render_tests.py =================================================================== diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index 36f5459c2f7..a66409d85c3 100755 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -9,77 +9,76 @@ import subprocess import sys -def render_file(filepath, output_filepath): - dirname = os.path.dirname(filepath) - basedir = os.path.dirname(dirname) - subject = os.path.basename(dirname) - - frame_filepath = output_filepath + '0001.png' - - common_args = [ - "-noaudio", - "--factory-startup", - "--enable-autoexec", - filepath, - "-E", "CYCLES", - "-o", output_filepath, - "-F", "PNG"] +def render_files(filepaths, output_filepaths): + command = [BLENDER, "--background"] # OSL and GPU examples # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"] # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"] custom_args = os.getenv('CYCLESTEST_ARGS') custom_args = shlex.split(custom_args) if custom_args else [] - common_args += custom_args - - if subject == 'opengl': - command = [BLENDER, "--window-geometry", "0", "0", "1", "1"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_opengl.py")] - elif subject == 'bake': - command = [BLENDER, "--background"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_bake.py")] - elif subject == 'denoise_animation': - command = [BLENDER, "--background"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_denoise.py")] - else: - command = [BLENDER, "--background"] - command += common_args - command += ["-f", "1"] + for filepath, output_filepath in zip(filepaths, output_filepaths): + dirname = os.path.dirname(filepath) + basedir = os.path.dirname(dirname) + subject = os.path.basename(dirname) + + frame_filepath = output_filepath + '0001.png' + + common_args = [ + "-noaudio", + "--factory-startup", + "--enable-autoexec", + filepath, + "-E", "CYCLES", + "-o", output_filepath, + "-F", "PNG"] + + common_args += custom_args + + if subject == 'bake': + command.extend(common_args) + command.extend(['--python', os.path.join(basedir, "util", "render_bake.py")]) + elif subject == 'denoise_animation': + command.extend(common_args) + command.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")]) + else: + command.extend(common_args) + command.extend(["-f", "1"]) + + error = None try: # Success output = subprocess.check_output(command) - if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(output.decode("utf-8")) - return None except subprocess.CalledProcessError as e: # Error - if os.path.exists(frame_filepath): - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(e.output.decode("utf-8")) - if b"Error: engine not found" in e.output: - return "NO_ENGINE" - elif b"blender probably wont start" in e.output: - return "NO_START" - return "CRASH" + error = "CRASH" except BaseException as e: # Crash - if os.path.exists(frame_filepath): - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) - print(e) - return "CRASH" + print(e.decode("utf-8")) + error = "CRASH" + + # Detect missing filepaths and consider those errors + errors = [] + for output_filepath in output_filepaths: + frame_filepath = output_filepath + '0001.png' + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) + errors.append(None) + else: + errors.append(error) + error = 'SKIPPED' + return errors def create_argparse(): parser = argparse.ArgumentParser() @@ -108,7 +107,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("cycles_renders") report.set_compare_engines('cycles', 'eevee') - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, render_files) sys.exit(not ok) diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py index c0536e05164..d2c37dbcb2e 100755 --- a/tests/python/eevee_render_tests.py +++ b/tests/python/eevee_render_tests.py @@ -49,57 +49,61 @@ if inside_blender: sys.exit(1) -def render_file(filepath, output_filepath): - dirname = os.path.dirname(filepath) - basedir = os.path.dirname(dirname) - subject = os.path.basename(dirname) - - frame_filepath = output_filepath + '0001.png' - +def render_files(filepaths, output_filepaths): command = [ BLENDER, "--background", "-noaudio", "--factory-startup", - "--enable-autoexec", - filepath, - "-E", "BLENDER_EEVEE", - "-P", - os.path.realpath(__file__), - "-o", output_filepath, - "-F", "PNG", - "-f", "1"] + "--enable-autoexec"] + + for filepath, output_filepath in zip(filepaths, output_filepaths): + frame_filepath = output_filepath + '0001.png' + if os.path.exists(frame_filepath): + os.remove(frame_filepath) + command.extend([ + filepath, + "-E", "BLENDER_EEVEE", + "-P", + os.path.realpath(__file__), + "-o", output_filepath, + "-F", "PNG", + "-f", "1"]) + + error = None try: # Success output = subprocess.check_output(command) - if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(output.decode("utf-8")) - return None except subprocess.CalledProcessError as e: # Error - if os.path.exists(frame_filepath): - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(e.output.decode("utf-8")) - if b"Error: engine not found" in e.output: - return "NO_ENGINE" - elif b"blender probably wont start" in e.output: - return "NO_START" - return "CRASH" + error = "CRASH" except BaseException as e: # Crash - if os.path.exists(frame_filepath): - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) - print(e) - return "CRASH" + print(e.decode("utf-8")) + error = "CRASH" + + # Detect missing filepaths and consider those errors + errors = [] + for output_filepath in output_filepaths: + frame_filepath = output_filepath + '0001.png' + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) + errors.append(None) + else: + errors.append(error) + error = 'SKIPPED' + + return errors def create_argparse(): @@ -129,7 +133,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("eevee_renders") report.set_compare_engines('eevee', 'cycles') - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, render_files) sys.exit(not ok) diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py index 5a2baa354c7..7dfc74904a7 100755 --- a/tests/python/modules/render_report.py +++ b/tests/python/modules/render_report.py @@ -374,41 +374,48 @@ class Report: return not failed - def _run_test(self, filepath, render_cb): - testname = test_get_name(filepath) - print_message(testname, 'SUCCESS', 'RUN') - time_start = time.time() - tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname) + def _run_tests(self, filepaths, render_cb): + # Run all tests together for performance, since Blender + # startup time is a significant factor. + tmp_filepaths = [] + for filepath in filepaths: + testname = test_get_name(filepath) + print_message(testname, 'SUCCESS', 'RUN') + tmp_filepaths.append(os.path.join(self.output_dir, "tmp_" + testname)) - error = render_cb(filepath, tmp_filepath) - status = "FAIL" - if not error: - if not self._diff_output(filepath, tmp_filepath): - error = "VERIFY" + run_errors = render_cb(filepaths, tmp_filepaths) + errors = [] - if os.path.exists(tmp_filepath): - os.remove(tmp_filepath) + for error, filepath, tmp_filepath in zip(run_errors, filepaths, tmp_filepaths): + if not error: + if os.path.getsize(tmp_filepath) == 0: + error = "VERIFY" + elif not self._diff_output(filepath, tmp_filepath): + error = "VERIFY" - time_end = time.time() - elapsed_ms = int((time_end - time_start) * 1000) - if not error: - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'SUCCESS', 'OK') - else: - if error == "NO_ENGINE": - print_message("Can't perform tests because the render engine failed to load!") - return error - elif error == "NO_START": - print_message('Can not perform tests because blender fails to start.', - 'Make sure INSTALL target was r @@ Diff output truncated at 10240 characters. @@ _______________________________________________ Bf-blender-cvs mailing list Bf-blender-cvs@blender.org https://lists.blender.org/mailman/listinfo/bf-blender-cvs