https://github.com/ilovepi updated https://github.com/llvm/llvm-project/pull/163282
>From 59fcff87c9d93f4ea2a53e90e0f8cbd6bf89a24a Mon Sep 17 00:00:00 2001 From: Paul Kirth <[email protected]> Date: Mon, 13 Oct 2025 14:29:34 -0700 Subject: [PATCH 1/8] [clang][utils] Add auto mode to reduction script The existing script allows us to run creduce/cvise automatically on clang reproducers. However, llvm-reduce is much more efficient for middle or backend issues. In the early stages, we can just determine which phase the crash happens, and choose the best tool. For now, we only support simple test scripts for opt and llc, but this should cover many more cases. This feature is behind a new flag `--auto`, and the default behavior of the script is otherwise unchanged. --- clang/utils/reduce-clang-crash.py | 200 ++++++++++++++++++++++++++++-- 1 file changed, 188 insertions(+), 12 deletions(-) diff --git a/clang/utils/reduce-clang-crash.py b/clang/utils/reduce-clang-crash.py index 075ff91af2f5a..8c4b58fdbeab5 100755 --- a/clang/utils/reduce-clang-crash.py +++ b/clang/utils/reduce-clang-crash.py @@ -18,12 +18,22 @@ import subprocess import shlex import tempfile -import shutil import multiprocessing +from enum import StrEnum, auto +from typing import List, Optional verbose = False creduce_cmd = None clang_cmd = None +opt_cmd = None +llc_cmd = None + + +class FailureType(StrEnum): + FrontEnd = auto() + MiddleEnd = auto() + BackEnd = auto() + Unknown = auto() def verbose_print(*args, **kwargs): @@ -73,6 +83,44 @@ def write_to_script(text, filename): os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC) +def extract_opt_level(args_list: List[str]) -> Optional[str]: + """ + Finds the last optimization flag (-O...) from a list of arguments. + + Args: + args_list: A list of string arguments passed to the compiler. + + Returns: + The last matching optimization flag string if found, otherwise None. + """ + valid_opt_flags = {"-O0", "-O1", "-O2", "-O3", "-Os", "-Oz", "-Og", "-Ofast"} + + # Iterate in reverse to find the last occurrence + for arg in reversed(args_list): + if arg in valid_opt_flags: + return arg + return None + + +def remove_first_line(file_path): + """ + Removes the first line from a specified file. + """ + try: + with open(file_path, "r") as f: + lines = f.readlines() + + # If the file is not empty, write all lines except the first one back. + if lines: + with open(file_path, "w") as f: + f.writelines(lines[1:]) + + except FileNotFoundError: + print(f"Error: File '{file_path}' not found.") + except Exception as e: + print(f"An error occurred: {e}") + + class Reduce(object): def __init__(self, crash_script, file_to_reduce, creduce_flags): crash_script_name, crash_script_ext = os.path.splitext(crash_script) @@ -88,6 +136,9 @@ def __init__(self, crash_script, file_to_reduce, creduce_flags): self.expected_output = [] self.needs_stack_trace = False self.creduce_flags = ["--tidy"] + creduce_flags + self.opt = opt_cmd + self.llc = llc_cmd + self.ir_file = "crash.ll" self.read_clang_args(crash_script, file_to_reduce) self.read_expected_output() @@ -189,22 +240,30 @@ def skip_function(func_name): self.expected_output = result - def check_expected_output(self, args=None, filename=None): + def check_expected_output(self, cmd=None, args=None, filename=None): + if not cmd: + cmd = self.clang if not args: args = self.clang_args if not filename: filename = self.file_to_reduce p = subprocess.Popen( - self.get_crash_cmd(args=args, filename=filename), + self.get_crash_cmd(cmd=cmd, args=args, filename=filename), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) crash_output, _ = p.communicate() return all(msg in crash_output.decode("utf-8") for msg in self.expected_output) - def write_interestingness_test(self): + def write_interestingness_test(self, cmd=None, use_llvm_reduce=False): print("\nCreating the interestingness test...") + if not cmd: + cmd = self.get_crash_cmd() + + # llvm-reduce interestingness tests take the file as the first argument. + # NOTE: this cannot be escaped by quote_cmd(), since it needs expansion. + filename = '< "$1"' if use_llvm_reduce else "" # Disable symbolization if it's not required to avoid slow symbolization. disable_symbolization = "" @@ -213,32 +272,39 @@ def write_interestingness_test(self): output = """#!/bin/bash %s -if %s >& t.log ; then +if %s %s >& t.log ; then exit 1 fi """ % ( disable_symbolization, - quote_cmd(self.get_crash_cmd()), + quote_cmd(cmd), + filename, ) for msg in self.expected_output: output += "grep -F %s t.log || exit 1\n" % shlex.quote(msg) write_to_script(output, self.testfile) - self.check_interestingness() + self.check_interestingness(cmd, use_llvm_reduce=use_llvm_reduce) - def check_interestingness(self): - testfile = os.path.abspath(self.testfile) + def check_interestingness(self, cmd, use_llvm_reduce=False): + test_cmd = [os.path.abspath(self.testfile)] + # llvm-reduce interestingness tests take the file as the first arg. + if use_llvm_reduce: + test_cmd += [self.ir_file] # Check that the test considers the original file interesting - returncode = subprocess.call(testfile, stdout=subprocess.DEVNULL) - if returncode: + result = subprocess.run(args=test_cmd, stdout=subprocess.DEVNULL) + if result.returncode: sys.exit("The interestingness test does not pass for the original file.") # Check that an empty file is not interesting # Instead of modifying the filename in the test file, just run the command with tempfile.NamedTemporaryFile() as empty_file: - is_interesting = self.check_expected_output(filename=empty_file.name) + new_args = cmd[1:] if use_llvm_reduce else cmd[1:-1] + is_interesting = self.check_expected_output( + cmd=cmd[0], args=new_args, filename=empty_file.name + ) if is_interesting: sys.exit("The interestingness test passes for an empty file.") @@ -427,11 +493,76 @@ def run_creduce(self): print("\n\nctrl-c detected, killed reduction tool") p.kill() + def run_llvm_reduce(self): + full_llvm_reduce_cmd = [ + llvm_reduce_cmd, + f"--test={self.testfile}", + self.ir_file, + ] + print("\nRunning llvm-reduce tool...") + verbose_print(quote_cmd(full_llvm_reduce_cmd)) + try: + p = subprocess.Popen(full_llvm_reduce_cmd) + p.communicate() + except KeyboardInterrupt: + # Hack to kill C-Reduce because it jumps into its own pgid + print("\n\nctrl-c detected, killed reduction tool") + p.kill() + + def classify_crash(self) -> FailureType: + print("classifying crash ...") + if self.check_expected_output(args=self.clang_args + ["-fsyntax-only"]): + print("Found Frontend Crash") + return FailureType.FrontEnd + + print("Found Middle/Backend failure") + args = self.clang_args + [ + "-mllvm", + "--print-on-crash", + "-mllvm", + f"--print-on-crash-path={self.ir_file}", + "-mllvm", + "--print-module-scope", + ] + + if not self.check_expected_output(args=args): + sys.exit("The interestingness test does not pass with '--print-on-crash'.") + + # The output from --print-on-crash has an invalid first line (pass name). + remove_first_line(self.ir_file) + + self.opt_level = extract_opt_level(self.clang_args) or "-O2" + + if self.check_expected_output( + cmd=self.opt, + args=[self.opt_level, "-disable-output"], + filename=self.ir_file, + ): + print("Found MiddleEnd Crash") + return FailureType.MiddleEnd + if self.check_expected_output( + cmd=self.llc, args=[self.opt_level], filename=self.ir_file + ): + print("Found BackEnd Crash") + return FailureType.BackEnd + print("Found Unknow Crash Type. Falling back to creduce") + return FailureType.Unknown + + def reduce_ir_crash(self, new_cmd: List[str]): + print("Writing interestingness test...") + self.write_interestingness_test(cmd=new_cmd, use_llvm_reduce=True) + print("Starting llvm-reduce with llc test case") + self.run_llvm_reduce() + print("Done Reducing IR file.") + def main(): global verbose global creduce_cmd + global llvm_reduce_cmd global clang_cmd + global opt_cmd + global llc_cmd parser = ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter) parser.add_argument( @@ -453,6 +584,20 @@ def main(): help="The path to the `clang` executable. " "By default uses the llvm-bin directory.", ) + parser.add_argument( + "--opt", + dest="opt", + type=str, + help="The path to the `opt` executable. " + "By default uses the llvm-bin directory.", + ) + parser.add_argument( + "--llc", + dest="llc", + type=str, + help="The path to the `llc` executable. " + "By default uses the llvm-bin directory.", + ) parser.add_argument( "--creduce", dest="creduce", @@ -460,6 +605,19 @@ def main(): help="The path to the `creduce` or `cvise` executable. " "Required if neither `creduce` nor `cvise` are on PATH.", ) + parser.add_argument( + "--llvm-reduce", + dest="llvm_reduce", + type=str, + help="The path to the `llvm-reduce` executable. " + "By default uses the llvm-bin directory.", + ) + parser.add_argument( + "--auto", + action="store_true", + help="Use auto reduction mode, that uses `creduce`/`cvise`" + "for frontend crashes and llvm-reduce for middle/backend crashes.", + ) parser.add_argument("-v", "--verbose", action="store_true") args, creduce_flags = parser.parse_known_args() verbose = args.verbose @@ -468,6 +626,8 @@ def main(): if not creduce_cmd: creduce_cmd = check_cmd("creduce", None, args.creduce) clang_cmd = check_cmd("clang", llvm_bin, args.clang) + opt_cmd = check_cmd("opt", llvm_bin, args.opt) + llc_cmd = check_cmd("llc", llvm_bin, args.llc) crash_script = check_file(args.crash_script[0]) file_to_reduce = check_file(args.file_to_reduce[0]) @@ -476,6 +636,22 @@ def main(): creduce_flags += ["--n", str(max(4, multiprocessing.cpu_count() // 2))] r = Reduce(crash_script, file_to_reduce, creduce_flags) + if args.auto: + crash_type = r.classify_crash() + match crash_type: + case FailureType.FrontEnd | FailureType.Unknown: + print("Starting reduction with creduce/cvise") + pass + case FailureType.MiddleEnd: + # TODO: parse the exact pass from the backtrace and set the + # pass pipeline directly. + new_cmd = [opt_cmd, "-disable-output", r.opt_level] + r.reduce_ir_crash(new_cmd) + return + case FailureType.BackEnd: + new_cmd = [llc_cmd, r.opt_level] + r.reduce_ir_crash(new_cmd) + return r.simplify_clang_args() r.write_interestingness_test() >From fd31ac4f8b32a651c61cbec8ca3f7eb2c42b7354 Mon Sep 17 00:00:00 2001 From: Paul Kirth <[email protected]> Date: Mon, 13 Oct 2025 18:37:51 -0700 Subject: [PATCH 2/8] Fix up detection for backend failures --- clang/utils/reduce-clang-crash.py | 33 ++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/clang/utils/reduce-clang-crash.py b/clang/utils/reduce-clang-crash.py index 8c4b58fdbeab5..2430d1d03ea3e 100755 --- a/clang/utils/reduce-clang-crash.py +++ b/clang/utils/reduce-clang-crash.py @@ -516,6 +516,23 @@ def classify_crash(self) -> FailureType: return FailureType.FrontEnd print("Found Middle/Backend failure") + + self.opt_level = extract_opt_level(self.clang_args) or "-O2" + # Try running w/ -emit-llvm to generate an IR file, + # if it succeeds we have a backend failure, and can use llc. + if not self.check_expected_output( + args=self.clang_args + ["-emit-llvm", "-o", self.ir_file]): + print("Checking llc for failure") + if self.check_expected_output( + cmd=self.llc, args=[self.opt_level, "-filetype=obj"], filename=self.ir_file + ): + print("Found BackEnd Crash") + return FailureType.BackEnd + elif os.path.exists(self.ir_file): + # clean up the IR file if we generated it, but won't use it. + print(f"clean up {self.ir_file}, since we can't repro w/ llc") + os.remove(self.ir_file) + args = self.clang_args + [ "-mllvm", "--print-on-crash", @@ -531,8 +548,9 @@ def classify_crash(self) -> FailureType: # The output from --print-on-crash has an invalid first line (pass name). remove_first_line(self.ir_file) - self.opt_level = extract_opt_level(self.clang_args) or "-O2" - + # TODO: parse the exact pass from the backtrace and set the pass + # pipeline directly via -passes="...". + print("Checking opt for failure") if self.check_expected_output( cmd=self.opt, args=[self.opt_level, "-disable-output"], @@ -540,12 +558,7 @@ def classify_crash(self) -> FailureType: ): print("Found MiddleEnd Crash") return FailureType.MiddleEnd - if self.check_expected_output( - cmd=self.llc, args=[self.opt_level], filename=self.ir_file - ): - print("Found BackEnd Crash") - return FailureType.BackEnd - print("Found Unknow Crash Type. Falling back to creduce") + print("Could not automatically detect crash type, falling back to creduce/cvise.") return FailureType.Unknown def reduce_ir_crash(self, new_cmd: List[str]): @@ -643,13 +656,11 @@ def main(): print("Starting reduction with creduce/cvise") pass case FailureType.MiddleEnd: - # TODO: parse the exact pass from the backtrace and set the - # pass pipeline directly. new_cmd = [opt_cmd, "-disable-output", r.opt_level] r.reduce_ir_crash(new_cmd) return case FailureType.BackEnd: - new_cmd = [llc_cmd, r.opt_level] + new_cmd = [llc_cmd, "-filetype=obj", r.opt_level] r.reduce_ir_crash(new_cmd) return >From 95b0c4ce98786c60b057eb2b8461cc080e7e1488 Mon Sep 17 00:00:00 2001 From: Paul Kirth <[email protected]> Date: Tue, 14 Oct 2025 09:57:31 -0700 Subject: [PATCH 3/8] Refactor IR extraction and checking --- clang/utils/reduce-clang-crash.py | 52 +++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/clang/utils/reduce-clang-crash.py b/clang/utils/reduce-clang-crash.py index 2430d1d03ea3e..d18c9bd232f93 100755 --- a/clang/utils/reduce-clang-crash.py +++ b/clang/utils/reduce-clang-crash.py @@ -505,12 +505,11 @@ def run_llvm_reduce(self): p = subprocess.Popen(full_llvm_reduce_cmd) p.communicate() except KeyboardInterrupt: - # Hack to kill C-Reduce because it jumps into its own pgid print("\n\nctrl-c detected, killed reduction tool") p.kill() def classify_crash(self) -> FailureType: - print("classifying crash ...") + print("Classifying crash ...") if self.check_expected_output(args=self.clang_args + ["-fsyntax-only"]): print("Found Frontend Crash") return FailureType.FrontEnd @@ -518,13 +517,20 @@ def classify_crash(self) -> FailureType: print("Found Middle/Backend failure") self.opt_level = extract_opt_level(self.clang_args) or "-O2" + backend_result = self.check_backend() + if backend_result == FailureType.BackEnd: + return backend_result + # Try running w/ -emit-llvm to generate an IR file, # if it succeeds we have a backend failure, and can use llc. if not self.check_expected_output( - args=self.clang_args + ["-emit-llvm", "-o", self.ir_file]): + args=self.clang_args + ["-emit-llvm", "-o", self.ir_file] + ): print("Checking llc for failure") if self.check_expected_output( - cmd=self.llc, args=[self.opt_level, "-filetype=obj"], filename=self.ir_file + cmd=self.llc, + args=[self.opt_level, "-filetype=obj"], + filename=self.ir_file, ): print("Found BackEnd Crash") return FailureType.BackEnd @@ -533,6 +539,37 @@ def classify_crash(self) -> FailureType: print(f"clean up {self.ir_file}, since we can't repro w/ llc") os.remove(self.ir_file) + # The IR file will be in 'self.ir_file' + self.extract_crashing_ir() + return self.check_middle_end() + + def check_llc_failure(self) -> bool: + return self.check_expected_output( + cmd=self.llc, args=[self.opt_level, "-filetype=obj"], filename=self.ir_file + ) + + def extract_backend_ir(self) -> bool: + return not self.check_expected_output( + args=self.clang_args + ["-emit-llvm", "-o", self.ir_file] + ) + + def check_backend(self) -> Optional[FailureType]: + # Try running w/ -emit-llvm to generate an IR file, + # if it succeeds we have a backend failure, and can use llc. + if self.extract_backend_ir(): + print("Checking llc for failure") + if self.check_llc_failure(): + print("Found BackEnd Crash") + return FailureType.BackEnd + elif os.path.exists(self.ir_file): + # clean up the IR file if we generated it, but won't use it. + print(f"clean up {self.ir_file}, since we can't repro w/ llc") + os.remove(self.ir_file) + + def extract_crashing_ir(self): + """ + Extract IR just before the crash using --print-on-crash + """ args = self.clang_args + [ "-mllvm", "--print-on-crash", @@ -548,6 +585,7 @@ def classify_crash(self) -> FailureType: # The output from --print-on-crash has an invalid first line (pass name). remove_first_line(self.ir_file) + def check_middle_end(self) -> FailureType: # TODO: parse the exact pass from the backtrace and set the pass # pipeline directly via -passes="...". print("Checking opt for failure") @@ -558,7 +596,9 @@ def classify_crash(self) -> FailureType: ): print("Found MiddleEnd Crash") return FailureType.MiddleEnd - print("Could not automatically detect crash type, falling back to creduce/cvise.") + print( + "Could not automatically detect crash type, falling back to creduce/cvise." + ) return FailureType.Unknown def reduce_ir_crash(self, new_cmd: List[str]): @@ -660,7 +700,7 @@ def main(): r.reduce_ir_crash(new_cmd) return case FailureType.BackEnd: - new_cmd = [llc_cmd, "-filetype=obj", r.opt_level] + new_cmd = [llc_cmd, "-filetype=obj", r.opt_level] r.reduce_ir_crash(new_cmd) return >From 47ec60cf76c7603b55f7549434f9a98563afe498 Mon Sep 17 00:00:00 2001 From: Paul Kirth <[email protected]> Date: Thu, 30 Oct 2025 12:25:10 -0500 Subject: [PATCH 4/8] Remove duplicated code after refactoring --- clang/utils/reduce-clang-crash.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/clang/utils/reduce-clang-crash.py b/clang/utils/reduce-clang-crash.py index d18c9bd232f93..2646ee3a4fa82 100755 --- a/clang/utils/reduce-clang-crash.py +++ b/clang/utils/reduce-clang-crash.py @@ -521,24 +521,6 @@ def classify_crash(self) -> FailureType: if backend_result == FailureType.BackEnd: return backend_result - # Try running w/ -emit-llvm to generate an IR file, - # if it succeeds we have a backend failure, and can use llc. - if not self.check_expected_output( - args=self.clang_args + ["-emit-llvm", "-o", self.ir_file] - ): - print("Checking llc for failure") - if self.check_expected_output( - cmd=self.llc, - args=[self.opt_level, "-filetype=obj"], - filename=self.ir_file, - ): - print("Found BackEnd Crash") - return FailureType.BackEnd - elif os.path.exists(self.ir_file): - # clean up the IR file if we generated it, but won't use it. - print(f"clean up {self.ir_file}, since we can't repro w/ llc") - os.remove(self.ir_file) - # The IR file will be in 'self.ir_file' self.extract_crashing_ir() return self.check_middle_end() >From 9fd4272430699659281775b6edc2fb276fdb9781 Mon Sep 17 00:00:00 2001 From: Paul Kirth <[email protected]> Date: Thu, 30 Oct 2025 15:21:39 -0500 Subject: [PATCH 5/8] Don't delete the crash file --- clang/utils/reduce-clang-crash.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/clang/utils/reduce-clang-crash.py b/clang/utils/reduce-clang-crash.py index 2646ee3a4fa82..25578f89a90c9 100755 --- a/clang/utils/reduce-clang-crash.py +++ b/clang/utils/reduce-clang-crash.py @@ -543,10 +543,6 @@ def check_backend(self) -> Optional[FailureType]: if self.check_llc_failure(): print("Found BackEnd Crash") return FailureType.BackEnd - elif os.path.exists(self.ir_file): - # clean up the IR file if we generated it, but won't use it. - print(f"clean up {self.ir_file}, since we can't repro w/ llc") - os.remove(self.ir_file) def extract_crashing_ir(self): """ >From dacadf73e7d0e82a82f409170f9f511a3d88a306 Mon Sep 17 00:00:00 2001 From: Paul Kirth <[email protected]> Date: Tue, 4 Nov 2025 16:39:42 -0600 Subject: [PATCH 6/8] Address comments - remove reliance on default arguments - restructure IR extraction - fall back to using clang w/ llvm-reduce - don't assume all -O options are going to work with other tools --- clang/utils/reduce-clang-crash.py | 115 +++++++++++++++++++----------- 1 file changed, 73 insertions(+), 42 deletions(-) diff --git a/clang/utils/reduce-clang-crash.py b/clang/utils/reduce-clang-crash.py index 25578f89a90c9..4648ddce31c16 100755 --- a/clang/utils/reduce-clang-crash.py +++ b/clang/utils/reduce-clang-crash.py @@ -83,7 +83,7 @@ def write_to_script(text, filename): os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC) -def extract_opt_level(args_list: List[str]) -> Optional[str]: +def extract_opt_level(args_list: List[str]) -> str: """ Finds the last optimization flag (-O...) from a list of arguments. @@ -91,15 +91,15 @@ def extract_opt_level(args_list: List[str]) -> Optional[str]: args_list: A list of string arguments passed to the compiler. Returns: - The last matching optimization flag string if found, otherwise None. + The last matching optimization flag string if found, otherwise -O2. """ - valid_opt_flags = {"-O0", "-O1", "-O2", "-O3", "-Os", "-Oz", "-Og", "-Ofast"} + valid_opt_flags = {"-O0", "-O1", "-O2", "-O3", "-Os", "-Oz"} # Iterate in reverse to find the last occurrence for arg in reversed(args_list): if arg in valid_opt_flags: return arg - return None + return "-O2" def remove_first_line(file_path): @@ -240,16 +240,10 @@ def skip_function(func_name): self.expected_output = result - def check_expected_output(self, cmd=None, args=None, filename=None): - if not cmd: - cmd = self.clang - if not args: - args = self.clang_args - if not filename: - filename = self.file_to_reduce - + def check_expected_output(self, cmd, args, filename): + cmd = self.get_crash_cmd(cmd=cmd, args=args, filename=filename) p = subprocess.Popen( - self.get_crash_cmd(cmd=cmd, args=args, filename=filename), + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) @@ -303,7 +297,7 @@ def check_interestingness(self, cmd, use_llvm_reduce=False): with tempfile.NamedTemporaryFile() as empty_file: new_args = cmd[1:] if use_llvm_reduce else cmd[1:-1] is_interesting = self.check_expected_output( - cmd=cmd[0], args=new_args, filename=empty_file.name + cmd[0], new_args, empty_file.name ) if is_interesting: sys.exit("The interestingness test passes for an empty file.") @@ -315,12 +309,16 @@ def clang_preprocess(self): cmd_preprocess_no_lines = cmd_preprocess + ["-P"] try: subprocess.check_call(cmd_preprocess_no_lines) - if self.check_expected_output(filename=tmpfile.name): + if self.check_expected_output( + self.clang, self.clang_args, tmpfile.name + ): print("Successfully preprocessed with line markers removed") shutil.copy(tmpfile.name, self.file_to_reduce) else: subprocess.check_call(cmd_preprocess) - if self.check_expected_output(filename=tmpfile.name): + if self.check_expected_output( + self.clang, self.clang_args, tmpfile.name + ): print("Successfully preprocessed without removing line markers") shutil.copy(tmpfile.name, self.file_to_reduce) else: @@ -359,7 +357,9 @@ def try_remove_args(self, args, msg=None, extra_arg=None, **kwargs): new_args.remove(extra_arg) new_args.append(extra_arg) - if new_args != args and self.check_expected_output(args=new_args): + if new_args != args and self.check_expected_output( + self.clang, new_args, self.file_to_reduce + ): if msg: verbose_print(msg) return new_args @@ -375,7 +375,7 @@ def try_remove_arg_by_index(self, args, index): del new_args[index] removed_arg += " " + args[index + 1] - if self.check_expected_output(args=new_args): + if self.check_expected_output(self.clang, new_args, self.file_to_reduce): verbose_print("Removed", removed_arg) return new_args, index return args, index + 1 @@ -510,13 +510,15 @@ def run_llvm_reduce(self): def classify_crash(self) -> FailureType: print("Classifying crash ...") - if self.check_expected_output(args=self.clang_args + ["-fsyntax-only"]): + if self.check_expected_output( + self.clang, self.clang_args + ["-fsyntax-only"], self.file_to_reduce + ): print("Found Frontend Crash") return FailureType.FrontEnd print("Found Middle/Backend failure") - self.opt_level = extract_opt_level(self.clang_args) or "-O2" + self.opt_level = extract_opt_level(self.clang_args) backend_result = self.check_backend() if backend_result == FailureType.BackEnd: return backend_result @@ -527,12 +529,14 @@ def classify_crash(self) -> FailureType: def check_llc_failure(self) -> bool: return self.check_expected_output( - cmd=self.llc, args=[self.opt_level, "-filetype=obj"], filename=self.ir_file + self.llc, [self.opt_level, "-filetype=obj"], self.ir_file ) def extract_backend_ir(self) -> bool: return not self.check_expected_output( - args=self.clang_args + ["-emit-llvm", "-o", self.ir_file] + self.clang, + self.clang_args + ["-emit-llvm", "-o", self.ir_file], + self.file_to_reduce, ) def check_backend(self) -> Optional[FailureType]: @@ -542,38 +546,67 @@ def check_backend(self) -> Optional[FailureType]: print("Checking llc for failure") if self.check_llc_failure(): print("Found BackEnd Crash") + self.new_cmd = [self.llc, self.opt_level, "-filetype=obj"] return FailureType.BackEnd - def extract_crashing_ir(self): + def extract_crashing_ir(self, use_print_on_crash=False): """ Extract IR just before the crash using --print-on-crash """ - args = self.clang_args + [ - "-mllvm", - "--print-on-crash", - "-mllvm", - f"--print-on-crash-path={self.ir_file}", - "-mllvm", - "--print-module-scope", - ] + if use_print_on_crash: + args = self.clang_args + [ + "-mllvm", + "--print-on-crash", + "-mllvm", + f"--print-on-crash-path={self.ir_file}", + "-mllvm", + "--print-module-scope", + ] - if not self.check_expected_output(args=args): - sys.exit("The interestingness test does not pass with '--print-on-crash'.") + if not self.check_expected_output(self.clang, args, self.file_to_reduce): + sys.exit( + "The interestingness test does not pass with '--print-on-crash'." + ) - # The output from --print-on-crash has an invalid first line (pass name). - remove_first_line(self.ir_file) + # The output from --print-on-crash has an invalid first line (pass name). + remove_first_line(self.ir_file) + return + args = self.clang_args + [ + "-disable-llvm-passes", + "-S", + "-emit-llvm", + "-o", + self.ir_file, + ] + if self.check_expected_output(self.clang, args, self.file_to_reduce): + sys.exit("The interestingness test does not pass when generating llvm ir.") + # print(quote_cmd(self.get_crash_cmd(self.clang, args, self.file_to_reduce))) def check_middle_end(self) -> FailureType: # TODO: parse the exact pass from the backtrace and set the pass # pipeline directly via -passes="...". print("Checking opt for failure") + args = [self.opt_level, "-disable-output"] + if self.check_expected_output( + self.opt, + args, + self.ir_file, + ): + print("Found MiddleEnd Crash") + self.new_cmd = [self.opt] + args + return FailureType.MiddleEnd + + print("Check clang on IR file") + args = self.clang_args[:-1] + ["ir"] if self.check_expected_output( - cmd=self.opt, - args=[self.opt_level, "-disable-output"], - filename=self.ir_file, + self.clang, + args, + self.ir_file, ): print("Found MiddleEnd Crash") + self.new_cmd = [self.clang] + args return FailureType.MiddleEnd + print( "Could not automatically detect crash type, falling back to creduce/cvise." ) @@ -674,12 +707,10 @@ def main(): print("Starting reduction with creduce/cvise") pass case FailureType.MiddleEnd: - new_cmd = [opt_cmd, "-disable-output", r.opt_level] - r.reduce_ir_crash(new_cmd) + r.reduce_ir_crash(r.new_cmd) return case FailureType.BackEnd: - new_cmd = [llc_cmd, "-filetype=obj", r.opt_level] - r.reduce_ir_crash(new_cmd) + r.reduce_ir_crash(r.new_cmd) return r.simplify_clang_args() >From d273bb724816e3dba5e1952f27dd79dbc3c7c6e5 Mon Sep 17 00:00:00 2001 From: Paul Kirth <[email protected]> Date: Tue, 26 May 2026 14:14:36 -0700 Subject: [PATCH 7/8] Add tests and update logic to match tool output Clang and other tools don't have 100% matching output, if we don't check the prefixes we can use opt or llc, but otherwise the script would always fall back to using clang. --- clang/test/lit.cfg.py | 18 ++++ .../reduce-clang-crash/reduce-clang-crash.cpp | 64 ++++++++++++++ .../reduce-clang-crash/reduce-clang-crash.ll | 44 ++++++++++ clang/utils/reduce-clang-crash.py | 83 ++++++++++++++++--- 4 files changed, 197 insertions(+), 12 deletions(-) create mode 100644 clang/test/utils/reduce-clang-crash/reduce-clang-crash.cpp create mode 100644 clang/test/utils/reduce-clang-crash/reduce-clang-crash.ll diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py index 4a00769469543..4f3c9ccb4c202 100644 --- a/clang/test/lit.cfg.py +++ b/clang/test/lit.cfg.py @@ -130,6 +130,8 @@ "clang-scan-deps", "clang-installapi", "opt", + "llvm-reduce", + "llc", "llvm-ifs", "yaml2obj", "clang-linker-wrapper", @@ -301,6 +303,17 @@ def have_host_clang_repl_cuda(): ) ) +config.substitutions.append( + ( + "%reduce-clang-crash", + '"%s" "%s"' + % ( + config.python_executable, + os.path.join(config.clang_src_dir, "utils", "reduce-clang-crash.py"), + ), + ) +) + # Determine whether the test target is compatible with execution on the host. if "aarch64" in config.host_arch: config.available_features.add("aarch64-host") @@ -485,6 +498,11 @@ def user_is_root(): # default configs for the test runs. config.environment["CLANG_NO_DEFAULT_CONFIG"] = "1" +if lit.util.which("creduce") or lit.util.which("cvise"): + config.available_features.add("reproducer-reduction") + creduce_path = lit.util.which("cvise") or lit.util.which("creduce") + config.substitutions.append(("%creduce", creduce_path)) + if lit_config.update_tests: import sys import os diff --git a/clang/test/utils/reduce-clang-crash/reduce-clang-crash.cpp b/clang/test/utils/reduce-clang-crash/reduce-clang-crash.cpp new file mode 100644 index 0000000000000..4be20ed019446 --- /dev/null +++ b/clang/test/utils/reduce-clang-crash/reduce-clang-crash.cpp @@ -0,0 +1,64 @@ +// REQUIRES: x86-registered-target +// +// RUN: rm -rf %t && mkdir -p %t +// RUN: split-file %s %t +// RUN: cd %t +// +// RUN: env LLVM_DISABLE_SYMBOLIZATION=0 %reduce-clang-crash --clang %clang --opt opt --llc llc --llvm-reduce llvm-reduce --llc-arg=-fast-isel-abort=3 --auto crash.sh reduce-clang-crash.cpp | FileCheck --check-prefix=BACKEND %s +// RUN: FileCheck --check-prefix=BACKEND-REDUCED %s < %t/reduced.ll +// +// BACKEND: Found Middle/Backend failure +// BACKEND-NEXT: Checking llc for failure +// BACKEND-NEXT: Found BackEnd Crash +// BACKEND-NEXT: Writing interestingness test... +// BACKEND-EMPTY: +// BACKEND-NEXT: Creating the interestingness test... +// BACKEND-NEXT: Starting llvm-reduce with llc test case +// BACKEND-EMPTY: +// BACKEND-NEXT: Running llvm-reduce tool... +// BACKEND-NEXT: Done Reducing IR file. +// +// BACKEND-REDUCED: define void @_Z3foov +// +// RUN: env LLVM_DISABLE_SYMBOLIZATION=0 %reduce-clang-crash --clang %clang --creduce %creduce --auto crash.sh reduce-clang-crash.cpp --n 1 | FileCheck --check-prefix=FALLBACK %s +// RUN: FileCheck --check-prefix=FALLBACK-REDUCED %s < %t/reduced.ll +// +// FALLBACK: Found Middle/Backend failure +// FALLBACK-NEXT: Checking llc for failure +// FALLBACK-NEXT: Checking opt for failure +// FALLBACK-NEXT: Check clang on IR file +// FALLBACK-NEXT: Found MiddleEnd Crash +// FALLBACK-NEXT: Writing interestingness test... +// FALLBACK-EMPTY: +// FALLBACK-NEXT: Creating the interestingness test... +// FALLBACK-NEXT: Starting llvm-reduce with clang test case +// FALLBACK-EMPTY: +// FALLBACK-NEXT: Running llvm-reduce tool... +// FALLBACK-NEXT: Done Reducing IR file. +// +// FALLBACK-REDUCED: define void @_Z3foov +// +// RUN: env LLVM_DISABLE_SYMBOLIZATION=0 %reduce-clang-crash --clang %clang --creduce %creduce --auto crash_pragma.sh crash_pragma.cpp --n 1 | FileCheck --check-prefix=PRAGMA %s +// RUN: FileCheck --check-prefix=PRAGMA-REDUCED %s < %t/crash_pragma.reduced.cpp +// +// PRAGMA: Found Frontend Crash +// PRAGMA: Starting reduction with creduce/cvise +// +// PRAGMA-REDUCED: #pragma clang __debug assert + +//--- reduce-clang-crash.cpp +void foo() { + volatile __int128 a = 1; + volatile __int128 b = 2; + volatile __int128 c = a * b; +} + +//--- crash_pragma.cpp +#pragma clang __debug assert +void foo() {} + +//--- crash.sh +clang -cc1 -triple x86_64-unknown-linux-gnu -O0 -emit-obj -mllvm -fast-isel-abort=3 -x c++ reduce-clang-crash.cpp + +//--- crash_pragma.sh +clang -cc1 -triple x86_64-unknown-linux-gnu -O0 -x c++ crash_pragma.cpp diff --git a/clang/test/utils/reduce-clang-crash/reduce-clang-crash.ll b/clang/test/utils/reduce-clang-crash/reduce-clang-crash.ll new file mode 100644 index 0000000000000..b62bdf26a8095 --- /dev/null +++ b/clang/test/utils/reduce-clang-crash/reduce-clang-crash.ll @@ -0,0 +1,44 @@ +; REQUIRES: x86-registered-target +; +; RUN: rm -rf %t && mkdir -p %t +; RUN: split-file %s %t +; RUN: cd %t +; +; RUN: env LLVM_DISABLE_SYMBOLIZATION=0 %reduce-clang-crash --clang %clang --opt opt --llvm-reduce llvm-reduce --opt-arg=-enable-matrix --opt-arg=-verify-matrix-shapes --auto crash_middleend.sh reduce-clang-crash.ll | FileCheck %s +; RUN: FileCheck --check-prefix=REDUCED %s < %t/reduced.ll +; +; CHECK: Found Middle/Backend failure +; CHECK-NEXT: Checking opt for failure +; CHECK-NEXT: Found MiddleEnd Crash +; CHECK-NEXT: Writing interestingness test... +; CHECK-EMPTY: +; CHECK-NEXT: Creating the interestingness test... +; CHECK-NEXT: Starting llvm-reduce with opt test case +; CHECK-EMPTY: +; CHECK-NEXT: Running llvm-reduce tool... +; CHECK-NEXT: Done Reducing IR file. +; +; REDUCED: phi <16 x float> + +//--- reduce-clang-crash.ll +define <16 x float> @foo(i1 %c, ptr %p1, ptr %p2) { +entry: + br i1 %c, label %bb1, label %bb2 + +bb1: + %a = call <16 x float> @llvm.matrix.column.major.load.v16f32.p0(ptr %p1, i64 4, i1 false, i32 4, i32 4) + br label %exit + +bb2: + %b = call <16 x float> @llvm.matrix.column.major.load.v16f32.p0(ptr %p2, i64 2, i1 false, i32 2, i32 8) + br label %exit + +exit: + %y = phi <16 x float> [ %a, %bb1 ], [ %b, %bb2 ] + ret <16 x float> %y +} + +declare <16 x float> @llvm.matrix.column.major.load.v16f32.p0(ptr, i64, i1, i32, i32) + +//--- crash_middleend.sh +clang -cc1 -triple x86_64-unknown-linux-gnu -O2 -emit-obj -mllvm -enable-matrix -mllvm -verify-matrix-shapes -x ir reduce-clang-crash.ll diff --git a/clang/utils/reduce-clang-crash.py b/clang/utils/reduce-clang-crash.py index 4648ddce31c16..9aea90885b93f 100755 --- a/clang/utils/reduce-clang-crash.py +++ b/clang/utils/reduce-clang-crash.py @@ -122,7 +122,14 @@ def remove_first_line(file_path): class Reduce(object): - def __init__(self, crash_script, file_to_reduce, creduce_flags): + def __init__( + self, + crash_script, + file_to_reduce, + creduce_flags, + opt_args=[], + llc_args=[], + ): crash_script_name, crash_script_ext = os.path.splitext(crash_script) file_reduce_name, file_reduce_ext = os.path.splitext(file_to_reduce) @@ -138,6 +145,8 @@ def __init__(self, crash_script, file_to_reduce, creduce_flags): self.creduce_flags = ["--tidy"] + creduce_flags self.opt = opt_cmd self.llc = llc_cmd + self.opt_args = opt_args + self.llc_args = llc_args self.ir_file = "crash.ll" self.read_clang_args(crash_script, file_to_reduce) @@ -248,7 +257,13 @@ def check_expected_output(self, cmd, args, filename): stderr=subprocess.STDOUT, ) crash_output, _ = p.communicate() - return all(msg in crash_output.decode("utf-8") for msg in self.expected_output) + decoded_output = crash_output.decode("utf-8") + cleaned_output = "\n".join( + self.clean_prefix(line) for line in decoded_output.splitlines() + ) + return all( + self.clean_prefix(msg) in cleaned_output for msg in self.expected_output + ) def write_interestingness_test(self, cmd=None, use_llvm_reduce=False): print("\nCreating the interestingness test...") @@ -276,7 +291,9 @@ def write_interestingness_test(self, cmd=None, use_llvm_reduce=False): ) for msg in self.expected_output: - output += "grep -F %s t.log || exit 1\n" % shlex.quote(msg) + output += "grep -F %s t.log || exit 1\n" % shlex.quote( + self.clean_prefix(msg) + ) write_to_script(output, self.testfile) self.check_interestingness(cmd, use_llvm_reduce=use_llvm_reduce) @@ -349,6 +366,27 @@ def filter_args( result.append(arg) return result + def get_clang_args_for_ir_extraction(self): + args = list(self.clang_args) + if "-emit-obj" in args: + idx = args.index("-emit-obj") + args[idx] = "-emit-llvm" + else: + args.append("-emit-llvm") + return args + + def clean_prefix(self, s: str) -> str: + """Strips common compiler wrapper prefixes to normalize diagnostics.""" + prefixes = ( + "fatal error: error in backend: ", + "error: error in backend: ", + "LLVM ERROR: ", + ) + for prefix in prefixes: + if s.startswith(prefix): + return s[len(prefix) :].strip() + return s.strip() + def try_remove_args(self, args, msg=None, extra_arg=None, **kwargs): new_args = self.filter_args(args, **kwargs) @@ -529,13 +567,14 @@ def classify_crash(self) -> FailureType: def check_llc_failure(self) -> bool: return self.check_expected_output( - self.llc, [self.opt_level, "-filetype=obj"], self.ir_file + self.llc, [self.opt_level, "-filetype=obj"] + self.llc_args, self.ir_file ) def extract_backend_ir(self) -> bool: + args = self.get_clang_args_for_ir_extraction() return not self.check_expected_output( self.clang, - self.clang_args + ["-emit-llvm", "-o", self.ir_file], + args + ["-o", self.ir_file], self.file_to_reduce, ) @@ -546,7 +585,11 @@ def check_backend(self) -> Optional[FailureType]: print("Checking llc for failure") if self.check_llc_failure(): print("Found BackEnd Crash") - self.new_cmd = [self.llc, self.opt_level, "-filetype=obj"] + self.new_cmd = [ + self.llc, + self.opt_level, + "-filetype=obj", + ] + self.llc_args return FailureType.BackEnd def extract_crashing_ir(self, use_print_on_crash=False): @@ -571,10 +614,8 @@ def extract_crashing_ir(self, use_print_on_crash=False): # The output from --print-on-crash has an invalid first line (pass name). remove_first_line(self.ir_file) return - args = self.clang_args + [ + args = self.get_clang_args_for_ir_extraction() + [ "-disable-llvm-passes", - "-S", - "-emit-llvm", "-o", self.ir_file, ] @@ -586,7 +627,7 @@ def check_middle_end(self) -> FailureType: # TODO: parse the exact pass from the backtrace and set the pass # pipeline directly via -passes="...". print("Checking opt for failure") - args = [self.opt_level, "-disable-output"] + args = [self.opt_level, "-disable-output"] + self.opt_args if self.check_expected_output( self.opt, args, @@ -615,7 +656,8 @@ def check_middle_end(self) -> FailureType: def reduce_ir_crash(self, new_cmd: List[str]): print("Writing interestingness test...") self.write_interestingness_test(cmd=new_cmd, use_llvm_reduce=True) - print("Starting llvm-reduce with llc test case") + tool_name = os.path.basename(new_cmd[0]) + print(f"Starting llvm-reduce with {tool_name} test case") self.run_llvm_reduce() print("Done Reducing IR file.") @@ -682,6 +724,20 @@ def main(): help="Use auto reduction mode, that uses `creduce`/`cvise`" "for frontend crashes and llvm-reduce for middle/backend crashes.", ) + parser.add_argument( + "--opt-arg", + dest="opt_args", + action="append", + default=[], + help="Additional arguments to pass to opt", + ) + parser.add_argument( + "--llc-arg", + dest="llc_args", + action="append", + default=[], + help="Additional arguments to pass to llc", + ) parser.add_argument("-v", "--verbose", action="store_true") args, creduce_flags = parser.parse_known_args() verbose = args.verbose @@ -692,6 +748,7 @@ def main(): clang_cmd = check_cmd("clang", llvm_bin, args.clang) opt_cmd = check_cmd("opt", llvm_bin, args.opt) llc_cmd = check_cmd("llc", llvm_bin, args.llc) + llvm_reduce_cmd = check_cmd("llvm-reduce", llvm_bin, args.llvm_reduce) crash_script = check_file(args.crash_script[0]) file_to_reduce = check_file(args.file_to_reduce[0]) @@ -699,7 +756,9 @@ def main(): if "--n" not in creduce_flags: creduce_flags += ["--n", str(max(4, multiprocessing.cpu_count() // 2))] - r = Reduce(crash_script, file_to_reduce, creduce_flags) + r = Reduce( + crash_script, file_to_reduce, creduce_flags, args.opt_args, args.llc_args + ) if args.auto: crash_type = r.classify_crash() match crash_type: >From da4db707fe25246b156aa901dac8f2fe7b1249f9 Mon Sep 17 00:00:00 2001 From: Paul Kirth <[email protected]> Date: Thu, 28 May 2026 19:28:07 -0700 Subject: [PATCH 8/8] Feature gate tests to only run if appropriate tools are available --- clang/test/utils/reduce-clang-crash/reduce-clang-crash.cpp | 1 + clang/test/utils/reduce-clang-crash/reduce-clang-crash.ll | 1 + 2 files changed, 2 insertions(+) diff --git a/clang/test/utils/reduce-clang-crash/reduce-clang-crash.cpp b/clang/test/utils/reduce-clang-crash/reduce-clang-crash.cpp index 4be20ed019446..16d19e24a2e3e 100644 --- a/clang/test/utils/reduce-clang-crash/reduce-clang-crash.cpp +++ b/clang/test/utils/reduce-clang-crash/reduce-clang-crash.cpp @@ -1,4 +1,5 @@ // REQUIRES: x86-registered-target +// REQUIRES: reproducer-reduction // // RUN: rm -rf %t && mkdir -p %t // RUN: split-file %s %t diff --git a/clang/test/utils/reduce-clang-crash/reduce-clang-crash.ll b/clang/test/utils/reduce-clang-crash/reduce-clang-crash.ll index b62bdf26a8095..1fbf1698eaa48 100644 --- a/clang/test/utils/reduce-clang-crash/reduce-clang-crash.ll +++ b/clang/test/utils/reduce-clang-crash/reduce-clang-crash.ll @@ -1,4 +1,5 @@ ; REQUIRES: x86-registered-target +; REQUIRES: reproducer-reduction ; ; RUN: rm -rf %t && mkdir -p %t ; RUN: split-file %s %t _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
