https://github.com/SLTozer created https://github.com/llvm/llvm-project/pull/202544
This patch adds a !label node to Dexter scripts, which references lines at a position in the source program marked with "!dex_label <identifier>". Each label use can be given a positive or negative offset, and file lookup is based on the provided --source-root-dir (using the test file's directory if none is provided). >From 14aae62077875be3ceb3a46285b4279654da23a4 Mon Sep 17 00:00:00 2001 From: Stephen Tozer <[email protected]> Date: Thu, 4 Jun 2026 15:02:48 +0100 Subject: [PATCH] [Dexter] Add label nodes for line references This patch adds a !label node to Dexter scripts, which references lines at a position in the source program marked with "!dex_label <identifier>". Each label use can be given a positive or negative offset, and file lookup is based on the provided --source-root-dir (using the test file's directory if none is provided). --- .../ScriptDebuggerController.py | 2 +- .../dexter/dex/evaluation/StateMatch.py | 24 ++-- .../dexter/dex/test_script/Nodes.py | 83 ++++++++++-- .../dexter/dex/test_script/Script.py | 118 ++++++++++++++++-- .../scripts/labels/Inputs/header.h | 8 ++ .../scripts/labels/invalid_label.cpp | 18 +++ .../feature_tests/scripts/labels/offset.cpp | 35 ++++++ .../scripts/labels/simple_labels.cpp | 38 ++++++ .../scripts/labels/source_root_dir.cpp | 41 ++++++ 9 files changed, 336 insertions(+), 31 deletions(-) create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/Inputs/header.h create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/invalid_label.cpp create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/offset.cpp create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/simple_labels.cpp create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/source_root_dir.cpp diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ScriptDebuggerController.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ScriptDebuggerController.py index 94483eaaa002d..54e924edd40c2 100644 --- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ScriptDebuggerController.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ScriptDebuggerController.py @@ -68,7 +68,7 @@ def add_where_entry_bp(self, where: Where, default_file: Optional[str] = None): file = where.file or default_file assert file, "Cannot set line breakpoints without a valid file!" # If this Where covers a range of lines, we breakpoint each of them to ensure that we don't miss any lines. - for line in where.get_lines(): + for line in where.get_lines(self.script.get_labels(file)): added_ids.append(self.debugger.add_breakpoint(file, line)) self._where_bps[where] = added_ids for id in added_ids: diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/StateMatch.py b/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/StateMatch.py index 2b4363cff38af..f9221043132f1 100644 --- a/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/StateMatch.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/StateMatch.py @@ -28,6 +28,7 @@ def is_subpath(subpath: str, superpath: str) -> bool: def match_where_to_frame( where: Where, frame: FrameIR, + labels: Dict[str, int], ) -> bool: """A very simple matcher, returns True iff `where` matches `frame`.""" if where.file is not None and not is_subpath(where.file, frame.loc.path): @@ -39,7 +40,7 @@ def match_where_to_frame( if where.function != fn: return False if where.lines is not None: - if frame.loc.lineno not in where.get_lines(): + if frame.loc.lineno not in where.get_lines(labels): return False if ( where.for_hit_count is not None @@ -74,6 +75,7 @@ def get_active_where_matches( def get_active_wheres(where: Where, scope: Scope): # For nested !wheres, we must match a specific frame relative to the parent !where. + expected_file = scope.get_known_file_for_where(where) if scope.where: if scope.where not in active_where_expects: # If the parent !where doesn't match any frame, then this !where cannot match any either. @@ -87,18 +89,20 @@ def get_active_wheres(where: Where, scope: Scope): # If the target frame is -1, we can't match the !where yet, but we should prepare to step into it. active_where_expects[scope.where].pending_wheres.append(where) return - if match_where_to_frame(where, step_info.frames[target_frame_idx]): + labels = script.get_labels( + expected_file or step_info.frames[target_frame_idx].loc.path + ) + if match_where_to_frame(where, step_info.frames[target_frame_idx], labels): active_where_expects[where] = WhereMatchResult(target_frame_idx) return # For this !where, search for the rootmost stack frame that matches it. - matching_frame_idx = next( - ( - frame_idx - for frame_idx, frame in reversed(list(enumerate(step_info.frames))) - if match_where_to_frame(where, frame) - ), - None, - ) + matching_frame_idx = None + for frame_idx, frame in reversed(list(enumerate(step_info.frames))): + labels = script.get_labels(expected_file or frame.loc.path) + if match_where_to_frame(where, frame, labels): + matching_frame_idx = frame_idx + break + if matching_frame_idx is not None: active_where_expects[where] = WhereMatchResult(matching_frame_idx) diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py index 932475dff3493..b4650102beff4 100644 --- a/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py @@ -10,6 +10,7 @@ import abc from dataclasses import dataclass +import re from typing import Any, Dict, Optional, Union import yaml from dex.dextIR.ValueIR import ValueIR @@ -21,6 +22,7 @@ def setup_yaml_parser(loader): Where, Value, DexRange, + Label, ] for c in reg_classes: c.register_yaml(loader) @@ -52,7 +54,10 @@ class Where: def __init__(self, attributes: dict, is_and: bool): self.file: Optional[str] = attributes.pop("file", None) self.function: Union[list[str], str, None] = attributes.pop("function", None) - self.lines: Union[int, DexRange, None] = attributes.pop("lines", None) + lines = attributes.pop("lines", None) + if isinstance(lines, (int, Label)): + lines = Line(lines) + self.lines: Union[Line, DexRange, None] = lines self.after_hit_count: Optional[int] = attributes.pop("after_hit_count", None) self.for_hit_count: Optional[int] = attributes.pop("for_hit_count", None) self.conditions: dict = attributes.pop("conditions", None) @@ -83,7 +88,7 @@ def get_attrs(self) -> Dict[str, Any]: return { "file": self.file, "function": self.function, - "lines": self.lines, + "lines": self.lines.value if isinstance(self.lines, Line) else self.lines, "for_hit_count": self.for_hit_count, "after_hit_count": self.after_hit_count, "conditions": self.conditions, @@ -110,17 +115,18 @@ def register_yaml(loader): yaml.add_constructor("!and", Where.get_constructor(True), loader) yaml.add_representer(Where, Where.representer) - def get_lines(self) -> range: + def get_lines(self, labels: Dict[str, int]) -> range: """Returns the range of line numbers that this Where references, returning an empty range if this Where does not refer to any lines.""" if not self.lines: return range(-1) - if isinstance(self.lines, int): - return range(self.lines, self.lines + 1) + if isinstance(self.lines, Line): + line_num = self.lines.to_line(labels) + return range(line_num, line_num + 1) assert isinstance( self.lines, DexRange ), f"Invalid type for lines: {self.lines}: ({type(self.lines)})" - return self.lines.to_range() + return self.lines.to_range(labels) ################### @@ -182,30 +188,81 @@ def register_yaml(loader): ## Utility Nodes: Can be used anywhere in a script as a form of syntactic sugar. +@dataclass(frozen=True) +class Line: + """Union class between an int or a Label, used to represent lines inside of Nodes.""" + + value: Union[int, "Label"] + + def to_line(self, labels: Dict[str, int]) -> int: + if isinstance(self.value, int): + return self.value + return self.value.to_line(labels) + + def __repr__(self): + return str(self.value) + + @dataclass(frozen=True) class DexRange: - start: int - stop: int + start: Line + stop: Line def __repr__(self) -> str: return f"[{self.start} - {self.stop}]" # We use an inclusive range in Dexter scripts, while python ranges are exclusive. - def to_range(self) -> range: - return range(self.start, self.stop + 1) + def to_range(self, labels: Dict[str, int]) -> range: + return range(self.start.to_line(labels), self.stop.to_line(labels) + 1) @staticmethod def constructor(loader: yaml.Loader, node): range_seq = loader.construct_sequence(node) - if len(range_seq) != 2 or not all(isinstance(elt, int) for elt in range_seq): + if len(range_seq) != 2 or not all( + isinstance(elt, (int, Label)) for elt in range_seq + ): raise DexterNodeError(node, "range must have exactly 2 int elements") - return DexRange(range_seq[0], range_seq[1]) + return DexRange(Line(range_seq[0]), Line(range_seq[1])) @staticmethod def representer(dumper, data: "DexRange"): - return dumper.represent_sequence("!range", [data.start, data.stop]) + return dumper.represent_sequence("!range", [data.start.value, data.stop.value]) @staticmethod def register_yaml(loader): yaml.add_constructor("!range", DexRange.constructor, loader) yaml.add_representer(DexRange, DexRange.representer) + + +@dataclass(frozen=True) +class Label: + name: str + + def to_line(self, labels: Dict[str, int]) -> int: + # Labels may contain offsets, which is accounted for here. + raw_label = self.name.strip() + label_str = raw_label + offset = 0 + if match := re.match(r"^([a-zA-Z_]\w*)\s*([+-])\s*(\d+)$", raw_label): + identifier, sign, number = match.groups() + offset = int(number) if sign == "+" else -int(number) + label_str = identifier + if label_str not in labels: + raise DexterNodeError(self, f'Label "{label_str}" not found') + return labels[label_str] + offset + + def __repr__(self): + return f"Label({self.name})" + + @staticmethod + def constructor(loader: yaml.Loader, node): + return Label(loader.construct_scalar(node)) + + @staticmethod + def representer(dumper, data: "Label"): + return dumper.represent_scalar("!label", data.name) + + @staticmethod + def register_yaml(loader): + yaml.add_constructor("!label", Label.constructor, loader) + yaml.add_representer(Label, Label.representer) diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py index a338f62f95af7..6f087ca52db17 100644 --- a/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py @@ -9,9 +9,11 @@ results. """ -from pathlib import PurePath +from collections import defaultdict +from pathlib import Path, PurePath import os -from typing import Any, Callable, Optional, Set +import re +from typing import Any, Callable, Dict, List, Optional, Set import yaml from dex.test_script.Nodes import ( @@ -20,6 +22,7 @@ setup_yaml_parser, ) +from dex.tools.Main import Context from dex.utils.Exceptions import Error from dex.utils.Timer import Timer @@ -28,6 +31,49 @@ class DexterScriptError(Error): pass +class LabelDict: + def __init__(self): + self.file_to_labels_to_lines: Dict[str, Dict[str, int]] = {} + + def set_labels_from_file(self, context: Context, file: str, base_dir: str): + """Given either an absolute filepath or a relative filepath and the directory it is relative to, searches the + file for !dex_labels and records the line numbers they appear on.""" + # Check first whether we've already checked this file for labels - even if the file doesn't contain any labels + # we store an empty dict to record that fact. + # NB: We don't bother detecting cases where different file strings refer to the same underlying file, since this + # is low cost. + if self.has_labels(file): + return + self.file_to_labels_to_lines[file] = {} + abs_path = file if os.path.isabs(file) else os.path.join(base_dir, file) + if not os.path.exists(abs_path): + context.logger.warning(f"Could not find !where file: {abs_path}") + return + with open(abs_path, "r", encoding="utf-8", errors="ignore") as r: + lines = r.readlines() + # Now that we have the lines from the file, search for labels. + dex_label_re = re.compile(r"!dex_label ([a-zA-Z_]\w*)") + for idx, line in enumerate(lines): + label_str_match = dex_label_re.search(line) + if not label_str_match: + continue + label = label_str_match.group(1) + line = idx + 1 + if label in self.file_to_labels_to_lines[file]: + # Ignore duplicate labels. + original_line = self.file_to_labels_to_lines[file][label] + context.logger.warning( + f'ignoring duplicate label "{label}" in "{file}"; original: {original_line}, new: {line}' + ) + else: + self.file_to_labels_to_lines[file][label] = line + + def has_labels(self, file: str) -> bool: + return file in self.file_to_labels_to_lines + + def get_labels(self, file: str) -> Dict[str, int]: + return self.file_to_labels_to_lines[file] + class Scope: """Helper class used to simplify queries about the context of a Node in the Dexter Script. The context for a given Node consists of some base context information in the root of the script, and then all Where nodes in the parent @@ -66,17 +112,49 @@ def add_where(self, where: Where): """Adds `where` to this Scope's chain.""" return Scope(where=where, parent_scope=self) + def get_known_file_for_where(self, where: Where) -> Optional[str]: + """For a `where` that exists directly in this Scope, determines whether there is a known file that `where` + expects - whether this is an explicitly-declared file, or implicitly the root scope file. There is no known file + for !where {function: ...} nodes, however.""" + if where.file: + return where.file + if where.function: + return None + next_scope = self + while where.is_and: + assert ( + next_scope.parent_scope and next_scope.where + ), "!and node at root scope?" + where = next_scope.where + if where.file: + return where.file + if where.function: + return None + next_scope = next_scope.parent_scope + while next_scope.file is None: + assert next_scope.parent_scope + next_scope = next_scope.parent_scope + return next_scope.file + class DexterScript: def __init__( self, - context, + context: Context, script_obj, scope: Scope, + source_root_dir: Optional[str], ): self.context = context self.script_obj = script_obj self.root_scope = scope + self.label_dict = LabelDict() + assert scope.file is not None + self.base_dir = ( + source_root_dir + if source_root_dir is not None + else os.path.dirname(scope.file) + ) # `visit_script` will validate the structure of the script, as it traverses the full script and raises an # exception if it sees anything unexpected. self.visit_script() @@ -130,6 +208,26 @@ def root_wheres(self) -> Set[Where]: def dump(self) -> str: return yaml.dump(self.script_obj) + def gather_labels(self): + """Pre-gather labels for all !where nodes with `lines` entries.""" + assert self.root_scope.file is not None + add_labels = lambda file: self.label_dict.set_labels_from_file( + self.context, file, str(self.base_dir) + ) + + def collect_file(where: Where, scope: Scope): + # If we have !where nodes that check lines without an explicit file, we default to the test file. + if not where.is_and and where.lines and not where.file: + add_labels(self.root_scope.file) + elif where.file: + add_labels(where.file) + + self.visit_script(visit_where=collect_file) + + def get_labels(self, file: str) -> Dict[str, int]: + if not self.label_dict.has_labels(file): + self.label_dict.set_labels_from_file(self.context, file, self.base_dir) + return self.label_dict.get_labels(file) # Helper function to apply a line offset to the errors reported by YAML while loading, to account for the YAML documents # being embedded in part of a file. @@ -160,7 +258,7 @@ def adjust_mark_loc(mark: Optional[yaml.Mark]) -> Optional[yaml.Mark]: raise e -def get_script(context, file, loader) -> DexterScript: +def get_script(context, file, loader, source_root_dir: Optional[str]) -> DexterScript: """Searches the given file for a valid Dexter script, and returns the first valid script that it finds or raises an Error if none is found.""" if not os.path.exists(file): @@ -182,6 +280,7 @@ def get_script(context, file, loader) -> DexterScript: context, try_load_yaml("\n".join(lines), loader), root_scope, + source_root_dir, ) except (Error, yaml.YAMLError) as e: raise Error(f"File '{file}' was not a valid Dexter script:\n{e}") @@ -203,6 +302,7 @@ def get_script(context, file, loader) -> DexterScript: "\n".join(lines[start_line:stop_line]), loader, start_line ), root_scope, + source_root_dir, ) except (Error, yaml.YAMLError) as e: attempted_scripts.append((start_line, e)) @@ -218,13 +318,17 @@ def get_script(context, file, loader) -> DexterScript: ) -def get_dexter_script(context, test_file, source_root_dir): +def get_dexter_script(context, test_file, source_root_dir: Optional[str]): setup_yaml_parser(yaml.CLoader) with Timer("parsing script"): - script = get_script(context, test_file, yaml.CLoader) + script = get_script(context, test_file, yaml.CLoader, source_root_dir) assert script.root_scope.file == test_file source_files = set() - source_dir = source_root_dir if source_root_dir else str(test_file) + source_dir = ( + source_root_dir + if source_root_dir is not None + else os.path.basename(test_file) + ) def check_explicit_files(where: Where, _: Scope): if not where.file: diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/Inputs/header.h b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/Inputs/header.h new file mode 100644 index 0000000000000..353053eb19689 --- /dev/null +++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/Inputs/header.h @@ -0,0 +1,8 @@ + +int multiply(int a, int b) { + int result = 0; + for (int i = 0; i < b; ++i) { + result += a; + } + return result; // !dex_label mul +} diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/invalid_label.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/invalid_label.cpp new file mode 100644 index 0000000000000..54418d0853bdd --- /dev/null +++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/invalid_label.cpp @@ -0,0 +1,18 @@ +// RUN: %dexter_regression_test_cxx_build %s -o %t +// RUN: not %dexter_regression_test_run --source-root-dir %S/Inputs --use-script --binary %t -- %s 2>&1 | FileCheck %s + +int main() { + int a = 4; + int b = 4; + return b - a; // !dex_label unused +} + +// CHECK: error: Error with node: Label(used): Label "used" not found + +/* +--- +!where {lines: !label used}: + !value a: 4 + !value b: 4 +... +*/ diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/offset.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/offset.cpp new file mode 100644 index 0000000000000..5c0bb7b995bcc --- /dev/null +++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/offset.cpp @@ -0,0 +1,35 @@ +// RUN: %dexter_regression_test_cxx_build %s -o %t +// RUN: %dexter_regression_test_run --use-script --binary %t -- %s | FileCheck %s + + +int main() { + int count = 0; + ++count; // !dex_label start + ++count; + ++count; + ++count; + return count; +} // !dex_label end + +// CHECK: total_watched_steps: 5 +// CHECK: correct_steps: 5 +// CHECK: incorrect_steps: 0 +// CHECK: missing_var_steps: 0 +// CHECK: unexpected_value_steps: 0 +// CHECK: seen_values: 5 +// CHECK: missing_values: 0 + +/* +--- +!where {lines: !label start}: + !value count: 0 +!where {lines: !label start + 1}: + !value count: 1 +!where {lines: !label start+ 2}: + !value count: 2 +!where {lines: !label start +3}: + !value count: 3 +!where {lines: !label end-1}: + !value count: 4 +... +*/ diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/simple_labels.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/simple_labels.cpp new file mode 100644 index 0000000000000..c65a9c8fca77d --- /dev/null +++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/simple_labels.cpp @@ -0,0 +1,38 @@ +// RUN: %dexter_regression_test_cxx_build %s -o %t +// RUN: %dexter_regression_test_run --use-script --binary %t -- %s | FileCheck %s + +#include "Inputs/header.h" + +int factorial(int n) { + int result = 1; + // !dex_label factorial_start + for (int i = 1; i <= n; ++i) { + result = multiply(result, i); + } + return result; // !dex_label factorial_end +} + +int main() { + int a = 4; + return factorial(a); // !dex_label call +} + +// CHECK: total_watched_steps: 20 +// CHECK: correct_steps: 20 +// CHECK: incorrect_steps: 0 +// CHECK: missing_var_steps: 0 +// CHECK: unexpected_value_steps: 0 +// CHECK: seen_values: 9 +// CHECK: missing_values: 0 + +/* +--- +!where {lines: !label call}: + !value a: 4 +!where {function: factorial}: + !and {lines: !range [!label factorial_start, !label factorial_end]}: + !value result: [1, 2, 6, 24] + !where {file: "Inputs/header.h", lines: !label mul}: + !value result: [1, 2, 6, 24] +... +*/ diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/source_root_dir.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/source_root_dir.cpp new file mode 100644 index 0000000000000..420d1b6b45e37 --- /dev/null +++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/labels/source_root_dir.cpp @@ -0,0 +1,41 @@ +// RUN: %dexter_regression_test_cxx_build %s -o %t +// RUN: %dexter_regression_test_run --source-root-dir %S/Inputs --use-script --binary %t -- %s | FileCheck %s + +// Check that when --source-root-dir is provided, labels will be checked +// relative to that directory. + +#include "Inputs/header.h" + +int factorial(int n) { + int result = 1; + // !dex_label factorial_start + for (int i = 1; i <= n; ++i) { + result = multiply(result, i); + } + return result; // !dex_label factorial_end +} + +int main() { + int a = 4; + return factorial(a); // !dex_label call +} + +// CHECK: total_watched_steps: 20 +// CHECK: correct_steps: 20 +// CHECK: incorrect_steps: 0 +// CHECK: missing_var_steps: 0 +// CHECK: unexpected_value_steps: 0 +// CHECK: seen_values: 9 +// CHECK: missing_values: 0 + +/* +--- +!where {lines: !label call}: + !value a: 4 +!where {function: factorial}: + !and {lines: !range [!label factorial_start, !label factorial_end]}: + !value result: [1, 2, 6, 24] + !where {file: "header.h", lines: !label mul}: + !value result: [1, 2, 6, 24] +... +*/ _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
