dannycjones commented on code in PR #3414:
URL: https://github.com/apache/iceberg-rust/pull/3414#discussion_r4260777147
##########
dev/release/licenses.py:
##########
@@ -56,6 +75,270 @@ def require_cargo_deny() -> None:
)
+def declared_clarifications() -> set[CrateName]:
+ """Crate names that deny.toml carries a [[licenses.clarify]] block for."""
+ with DENY_TOML.open("rb") as handle:
+ config = tomllib.load(handle)
+
+ blocks = config.get("licenses", {}).get("clarify", [])
+ if not blocks:
+ raise Failure(
+ f"Expected at least one [[licenses.clarify]] block in {DENY_TOML},
found none.\n"
+ "If the last one was deliberately removed, remove this check with
it."
+ )
+
+ pinned = sorted(block["crate"] for block in blocks if "@" in
block["crate"])
+ if pinned:
+ raise Failure(
+ textwrap.dedent("""
+ {deny_toml} pins a [[licenses.clarify]] block to a crate
version:
+ {specs}
+
+ Version pins are rejected by this script.
+ cargo-deny supports them, but will not report when they become
stale.
+ To keep things simple, this script requires that no pinning is
used.
+ Where newer versions change licensing, the script will reject
the clarification
+ and the deny.toml file should be updated.
+ """).format(
+ deny_toml=DENY_TOML,
+ specs="\n".join(f" {spec}" for spec in pinned),
+ )
+ )
+
+ return {block["crate"] for block in blocks}
+
+
+def evaluated_crates() -> tuple[
+ dict[CrateName, set[CrateVersion]], dict[CrateName, set[CrateVersion]]
+]:
+ """Every crate cargo-deny evaluated, along with the subset that had a
clarification applied."""
+ result = subprocess.run(
+ [
+ "cargo",
+ "deny",
+ "--format=json",
+ "check",
+ "license",
+ # We emit the license accepted messages as warnings so we can
count them.
+ "--warn=accepted",
+ ],
+ cwd=REPO_ROOT,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ diagnostics = result.stderr
+ if result.returncode != 0:
+ print(diagnostics, file=sys.stderr, end="")
+ raise Failure(
+ "cargo-deny failed while re-running to verify that no license "
+ "clarifications are ignored."
+ )
+
+ evaluated: dict[CrateName, set[CrateVersion]] = defaultdict(set)
+ clarified: dict[CrateName, set[CrateVersion]] = defaultdict(set)
+ for line in diagnostics.splitlines():
+ if not line.strip():
+ continue
+ line_truncated = line[:200]
+ try:
+ diagnostic = json.loads(line)
+ except json.JSONDecodeError:
+ raise Failure(
+ "cargo-deny emitted a line that is not JSON, so its output
cannot be "
+ f"parsed:\n {line_truncated}"
+ )
+
+ fields = diagnostic.get("fields", {})
+
+ # Anything not accepted should already be reported.
+ if fields.get("code") != "accepted":
+ continue
+
+ graphs = fields.get("graphs") or []
+ # An `accepted` verdict covers exactly one crate. Other checks attach
several.
+ if len(graphs) != 1:
+ raise Failure(
+ f"cargo-deny reported an 'accepted' license verdict covering "
+ f"{len(graphs)} crates, but one was expected:\n
{line_truncated}"
+ )
+ krate = graphs[0].get("Krate", {})
+
+ name, version = krate.get("name"), krate.get("version")
+ if name is None or version is None:
+ raise Failure(
+ "cargo-deny reported an 'accepted' license verdict without
naming both "
+ f"the crate and its version:\n {line_truncated}"
+ )
+
+ evaluated[name].add(version)
+ labels = fields.get("labels") or []
+ CLARIFICATION_APPLIED_MSG = "license expression retrieved via user
override"
+ if any(label.get("message") == CLARIFICATION_APPLIED_MSG for label in
labels):
+ clarified[name].add(version)
+
+ return dict(evaluated), dict(clarified)
+
+
+def cargo_metadata() -> dict:
+ """Resolved cargo metadata for this workspace."""
+ result = subprocess.run(
+ [
+ "cargo",
+ "metadata",
+ "--format-version",
+ "1",
+ "--locked",
+ "--all-features",
+ "--manifest-path",
+ str(REPO_ROOT / "Cargo.toml"),
+ ],
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ if result.returncode != 0:
+ raise Failure(f"cargo metadata failed:\n{result.stderr}")
+ return json.loads(result.stdout)
+
+
+def crate_source_dirs(crates: set[Crate]) -> dict[Crate, str]:
+ """Where cargo unpacked each crate's sources.
+
+ The directory is `<name>-<version>`, but names often contain hyphens (eg.
my-crate-5.0.3) and
+ versions can carry build metadata (eg. my-crate-5.0.3+mybuild).
+ """
+ packages = cargo_metadata()["packages"]
+
+ dict_crate_to_path = {}
+
+ for package in packages:
+ crate = (package["name"], package["version"])
+ if crate not in crates:
+ continue
+ dict_crate_to_path[crate] = str(Path(package["manifest_path"]).parent)
+
+ return dict_crate_to_path
+
+
+def require_clarifications_applied(
+ declared: set[CrateName],
+ evaluated: dict[CrateName, set[CrateVersion]],
+ actually_clarified: dict[CrateName, set[CrateVersion]],
+) -> None:
+ """
+ Fail if a clarification did not reach a crate version that was evaluated.
+
+ `declared` is all crates declared with a clarification in deny.toml,
+ `evaluated` is the list of all crate versions evaluated as part of the
dependency graph,
+ and `actually_clarified` is the subset of crate versions evaluated which
had a clarification applied.
+ """
+ crates_with_clarifications_missed: list[Crate] = []
+ for crate_name in declared:
+ if crate_name not in evaluated:
+ continue
+ versions_evaluated = evaluated[crate_name]
+ versions_clarified = actually_clarified.get(crate_name, set())
+ for crate_version in versions_evaluated - versions_clarified:
+ crates_with_clarifications_missed.append((crate_name,
crate_version))
+ crates_with_clarifications_missed.sort()
+ if not crates_with_clarifications_missed:
+ # Nothing to report
+ return
+
+ source_dirs = crate_source_dirs(set(crates_with_clarifications_missed))
+ error_line_output = []
+ for crate_name, crate_version in crates_with_clarifications_missed:
+ source_dir = source_dirs.get(
+ (crate_name, crate_version), "<source not unpacked>"
+ )
+ error_line_output.append(
+ f" {crate_name} {crate_version} ({source_dir}): clarification did
not apply"
+ )
+
+ raise Failure(
+ textwrap.dedent("""
+ {deny_toml} declares license clarifications that cargo-deny did
not apply:
+ {crates}
+
+ A clarification is discarded by cargo-deny when any of its
attested license
+ files are no longer valid (hash mismatch, etc.).
+
+ Review the clarification, checking all files still exist and the
hash matches for each.
+ You can compute the hash for a license file using the following
command:
+
+ uv run {script} dependencies clarification-hash \\
+ <crate directory above>/<license file path>
+
+ If a hash has changed, read the file and ensure the license
expression still matches.
+ If it does, an updated deny.toml should be committed.
+ """).format(
+ deny_toml=DENY_TOML,
+ crates="\n".join(error_line_output),
+ script=SCRIPT_PATH,
+ )
+ )
+
+
+def require_no_stale_clarifications(
+ declared: set[CrateName], evaluated: dict[CrateName, set[CrateVersion]]
+) -> None:
+ """Fail if a clarification names a crate that is not in the dependency
graph."""
+ stale: list[CrateName] = sorted(declared - evaluated.keys())
+ if not stale:
+ return
+
+ raise Failure(
+ textwrap.dedent(
+ """
+ {deny_toml} declares license clarifications for crates that
cargo-deny did
+ not find in the dependency graph:
+ {crates}
+
+ These are likely stale entries that should be removed.
+ """
+ ).format(
+ deny_toml=DENY_TOML,
+ crates="\n".join(f" {crate_name}" for crate_name in stale),
+ )
+ )
+
+
+def clarification_file_hash(path: Path) -> str:
+ """The hash cargo-deny expects in a [[licenses.clarify]] `license-files`
entry.
+
+ XxHash32 with seed 0, over the file with every line terminated by exactly
one LF.
+
+ cargo-deny appears to offer no way to compute this value,
+ so this command is added to make it easy (based on the cargo-deny source
code).
+ """
+ try:
+ # Import here, so other commands of the script can still work if
missing.
+ import xxhash
+ except ModuleNotFoundError:
+ raise Failure(
+ "The module 'xxhash' is required to compute the hash for
cargo-deny's deny.toml file."
+ )
+
+ lines = path.read_bytes().split(b"\n")
+ # Splitting on \n may leave final element with only line feed - drop it.
+ if lines and lines[-1] == b"":
+ lines.pop()
+ # Drop any line feed, ensure every line ends '\n'
+ text = b"".join(line.rstrip(b"\r") + b"\n" for line in lines)
+
+ digest = xxhash.xxh32(text, seed=0).intdigest()
+ return f"0x{digest:08x}"
+
+
+def command_clarification_hash(args: argparse.Namespace) -> None:
+ """Print the clarification hash of one license file, and nothing else."""
+ path: Path = args.file
+ if not path.is_file():
+ raise Failure(f"No such file: {path}")
+ print(clarification_file_hash(path))
+
+
def command_check_dependencies(_args: argparse.Namespace) -> None:
step("Check dependency licenses")
require_cargo_deny()
Review Comment:
OK, I may opt to make this change. I was originally okay with running it
twice such that you see clear and familiar output when a normal error is
reported, but I think it's weird, unclear, and wasteful to run twice.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
