dannycjones commented on code in PR #3299: URL: https://github.com/apache/iceberg-rust/pull/3299#discussion_r4135902775
########## dev/release/licenses.py: ########## @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# /// script +# requires-python = ">=3.11" +# dependencies = ["xxhash"] +# /// + +""" +CLI for assisting maintenance of the project's license checks and any generated artifacts. +""" + +import argparse +import json +import subprocess +import sys +import textwrap +from collections import defaultdict +from pathlib import Path + +if sys.version_info < (3, 11): + sys.exit( + "This script needs Python 3.11 or newer for tomllib; found " + f"{sys.version.split()[0]} at {sys.executable}." + ) + +import tomllib + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +DENY_TOML = REPO_ROOT / "deny.toml" + +type CrateName = str +type CrateVersion = str +type Crate = tuple[CrateName, CrateVersion] + + +class Failure(Exception): + """A user-facing error. main() prints it and exits non-zero.""" + + def __init__(self, message: str): + super().__init__(message) + + +def step(message: str) -> None: + print(f"==> {message}", flush=True) + + +def ok(message: str) -> None: + print(f"OK: {message}", flush=True) + + +def require_cargo_deny() -> None: + """Assert cargo-deny is installed. + + Deliberately not version-pinned. The pin in dev/release/generate-dependency-tsv-files.sh exists + because the DEPENDENCIES.rust.tsv files it generates differ between cargo-deny + versions; nothing here is committed, so a mismatched binary can only change the + wording of a diagnostic. + """ + try: + subprocess.run( + ["cargo", "deny", "--version"], + capture_output=True, + check=True, + ) + except FileNotFoundError: + raise Failure("This script requires 'cargo', but it is not installed.") + except subprocess.CalledProcessError: + raise Failure( + "This script requires 'cargo-deny' for dependency license checks.\n" + "Install it with: cargo install --locked cargo-deny" + ) + + +def declared_clarifications() -> set[CrateName]: + """Crate names that deny.toml carries a [[licenses.clarify]] block for.""" + if not DENY_TOML.is_file(): + raise Failure( + f"Expected the cargo-deny config at {DENY_TOML}, but it is missing." + ) + + with DENY_TOML.open("rb") as handle: + config = tomllib.load(handle) + + blocks = config.get("licenses", {}).get("clarify", []) + if not blocks: + raise Failure( + f"Expected at least one [[licenses.clarify]] block in {DENY_TOML}, found none.\n" + "If the last one was deliberately removed, remove this check with it." + ) + + unnamed = [index for index, block in enumerate(blocks) if "crate" not in block] + if unnamed: + raise Failure( + f"{DENY_TOML} has [[licenses.clarify]] block(s) with no `crate` key, at " + f"position(s) {', '.join(str(index + 1) for index in unnamed)}." + ) + return {block["crate"] for block in blocks} + + +def evaluated_crates() -> tuple[ + dict[CrateName, set[CrateVersion]], dict[CrateName, set[CrateVersion]] +]: + """Every crate cargo-deny judged, along with the subset that had a clarification applied.""" + # cargo-deny is invoked again here, but output is suppressed and JSON is captured to processing. + result = subprocess.run( + ["cargo", "deny", "--format", "json", "check", "license", "-W", "accepted"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + # cargo-deny writes its diagnostics to stderr, including in JSON mode. + diagnostics = result.stderr or result.stdout + if result.returncode != 0: + print(diagnostics, file=sys.stderr, end="") + raise Failure( + "cargo-deny failed while re-running to verify that no license " + "clarifications are ignored." + ) + + evaluated: dict[CrateName, set[CrateVersion]] = defaultdict(set) + clarified: dict[CrateName, set[CrateVersion]] = defaultdict(set) + for line in diagnostics.splitlines(): + if not line.strip(): + continue + try: + diagnostic = json.loads(line) + except json.JSONDecodeError: + raise Failure( + "cargo-deny emitted a line that is not JSON, so its output cannot be " + f"read reliably:\n {line[:200]}" + ) + + fields = diagnostic.get("fields", {}) + + if fields.get("code") != "accepted": + # This fn is to fail on any bad accepted outcomes, so ignoring anything else is fine. + continue + + # `or []`, not a .get default: a JSON null would otherwise reach len() below + # as None and raise TypeError instead of the Failure written for it. + graphs = fields.get("graphs") or [] + if len(graphs) != 1: + # There's a few types of diagnostic, + # but for 'accepted' we expect a single graph for the one crate it covers. + raise Failure( + f"cargo-deny reported an 'accepted' license verdict covering " + f"{len(graphs)} crates, but one was expected, so the crate it applies " + f"to is ambiguous:\n {line[:200]}" + ) + krate = graphs[0].get("Krate", {}) + + name, version = krate.get("name"), krate.get("version") + if name is None or version is None: + raise Failure( + "cargo-deny reported an 'accepted' license verdict without naming both " + f"the crate and its version:\n {line[:200]}" + ) + + evaluated[name].add(version) + labels = fields.get("labels") or [] + CLARIFICATION_APPLIED_MSG = "license expression retrieved via user override" + if any(label.get("message") == CLARIFICATION_APPLIED_MSG for label in labels): + clarified[name].add(version) + + return dict(evaluated), dict(clarified) + + +def cargo_metadata() -> dict: + """Resolved cargo metadata for this workspace. + + `--locked` so resolving it cannot rewrite Cargo.lock, least of all part-way through + cutting a release. + """ + result = subprocess.run( + [ + "cargo", + "metadata", + "--format-version", + "1", + "--locked", + "--manifest-path", + str(REPO_ROOT / "Cargo.toml"), + ], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise Failure(f"cargo metadata failed:\n{result.stderr}") + return json.loads(result.stdout) + + +def crate_source_dirs(crates: set[Crate]) -> dict[Crate, str]: + """Where cargo unpacked each crate's sources. + + + The directory is `<name>-<version>`, but names contain hyphens (brotli-decompressor-5.0.3) and + versions carry build metadata (zstd-sys-2.0.16+zstd.1.5.7). + """ + packages = cargo_metadata()["packages"] + + dict_crate_to_path = {} + + for package in packages: + crate = (package["name"], package["version"]) + if crate not in crates: + continue + dict_crate_to_path[crate] = str(Path(package["manifest_path"]).parent) + + return dict_crate_to_path + + +def require_clarifications_applied( + declared: set[CrateName], + evaluated: dict[CrateName, set[CrateVersion]], + clarified: dict[CrateName, set[CrateVersion]], +) -> None: + """ + Fail if a clarification did not reach a crate version that was evaluated. + + `declared` is all crates declared with a clarification in deny.toml, + `evaluated` is the list of all crate versions evaluated as part of the dependency graph, + and `clarified` is the subset of crate versions evaluated which had a clarification applied. + """ + crates_with_clarifications_missed: list[Crate] = sorted( + (crate_name, crate_version) + for crate_name in declared & evaluated.keys() + for crate_version in evaluated[crate_name] - clarified.get(crate_name, set()) + ) + if not crates_with_clarifications_missed: + return + + source_dirs = crate_source_dirs(set(crates_with_clarifications_missed)) + error_line_output = [] + for crate_name, crate_version in crates_with_clarifications_missed: + source_dir = source_dirs.get( + (crate_name, crate_version), "<source not unpacked>" + ) + error_line_output.append( + f" {crate_name} {crate_version} ({source_dir}): clarification did not apply" + ) + + raise Failure( + textwrap.dedent(""" + {deny_toml} declares license clarifications that cargo-deny did not apply: + {crates} + + A clarification is discarded by cargo-deny when any of its attested license + files are no longer valid (hash mismatch, etc.). + + Read the file first and check it still says what the block claims. + If it does, compute correct hashes for the license files using the following command. + + uv run {script} dependencies clarification-hash \\ + <crate directory above>/<license file> + """).format( + deny_toml=DENY_TOML, + crates="\n".join(error_line_output), + script=Path(__file__).name, Review Comment: This is a useful improvement, I'll apply it. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
