This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-3528-36aa0e435a13ed98e22af1851f81d16a3651c4d9 in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
commit 340b6fa0d1e44fbcfdac68174298519e35819e47 Author: NingLin Liu <[email protected]> AuthorDate: Tue Oct 27 05:09:03 2026 +0000 chore(dev): fold the crate LICENSE/NOTICE check into licenses.py (#3528) * chore(dev): fold the crate LICENSE/NOTICE check into licenses.py * chore(dev): address review on crates check - drop --no-verify (implied by --list) and inline the --allow-dirty comment - check cargo the same way as cargo-deny and keep the two helpers together - don't capture cargo metadata stderr so warnings stay visible - compare against the root file without raising when either side is missing - reword the crates sub-command help --------- Co-authored-by: NingLin Liu <[email protected]> --- Makefile | 2 +- dev/check_license_notice.sh | 188 -------------------------------------------- dev/release/licenses.py | 144 +++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 189 deletions(-) diff --git a/Makefile b/Makefile index 9da47d186..485d2866b 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,7 @@ check-public-api: install-cargo-public-api if [ $$fail -ne 0 ]; then exit 1; fi check-license-notice: - bash ./dev/check_license_notice.sh + python3 ./dev/release/licenses.py crates check # Keep version in sync with the CI lint install step in .github/workflows/ci.yml. install-cargo-deny: diff --git a/dev/check_license_notice.sh b/dev/check_license_notice.sh deleted file mode 100755 index 17757fb9a..000000000 --- a/dev/check_license_notice.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Every distributed artifact must ship its own LICENSE and NOTICE. -# Checking the packaged file list, rather than the crate directory, -# also catches files that exist on disk but are excluded from the published -# crate. Each crate's copy must be the repository root file itself, so the -# license text cannot drift from one crate to the next. - -set -Eeuo pipefail - -REQUIRED_FILES=(LICENSE NOTICE) - -usage() { - cat <<USAGE -Usage: - $0 [-h|--help] - -Checks that every publishable crate in the workspace packages a top-level -LICENSE and NOTICE file, as required for Apache source and binary releases. -USAGE -} - -require_command() { - local command_name="$1" - if ! command -v "${command_name}" >/dev/null 2>&1; then - echo "This script requires '${command_name}', but it is not installed." >&2 - return 1 - fi -} - -report_error() { - # GitHub Actions turns `::error::` lines into annotations on the job summary. - if [ "${GITHUB_ACTIONS:-}" = "true" ]; then - echo "::error::$1" - else - echo "ERROR: $1" >&2 - fi -} - -# Return name and manifest path for every crate that would be published in a release. -publishable_crates() { - # `publish` is an empty list for `publish = false` crates and null when the - # crate may be published to any registry. - cargo metadata --no-deps --format-version 1 | - jq -r '.packages[] | select(.publish != []) | "\(.name)\t\(.manifest_path)"' -} - -# Check the path resolves to a non-empty regular file, following any symlinks. -check_file_exists() { - local path="$1" - - # `-f` follows symlinks and rejects directories. - # `-s` additionally requires content, so an emptied file fails too. - if [ ! -f "${path}" ] || [ ! -s "${path}" ]; then - return 1 - fi -} - -# Return true if both paths are the same file. -is_same_file() { - # `-ef` is short for "equal file": it is true when both arguments resolve to the - # same device and inode. - [ "$1" -ef "$2" ] -} - -# Return true if the given path (second arg) is present in the list of paths (first arg). -packages_file_at_root() { - local packaged_paths="$1" - local file_name="$2" - - local packaged_path - - while IFS= read -r packaged_path; do - if [ "${packaged_path}" = "${file_name}" ]; then - return 0 - fi - done <<<"${packaged_paths}" - - return 1 -} - -# Verify that the necessary files are bundled in the crate. -# Accepts both crate name and its directory, so that artifacts can be fully verified including symlink resolution. -check_crate() { - local crate="$1" - local crate_dir="$2" - - local packaged_paths status repo_path - status=0 - - echo "Checking ${crate}..." - # --no-verify is no-op at the time of writing, but to be clear: packages will not be built, only expected files enumerated - # --allow-dirty allows for `make check` during development - if ! packaged_paths="$(cargo package --package "${crate}" --list --locked --no-verify --allow-dirty)"; then - report_error "${crate} could not be listed by cargo; see the error above" - return 1 - fi - - for required_file in "${REQUIRED_FILES[@]}"; do - # Not only verify that the path is included, but verify that it links to the root file. - repo_path="${crate_dir}/${required_file}" - if ! packages_file_at_root "${packaged_paths}" "${required_file}"; then - report_error "${crate} crate is missing ${required_file} in packaged file list" - status=1 - elif ! check_file_exists "${repo_path}"; then - report_error "${crate} ${required_file} does not resolve to a non-empty file: ${repo_path}" - status=1 - elif ! is_same_file "${repo_path}" "${REPO_ROOT}/${required_file}"; then - report_error "${crate} ${required_file} does not resolve to the repository root ${required_file}: ${repo_path}" - status=1 - fi - done - - return "${status}" -} - -# Route CLI args -while [ "$#" -gt 0 ]; do - case "$1" in - -h | --help) - usage - exit 0 - ;; - *) - echo "Unknown argument: $1" >&2 - usage >&2 - exit 1 - ;; - esac -done - -require_command cargo -require_command jq - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -cd "${REPO_ROOT}" - -FAILED=0 - -echo "Checking repository root..." -for required_file in "${REQUIRED_FILES[@]}"; do - if ! check_file_exists "${REPO_ROOT}/${required_file}"; then - report_error "Repository root ${required_file} does not resolve to a non-empty file: ${REPO_ROOT}/${required_file}" - FAILED=1 - fi -done - -# Check for non-zero exit code, otherwise store in `PUBLISHABLE_CRATES` -# Must be checked here rather than piped directly, otherwise the error code is ignored -if ! PUBLISHABLE_CRATES="$(publishable_crates)"; then - report_error "Could not list the publishable crates" - exit 1 -fi - -# Verify at least one crate was listed -if [ -z "${PUBLISHABLE_CRATES}" ]; then - report_error "No publishable crates found in the workspace, expected at least one" - exit 1 -fi - -while IFS=$'\t' read -r crate manifest_path; do - check_crate "${crate}" "$(dirname "${manifest_path}")" || FAILED=1 -done <<<"${PUBLISHABLE_CRATES}" - -if [ "${FAILED}" -ne 0 ]; then - echo "Every publishable crate must package a top-level LICENSE and NOTICE resolving to the repository root copies." >&2 - exit 1 -fi - -echo "All publishable crates package LICENSE and NOTICE." diff --git a/dev/release/licenses.py b/dev/release/licenses.py index 2fc744987..a0bc83c19 100755 --- a/dev/release/licenses.py +++ b/dev/release/licenses.py @@ -20,12 +20,17 @@ CLI for assisting maintenance of the project's license checks. """ import argparse +import json +import os import subprocess import sys from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent.parent +# Every distributed artifact must ship its own copy of these files. +REQUIRED_CRATE_FILES = ("LICENSE", "NOTICE") + class Failure(Exception): """A user-facing error.""" @@ -39,6 +44,26 @@ def ok(message: str) -> None: print(f"OK: {message}", flush=True) +def report_error(message: str) -> None: + # GitHub Actions turns `::error::` lines into annotations on the job summary. + if os.environ.get("GITHUB_ACTIONS") == "true": + print(f"::error::{message}", flush=True) + else: + print(f"ERROR: {message}", file=sys.stderr, flush=True) + + +def require_cargo() -> None: + """Assert cargo is installed.""" + try: + subprocess.run( + ["cargo", "--version"], + capture_output=True, + check=True, + ) + except (FileNotFoundError, subprocess.CalledProcessError): + raise Failure("This script requires 'cargo', but it is not installed.") + + def require_cargo_deny() -> None: """Assert cargo-deny is installed.""" try: @@ -56,6 +81,113 @@ def require_cargo_deny() -> None: ) +def is_non_empty_file(path: Path) -> bool: + """Whether the path resolves, following symlinks, to a non-empty regular file.""" + return path.is_file() and path.stat().st_size > 0 + + +def is_same_file(path: Path, other: Path) -> bool: + """Whether both paths exist and resolve to the same file; False if either is missing.""" + try: + return path.samefile(other) + except OSError: + return False + + +def publishable_crates() -> list[tuple[str, Path]]: + """Name and directory of every crate that would be published in a release.""" + result = subprocess.run( + ["cargo", "metadata", "--no-deps", "--format-version", "1"], + cwd=REPO_ROOT, + stdout=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode != 0: + raise Failure("Could not list the publishable crates.") + + # `publish` is an empty list for `publish = false` crates and null when the + # crate may be published to any registry. + return [ + (package["name"], Path(package["manifest_path"]).parent) + for package in json.loads(result.stdout)["packages"] + if package["publish"] != [] + ] + + +def check_crate_files(crate: str, crate_dir: Path) -> bool: + """Check the packaged crate ships LICENSE and NOTICE resolving to the root files.""" + print(f"Checking {crate}...", flush=True) + result = subprocess.run( + [ + "cargo", + "package", + "--package", + crate, + "--list", + "--locked", + # allows `make check` during development. + "--allow-dirty", + ], + cwd=REPO_ROOT, + stdout=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode != 0: + report_error(f"{crate} could not be listed by cargo; see the error above") + return False + + packaged_paths = set(result.stdout.splitlines()) + passed = True + for file_name in REQUIRED_CRATE_FILES: + repo_path = crate_dir / file_name + if file_name not in packaged_paths: + report_error(f"{crate} crate is missing {file_name} in packaged file list") + passed = False + elif not is_non_empty_file(repo_path): + report_error(f"{crate} {file_name} does not resolve to a non-empty file: {repo_path}") + passed = False + elif not is_same_file(repo_path, REPO_ROOT / file_name): + report_error( + f"{crate} {file_name} does not resolve to the repository root {file_name}: {repo_path}" + ) + passed = False + return passed + + +def command_check_crates(_args: argparse.Namespace) -> None: + # Checking the packaged file list, rather than the crate directory, also + # catches files that exist on disk but are excluded from the published crate. + # Each crate's copy must be the repository root file itself, so the license + # text cannot drift from one crate to the next. + step("Check crate LICENSE and NOTICE files") + require_cargo() + + passed = True + for file_name in REQUIRED_CRATE_FILES: + if not is_non_empty_file(REPO_ROOT / file_name): + report_error( + f"Repository root {file_name} does not resolve to a non-empty file: {REPO_ROOT / file_name}" + ) + passed = False + + crates = publishable_crates() + if not crates: + raise Failure("No publishable crates found in the workspace, expected at least one.") + + for crate, crate_dir in crates: + passed = check_crate_files(crate, crate_dir) and passed + + if not passed: + raise Failure( + "Every publishable crate must package a top-level LICENSE and NOTICE " + "resolving to the repository root copies." + ) + + ok("Check crate LICENSE and NOTICE files") + + def command_check_dependencies(_args: argparse.Namespace) -> None: step("Check dependency licenses") require_cargo_deny() @@ -87,6 +219,18 @@ def main() -> int: ) check.set_defaults(func=command_check_dependencies) + crates = cmd_groups.add_parser( + "crates", + help="Sub-commands for working with licensing on crates", + ) + crates_commands = crates.add_subparsers(dest="command", required=True) + + check = crates_commands.add_parser( + "check", + help="assert every publishable crate packages the repository root LICENSE and NOTICE", + ) + check.set_defaults(func=command_check_crates) + args = parser.parse_args() try: args.func(args)
