This is an automated email from the ASF dual-hosted git repository. vinishjail97 pushed a commit to branch 700-validate-bundled-license-texts in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
commit 231f88ca0e130396042c9980882b4c0f362e16b9 Author: Vinish Reddy <[email protected]> AuthorDate: Thu Aug 6 14:09:32 2026 -0700 [700] Validate bundled license texts against the built jars Checks each *-bundled.jar rather than the poms: reads the coordinate listing the jar carries in META-INF/LICENSE, confirms each non-Apache-2.0 coordinate's classes are actually present, and requires a matching META-INF/licenses/LICENSE-<artifactId>. The reverse direction is checked too, so a text with no bundled dependency behind it fails as well. The published artifact is the only thing that settles what is bundled. A shade <includes> list can name dependencies that no longer resolve, and a module with <filters> ships a subset of what the dependency tree describes. It is also what a release reviewer inspects. This currently fails, which is the point: on main it reports 12 bundled non-Apache-2.0 dependencies with no license text (2 in xtable-aws, 10 in xtable-hive-metastore) plus a further 10 whose text ships under a non-conventional name. It must not be merged before those are addressed. Enforces the LICENSE-<artifactId> convention, so texts carried over from the hand-maintained set (LICENSE-jruby-jcodings, LICENSE-leveldbjni, LICENSE-javax-annotation-api) are reported until renamed. --- .github/workflows/mvn-license-check.yml | 9 ++ release/scripts/validate_bundled_license_texts.py | 152 ++++++++++++++++++++++ 2 files changed, 161 insertions(+) diff --git a/.github/workflows/mvn-license-check.yml b/.github/workflows/mvn-license-check.yml index c32daa62..9cb56592 100644 --- a/.github/workflows/mvn-license-check.yml +++ b/.github/workflows/mvn-license-check.yml @@ -47,3 +47,12 @@ jobs: - name: Validate Shaded Dependency License Metadata run: release/scripts/validate_shaded_license_coverage.sh + + - name: Build Shaded Bundles + run: > + ./mvnw -B -DskipTests -Dmaven.build.cache.enabled=false + -pl xtable-aws,xtable-hive-metastore,xtable-hudi-support/xtable-hudi-support-extensions + -am package + + - name: Validate Bundled License Texts + run: python3 release/scripts/validate_bundled_license_texts.py diff --git a/release/scripts/validate_bundled_license_texts.py b/release/scripts/validate_bundled_license_texts.py new file mode 100644 index 00000000..8770aa3f --- /dev/null +++ b/release/scripts/validate_bundled_license_texts.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Check that every bundled non-Apache-2.0 dependency ships its license text. + +This reads the built ``*-bundled.jar`` rather than the poms. The published +artifact is the only thing that settles what is actually bundled: a shade +``<includes>`` list can name dependencies that no longer resolve, and a module +with ``<filters>`` or an uber-jar dependency ships classes the dependency tree +does not describe. It is also the artifact a release reviewer inspects. + +For each bundled jar, two directions are checked against the coordinate listing +the jar carries in ``META-INF/LICENSE``: + + missing a non-Apache-2.0 dependency whose classes are in the jar, with no + ``META-INF/licenses/LICENSE-<artifactId>`` + orphaned a ``META-INF/licenses/`` text with no such dependency behind it + +Usage: validate_bundled_license_texts.py [<jar> ...] +With no arguments, every ``*/target/*-bundled.jar`` in the repo is checked, and +the run fails if none were found -- an empty check must never report success. +""" +from __future__ import annotations + +import pathlib +import re +import sys +import zipfile + +APACHE_FAMILY = "Apache License 2.0" +M2 = pathlib.Path.home() / ".m2" / "repository" +ROOT = pathlib.Path(__file__).resolve().parents[2] + +# A dependency's own classes are sampled and looked for in the bundle. Shade +# drops unreferenced classes in modules that configure <filters>, so a bundled +# dependency does not contribute all of its classes -- but it always +# contributes far more than none. +SAMPLE = 80 +PRESENT_RATIO = 4 + + +def parse_bundled_listing(text: str) -> dict[str, list[tuple[str, str, str]]]: + """family -> coordinates, from the LICENSE-bundled layout. + + A family is a line immediately followed by a dashed underline; coordinates + are the ``group:artifact:version`` lines beneath it. + """ + groups: dict[str, list[tuple[str, str, str]]] = {} + family = None + lines = text.splitlines() + for index, line in enumerate(lines): + stripped = line.strip() + if not stripped: + family = None + elif index + 1 < len(lines) and re.fullmatch(r"-{3,}", lines[index + 1].strip()): + family = stripped + elif re.fullmatch(r"-{3,}", stripped): + continue + elif family and stripped.count(":") == 2: + groups.setdefault(family, []).append(tuple(stripped.split(":"))) + return groups + + +def dependency_classes(group: str, artifact: str, version: str) -> list[str] | None: + """Class entries of the dependency's own jar, or None if it cannot be found.""" + base = M2 / pathlib.Path(group.replace(".", "/")) / artifact / version + candidates = [base / f"{artifact}-{version}.jar", *sorted(base.glob(f"{artifact}-{version}-*.jar"))] + for path in candidates: + if not path.exists(): + continue + with zipfile.ZipFile(path) as dependency: + classes = [entry for entry in dependency.namelist() if entry.endswith(".class")] + if classes: + return classes[:SAMPLE] + return None + + +def check(jar_path: pathlib.Path) -> tuple[list, list, int]: + with zipfile.ZipFile(jar_path) as jar: + names = jar.namelist() + listing_entry = "META-INF/LICENSE" if "META-INF/LICENSE" in names else "META-INF/LICENSE-bundled" + if listing_entry not in names: + raise SystemExit(f"FAIL {jar_path.name}: no META-INF/LICENSE to validate against") + listing = jar.read(listing_entry).decode("utf-8") + bundled_classes = {entry for entry in names if entry.endswith(".class")} + texts = {entry.split("/")[-1] for entry in names if entry.startswith("META-INF/licenses/") and not entry.endswith("/")} + + missing, in_jar_non_apache = [], [] + for family, coords in parse_bundled_listing(listing).items(): + if family == APACHE_FAMILY: + continue + for group, artifact, version in coords: + sample = dependency_classes(group, artifact, version) + if sample is None: + continue # unresolvable, so nothing of it can be in the jar + if sum(entry in bundled_classes for entry in sample) < max(1, len(sample) // PRESENT_RATIO): + continue # listed, but its classes are not here + in_jar_non_apache.append((group, artifact, version, family)) + if f"LICENSE-{artifact}" not in texts: + missing.append((group, artifact, version, family)) + + expected = {f"LICENSE-{artifact}" for _, artifact, _, _ in in_jar_non_apache} + orphaned = sorted(texts - expected) + return missing, orphaned, len(in_jar_non_apache) + + +def main() -> None: + if len(sys.argv) > 1: + jars = [pathlib.Path(argument) for argument in sys.argv[1:]] + else: + jars = sorted(ROOT.glob("**/target/*-bundled.jar")) + if not jars: + raise SystemExit( + "FAIL: no */target/*-bundled.jar found; nothing was validated.\n" + "Build the shaded modules first: ./mvnw -DskipTests package" + ) + + status = 0 + for jar_path in jars: + missing, orphaned, checked = check(jar_path) + if missing: + status = 1 + print(f"FAIL {jar_path.name}: bundled non-Apache-2.0 dependencies with no META-INF/licenses/ text:") + for group, artifact, version, family in sorted(missing): + print(f" - LICENSE-{artifact} ({group}:{artifact}:{version}, {family})") + if orphaned: + status = 1 + print(f"FAIL {jar_path.name}: META-INF/licenses/ texts with no bundled non-Apache-2.0 dependency:") + for name in orphaned: + print(f" - {name}") + if not missing and not orphaned: + print(f"OK {jar_path.name}: {checked} bundled non-Apache-2.0 dependencies, all with license texts.") + + raise SystemExit(status) + + +if __name__ == "__main__": + main()
