This is an automated email from the ASF dual-hosted git repository.
FrankChen021 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 46d2de3df9f build: only warn about stale licenses.yaml versions in PR
CI (#20774)
46d2de3df9f is described below
commit 46d2de3df9f8b0757336be0f59428bbab624b99f
Author: Andreas Maechler <[email protected]>
AuthorDate: Sat Oct 31 20:49:31 2026 -0600
build: only warn about stale licenses.yaml versions in PR CI (#20774)
* build: only warn about stale licenses.yaml versions in PR CI
check-licenses.py gets --stale-versions=fail|warn, defaulting to fail. With
warn, a reported dependency becomes a warning when its groupId and
artifactId
are registered under other versions, all with the reported license, and one
of those versions is no longer in any report. The warning gives the commands
to refresh licenses.yaml. Unregistered dependencies, changed or unknown
licenses, license_file_path problems and unknown module tags still fail, and
so does a module that starts bundling another version of a library while the
registered version still ships. If the dependency ships in the release,
registrations under excluded modules do not count.
distribution/pom.xml passes druid.distribution.licenses.staleVersions
(default
fail) to the check, and packaging-check.sh sets it to warn, so release
builds
stay strict.
* build: report stale licenses.yaml versions in strict mode too
check-licenses.py now separates stale versions from missing licenses in both
modes, so the release build lists them with the refresh hint instead of as
missing licenses. --stale-versions only decides whether they fail the check.
Drop the --stale-versions flag from license_checks_script.sh: nothing calls
that script.
* build: keep a shipped version registered only under excluded modules
missing
split_stale_versions treated a reported version that is registered only
under an excluded module, with an older version registered under a shipped
module, as stale. When an extension ships that version, the release has no
registration for the jar and refresh-license-versions.py neither updates nor
flags the entry. Only treat a missing version as stale when it is in no
licenses.yaml entry at all.
---
.github/scripts/packaging-check.sh | 4 +-
dev/license.md | 3 +
distribution/bin/check-licenses.py | 57 ++++++++++++++-
distribution/bin/test_check_licenses.py | 125 +++++++++++++++++++++++++++++++-
distribution/pom.xml | 3 +
5 files changed, 185 insertions(+), 7 deletions(-)
diff --git a/.github/scripts/packaging-check.sh
b/.github/scripts/packaging-check.sh
index 34584f57c8b..26dfda081e5 100755
--- a/.github/scripts/packaging-check.sh
+++ b/.github/scripts/packaging-check.sh
@@ -23,6 +23,8 @@ CODE_CHECKS_SKIP="-Dcheckstyle.skip -Dpmd.skip
-Dforbiddenapis.skip -Danimal.sni
./mvnw -B install -Papache-release -Prat --fail-at-end \
-pl '!benchmarks, !distribution' -P skip-tests -Dweb.console.skip=false \
-Dgpg.skip -Ddependency-check.skip $CODE_CHECKS_SKIP
+# Stale licenses.yaml versions only warn here; release builds fail on them.
./mvnw -B install -Papache-release -Prat -Pdist -Pbundle-contrib-exts
--fail-at-end \
-pl 'distribution' -P skip-tests -Dweb.console.skip=false \
- -Dgpg.skip -Ddependency-check.skip $CODE_CHECKS_SKIP
+ -Dgpg.skip -Ddependency-check.skip $CODE_CHECKS_SKIP \
+ -Ddruid.distribution.licenses.staleVersions=warn
diff --git a/dev/license.md b/dev/license.md
index 6ec63aa580c..c9907d345a6 100644
--- a/dev/license.md
+++ b/dev/license.md
@@ -134,3 +134,6 @@ it cannot update on its own, such as an entry whose
libraries moved to different
distribution/bin/generate-license-dependency-reports.py . target
--clean-maven-artifact-transfer --parallel 2
distribution/bin/refresh-license-versions.py licenses.yaml
target/license-reports
```
+
+If you leave a version as it is, PR CI only prints a warning, provided the
version it replaced is registered with the
+same license. Release builds still fail on these. A dependency that is new, or
whose license changed, fails PR CI too.
diff --git a/distribution/bin/check-licenses.py
b/distribution/bin/check-licenses.py
index 8051266b40c..564a48bac40 100755
--- a/distribution/bin/check-licenses.py
+++ b/distribution/bin/check-licenses.py
@@ -413,6 +413,29 @@ def find_missing_licenses(reported_dep_to_licenses,
reported_modules, registered
missing_licenses.append((", ".join(modules),) + key +
(reported_license_druid_module[0],))
return missing_licenses
+def split_stale_versions(missing_licenses, reported_modules,
registered_dep_to_licenses, registered_modules, exclude_module_prefixes):
+ # A missing dependency only has a stale version when its own version is in
no licenses.yaml entry, its groupId and
+ # artifactId are registered under other versions, all with the reported
license, and one of those versions is no
+ # longer reported. If it ships in the release, only registrations outside
the excluded modules count.
+ # Returns the stale ones, each with its registered versions appended, and
the rest.
+ stale_licenses = []
+ remaining_licenses = []
+ for missing_license in missing_licenses:
+ key, reported_license = missing_license[1:4], missing_license[4]
+ others = [k for k in registered_dep_to_licenses if k[:2] == key[:2]]
+ if any(not is_excluded_module(m, exclude_module_prefixes) for m in
reported_modules[key]):
+ others = [k for k in others if any(not is_excluded_module(m,
exclude_module_prefixes) for m in registered_modules[k])]
+ if (
+ key not in registered_dep_to_licenses
+ and others
+ and all(registered_dep_to_licenses[k] == reported_license for k in
others)
+ and any(k not in reported_modules for k in others)
+ ):
+ stale_licenses.append(missing_license + (", ".join(sorted(k[2] for
k in others)),))
+ else:
+ remaining_licenses.append(missing_license)
+ return stale_licenses, remaining_licenses
+
def find_unknown_licenses(reported_dep_to_licenses, reported_modules,
reported_license_names, skipping_licenses, exclude_module_prefixes):
# A dependency that ships in the release must report at least one license
in compatible_license_names.
unknown_licenses = []
@@ -449,7 +472,7 @@ def read_dependency_reports(dependency_reports_root,
compatible_license_names):
raise Exception("No dependency reports are found")
return reported_dep_to_licenses, reported_modules, reported_license_names
-def check_licenses(license_yaml, dependency_reports_root,
exclude_module_prefixes):
+def check_licenses(license_yaml, dependency_reports_root,
exclude_module_prefixes, stale_versions):
# Build a dictionary to facilitate comparing reported licenses and
registered ones.
# These dictionaries are the mapping of (group_id, artifact_id, version)
to license_name.
@@ -568,6 +591,23 @@ def check_licenses(license_yaml, dependency_reports_root,
exclude_module_prefixe
reported_dep_to_licenses, reported_modules, registered_modules,
skipping_licenses, exclude_module_prefixes
)
+ stale_licenses, missing_licenses = split_stale_versions(
+ missing_licenses, reported_modules, registered_dep_to_licenses,
registered_modules, exclude_module_prefixes
+ )
+
+ if len(stale_licenses) > 0:
+ print_log_to_stderr("{}: found {} dependencies whose previous versions
are registered with the same license. Release builds fail on these.".format(
+ "Warn" if stale_versions == "warn" else "Error",
len(stale_licenses)
+ ))
+ for stale_license in stale_licenses:
+ print_log_to_stderr("druid_module: {}, groupId: {}, artifactId:
{}, version: {}, license: {}, registered_versions: {}".format(*stale_license))
+ print_log_to_stderr(
+ "To update licenses.yaml, run
distribution/bin/generate-license-dependency-reports.py . target "
+ "--clean-maven-artifact-transfer --parallel 2 and then
distribution/bin/refresh-license-versions.py "
+ "licenses.yaml target/license-reports, which lists the entries it
cannot update."
+ )
+ print_log_to_stderr("")
+
if len(missing_licenses) > 0:
print_log_to_stderr("Error: found {} missing licenses. These licenses
are reported, but missing in the registry".format(len(missing_licenses)))
for missing_license in missing_licenses:
@@ -604,10 +644,14 @@ def check_licenses(license_yaml, dependency_reports_root,
exclude_module_prefixe
print_log_to_stderr("groupId: {}, artifactId: {}, version: {},
reported_license: {}".format(unchecked_license[0], unchecked_license[1],
unchecked_license[2], unchecked_license[3]))
print_log_to_stderr("")
- if len(mismatched_licenses) > 0 or len(missing_licenses) > 0 or
len(unknown_licenses) > 0:
+ if (
+ len(mismatched_licenses) > 0 or len(missing_licenses) > 0 or
len(unknown_licenses) > 0
+ or (len(stale_licenses) > 0 and stale_versions == "fail")
+ ):
print_log_to_stderr("Mismatched licenses:
{}".format(mismatched_licenses))
print_log_to_stderr("Missing licenses: {}".format(missing_licenses))
print_log_to_stderr("Unknown licenses: {}".format(unknown_licenses))
+ print_log_to_stderr("Stale versions: {}".format(stale_licenses))
sys.exit(1)
@@ -623,11 +667,18 @@ if __name__ == "__main__":
help='Module prefix whose licenses.yaml entries the release
LICENSE omits. Repeatable. A dependency '
'that ships in the release must be registered under at least
one module not matching these.'
)
+ parser.add_argument(
+ '--stale-versions',
+ choices=['fail', 'warn'],
+ default='fail',
+ help='Whether a dependency whose previous version is registered
with the same license fails the check '
+ '(default) or only prints a warning.'
+ )
args = parser.parse_args()
license_yaml = args.license_yaml
dependency_reports_root = args.dependency_reports_root
- check_licenses(license_yaml, dependency_reports_root,
args.exclude_module_prefix)
+ check_licenses(license_yaml, dependency_reports_root,
args.exclude_module_prefix, args.stale_versions)
except KeyboardInterrupt:
print('Interrupted, closing.')
diff --git a/distribution/bin/test_check_licenses.py
b/distribution/bin/test_check_licenses.py
index 4eee01a8bee..1859b13acb5 100755
--- a/distribution/bin/test_check_licenses.py
+++ b/distribution/bin/test_check_licenses.py
@@ -15,9 +15,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import contextlib
import importlib.util
import io
import os
+import tempfile
import unittest
spec = importlib.util.spec_from_file_location(
@@ -58,6 +60,77 @@ class FindMissingLicensesTest(unittest.TestCase):
self.assertEqual([], find_missing([AZURE], [], skipping={GAV: {}}))
+class SplitStaleVersionsTest(unittest.TestCase):
+ MIT = "MIT License"
+ NEW = check_licenses.get_dep_key("org.example", "a", "1.1")
+
+ def split(self, registered, reporting_modules=(AZURE,),
reported_license=APACHE, also_reported=()):
+ # registered: (version, license, modules) of each org.example:a
registration;
+ # also_reported: "g:a:v" and module pairs reported next to
org.example:a:1.1.
+ registered_dep_to_licenses = {}
+ registered_modules = {}
+ for version, license, modules in registered:
+ key = check_licenses.get_dep_key("org.example", "a", version)
+ registered_dep_to_licenses[key] = license
+ registered_modules[key] = set(modules)
+ reported_modules = reports(*also_reported)
+ reported_modules[self.NEW] = set(reporting_modules)
+ missing = [(", ".join(reporting_modules),) + self.NEW +
(reported_license,)]
+ return check_licenses.split_stale_versions(
+ missing, reported_modules, registered_dep_to_licenses,
registered_modules, EXCLUDED
+ )
+
+ def assertStale(self, registered_versions, split, module=AZURE):
+ self.assertEqual(([(module,) + self.NEW + (APACHE,
registered_versions)], []), split)
+
+ def assertMissing(self, split, reported_license=APACHE):
+ self.assertEqual(([], [(AZURE,) + self.NEW + (reported_license,)]),
split)
+
+ def
test_registered_under_another_version_with_the_same_license_is_stale(self):
+ self.assertStale("1.0", self.split([("1.0", APACHE, [AZURE])]))
+
+ def test_registered_under_another_version_of_another_module_is_stale(self):
+ self.assertStale("1.0", self.split([("1.0", APACHE, ["java-core"])]))
+
+ def test_unregistered_is_missing(self):
+ self.assertMissing(self.split([]))
+
+ def
test_registered_under_another_version_with_another_license_is_missing(self):
+ self.assertMissing(self.split([("1.0", self.MIT, [AZURE])]))
+
+ def
test_registered_under_versions_with_different_licenses_is_missing(self):
+ self.assertMissing(self.split([("1.0", APACHE, [AZURE]), ("0.9",
self.MIT, ["java-core"])]))
+
+ def
test_registered_under_several_versions_with_the_same_license_is_stale(self):
+ self.assertStale("0.9, 1.0", self.split([("1.0", APACHE, [AZURE]),
("0.9", APACHE, ["java-core"])]))
+
+ def
test_registered_only_under_excluded_module_is_missing_when_shipped(self):
+ self.assertMissing(self.split([("1.0", APACHE, [RANGER])]))
+
+ def
test_current_version_registered_only_under_excluded_module_is_missing_when_shipped(self):
+ self.assertMissing(self.split([("1.1", APACHE, [RANGER]), ("1.0",
APACHE, ["java-core"])]))
+
+ def
test_registrations_under_excluded_modules_are_ignored_when_shipped(self):
+ self.assertStale("1.0", self.split([("1.0", APACHE, [AZURE]), ("0.9",
self.MIT, [RANGER])]))
+
+ def test_registered_under_excluded_module_is_stale_when_not_shipped(self):
+ self.assertStale(
+ "1.0", self.split([("1.0", APACHE, [TESTCONTAINERS])],
reporting_modules=(TESTCONTAINERS,)), module=TESTCONTAINERS
+ )
+
+ def test_registered_version_still_reported_is_missing(self):
+ # A module that starts bundling another version of a library is not a
version bump.
+ self.assertMissing(self.split([("1.0", APACHE, ["java-core"])],
also_reported=[("org.example:a:1.0", "java-core")]))
+
+ def test_one_registered_version_no_longer_reported_is_stale(self):
+ self.assertStale("0.9, 1.0", self.split(
+ [("1.0", APACHE, [AZURE]), ("0.9", APACHE, ["java-core"])],
also_reported=[("org.example:a:0.9", "java-core")]
+ ))
+
+ def test_unknown_reported_license_is_missing(self):
+ self.assertMissing(self.split([("1.0", APACHE, [AZURE])],
reported_license=None), reported_license=None)
+
+
def find_unknown(license, reporting_modules, skipping=None):
return check_licenses.find_unknown_licenses(
{GAV: (license, reporting_modules[0])}, {GAV: set(reporting_modules)},
{GAV: ["Some New License"]}, skipping or {}, EXCLUDED
@@ -238,16 +311,19 @@ class RefreshVersionsTest(unittest.TestCase):
self.assertEqual((text, [], []), self.refresh(text,
reports(("org.s:a:1.1", "java-core"), ("org.c:a:1.1", "java-core"))))
-def parse_report(*licenses):
+def report_html(*licenses):
row = ", ".join("<a>{}</a>".format(license) for license in licenses)
- report = (
+ return (
'<a id="Project_Dependencies_compile"></a><table>'
"<tr><th>GroupId</th><th>ArtifactId</th><th>Version</th><th>Type</th><th>Licenses</th></tr>"
"<tr><td>org.tukaani</td><td>xz</td><td>1.10</td><td>jar</td><td>{}</td></tr>"
"</table>".format(row)
)
+
+
+def parse_report(*licenses):
parser = check_licenses.DependencyReportParser("java-core",
check_licenses.build_compatible_license_names())
- dep_to_license = parser.parse(io.StringIO(report))
+ dep_to_license = parser.parse(io.StringIO(report_html(*licenses)))
return dep_to_license, parser.dep_to_license_names
@@ -291,6 +367,49 @@ class CompatibleLicenseNamesTest(unittest.TestCase):
self.assertEqual([], sorted(set(compatible_license_names.values()) -
set(compatible_license_names)))
+class CheckLicensesTest(unittest.TestCase):
+ # The report lists org.tukaani:xz:1.10 under the Apache license.
+
+ def check(self, registered_entry, stale_versions):
+ # Returns the stderr output and the exit code, None when the check
passes.
+ with tempfile.TemporaryDirectory() as root:
+ os.makedirs(os.path.join(root, "licenses", "bin"))
+ os.makedirs(os.path.join(root, "reports", "core"))
+ with open(os.path.join(root, "licenses.yaml"), "w",
encoding="utf-8") as f:
+ f.write(licenses_yaml(registered_entry))
+ with open(os.path.join(root, "reports", "core",
"dependencies.html"), "w", encoding="utf-8") as f:
+ f.write(report_html("Apache-2.0"))
+ stderr = io.StringIO()
+ with contextlib.redirect_stderr(stderr):
+ try:
+ check_licenses.check_licenses(
+ os.path.join(root, "licenses.yaml"),
os.path.join(root, "reports"), EXCLUDED, stale_versions
+ )
+ except SystemExit as e:
+ return stderr.getvalue(), e.code
+ return stderr.getvalue(), None
+
+ def assertMissing(self, registered_entry, stale_versions):
+ output, code = self.check(registered_entry, stale_versions)
+ self.assertEqual(1, code)
+ self.assertIn("Error: found 1 missing licenses", output)
+
+ def test_stale_version_fails_in_fail_mode(self):
+ output, code = self.check(entry("XZ", "java-core", "1.9",
["org.tukaani: xz"]), "fail")
+ self.assertEqual(1, code)
+ self.assertIn("Error: found 1 dependencies whose previous versions are
registered", output)
+ self.assertIn("refresh-license-versions.py licenses.yaml
target/license-reports", output)
+
+ def test_stale_version_only_warns_in_warn_mode(self):
+ output, code = self.check(entry("XZ", "java-core", "1.9",
["org.tukaani: xz"]), "warn")
+ self.assertIsNone(code)
+ self.assertIn("version: 1.10, license: Apache License version 2.0,
registered_versions: 1.9", output)
+ self.assertIn("refresh-license-versions.py licenses.yaml
target/license-reports", output)
+
+ def test_unregistered_dependency_fails_in_warn_mode(self):
+ self.assertMissing(entry("Other", "java-core", "1.9", ["org.example:
other"]), "warn")
+
+
class LoadLicensesYamlTest(unittest.TestCase):
def test_versions_that_look_like_numbers_are_loaded_as_strings(self):
docs = check_licenses.licenses_yaml.load_all("version:
1.10\n---\nversion: 2\n---\nversion: 1.5.7-11\n")
diff --git a/distribution/pom.xml b/distribution/pom.xml
index ff0ead00c6a..bff170aeadc 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -50,6 +50,8 @@
<properties>
<!-- the default value is a repeated flag from the command line, since
blank value is not allowed -->
<druid.distribution.pulldeps.opts>--clean</druid.distribution.pulldeps.opts>
+ <!-- PR CI sets this to warn; release builds must fail on stale
licenses.yaml versions -->
+
<druid.distribution.licenses.staleVersions>fail</druid.distribution.licenses.staleVersions>
</properties>
<build>
@@ -343,6 +345,7 @@
LICENSE, which leaves out contrib
entries. -->
<argument>--exclude-module-prefix=extensions-contrib/</argument>
<argument>--exclude-module-prefix=extensions/druid-testcontainers</argument>
+
<argument>--stale-versions=${druid.distribution.licenses.staleVersions}</argument>
</arguments>
</configuration>
</execution>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
