This is an automated email from the ASF dual-hosted git repository.
FrankChen021 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 98aa0498fef fix: license check accepts entries the release LICENSE
drops (#20707)
98aa0498fef is described below
commit 98aa0498fefb746e6bc8b31538eb174af478b8d1
Author: Andreas Maechler <[email protected]>
AuthorDate: Fri Oct 23 23:40:59 2026 -0600
fix: license check accepts entries the release LICENSE drops (#20707)
* fix: license check accepts entries the release LICENSE drops
The release build generates LICENSE.BINARY and NOTICE.BINARY without
licenses.yaml entries tagged extensions-contrib/ or
extensions/druid-testcontainers, but check-licenses.py only required an
entry under any module. A jar that ships in the release and is registered
only under a contrib module passed the check and was left out of the
release LICENSE.
check-licenses.py now takes the same --exclude-module-prefix flags as the
release generators and requires a dependency reported by a shipped module
to have an entry under a module the release keeps. It records every module
that reports or registers a dependency instead of keeping only the last
one, and maps report directories to licenses.yaml module tags. Extension
reports are now named after the artifactId so the tags match. The
missing-license rule has unit tests, which the apache-release build runs
before the check.
Registers net.java.dev.jna:jna-platform 5.19.1 under
extensions/druid-azure-extensions; it was registered only under
extensions-contrib/druid-ranger-security.
* Shorten the license check exclusion comment
---
.github/scripts/license_checks_script.sh | 3 +-
distribution/bin/check-licenses.py | 54 ++++++++++++++----
.../bin/generate-license-dependency-reports.py | 11 ++--
distribution/bin/test_check_licenses.py | 66 ++++++++++++++++++++++
distribution/pom.xml | 14 +++++
licenses.yaml | 10 ++++
6 files changed, 143 insertions(+), 15 deletions(-)
diff --git a/.github/scripts/license_checks_script.sh
b/.github/scripts/license_checks_script.sh
index 168b616750e..ce7adfb575b 100755
--- a/.github/scripts/license_checks_script.sh
+++ b/.github/scripts/license_checks_script.sh
@@ -24,4 +24,5 @@ set -e
# Generate dependency reports and checks they are valid.
mkdir -p target
distribution/bin/generate-license-dependency-reports.py . target
--clean-maven-artifact-transfer --parallel 2
-distribution/bin/check-licenses.py licenses.yaml target/license-reports
+# Same exclusions as the release LICENSE generation in distribution/pom.xml
+distribution/bin/check-licenses.py licenses.yaml target/license-reports
--exclude-module-prefix=extensions-contrib/
--exclude-module-prefix=extensions/druid-testcontainers
diff --git a/distribution/bin/check-licenses.py
b/distribution/bin/check-licenses.py
index 311a0c09579..0125778e57e 100755
--- a/distribution/bin/check-licenses.py
+++ b/distribution/bin/check-licenses.py
@@ -348,6 +348,7 @@ def get_version_string(version):
return str(version)
def find_druid_module_name(dirpath):
+ # Returns the licenses.yaml module tag of the report:
extensions/<artifactId> or java-core.
ext_start = dirpath.find("/ext/")
if ext_start > 0:
# Found an extension
@@ -356,10 +357,10 @@ def find_druid_module_name(dirpath):
if ext_name_end < 0:
raise Exception("Can't determine extension name from
[{}]".format(dirpath))
else:
- return subpath[0:ext_name_end]
+ return "extensions/" + subpath[0:ext_name_end]
else:
# Druid core
- return "core"
+ return "java-core"
def check_license_files(license_yaml, licenses_list):
# Every license_file_path must exist, and every file in licenses/bin must
be referenced by a license_file_path,
@@ -386,7 +387,27 @@ def check_license_files(license_yaml, licenses_list):
if missing_files or unreferenced_files:
sys.exit(1)
-def check_licenses(license_yaml, dependency_reports_root):
+def is_excluded_module(module, exclude_module_prefixes):
+ return any(module.startswith(prefix) for prefix in exclude_module_prefixes)
+
+def find_missing_licenses(reported_dep_to_licenses, reported_modules,
registered_modules, skipping_licenses, exclude_module_prefixes):
+ # A dependency that ships in the release needs at least one entry outside
the excluded modules.
+ missing_licenses = []
+ for key, reported_license_druid_module in reported_dep_to_licenses.items():
+ if reported_license_druid_module[0] == "-" or key in skipping_licenses:
+ continue
+ shipped_modules = sorted(m for m in reported_modules[key] if not
is_excluded_module(m, exclude_module_prefixes))
+ released = any(not is_excluded_module(m, exclude_module_prefixes) for
m in registered_modules.get(key, ()))
+ if key not in registered_modules:
+ modules = sorted(reported_modules[key])
+ elif shipped_modules and not released:
+ modules = shipped_modules
+ else:
+ continue
+ missing_licenses.append((", ".join(modules),) + key +
(reported_license_druid_module[0],))
+ return missing_licenses
+
+def check_licenses(license_yaml, dependency_reports_root,
exclude_module_prefixes):
# Build a dictionary to facilitate comparing reported licenses and
registered ones.
# These dictionaries are the mapping of (group_id, artifact_id, version)
to license_name.
@@ -396,6 +417,7 @@ def check_licenses(license_yaml, dependency_reports_root):
# Build reported license dictionary.
reported_dep_to_licenses = {}
+ reported_modules = {}
compatible_license_names = build_compatible_license_names()
for dirpath, dirnames, filenames in os.walk(dependency_reports_root):
for filename in filenames:
@@ -406,7 +428,10 @@ def check_licenses(license_yaml, dependency_reports_root):
print_log_to_stderr("Parsing {}".format(full_path))
with open(full_path, encoding="utf-8") as report_file:
parser = DependencyReportParser(druid_module_name,
compatible_license_names)
- reported_dep_to_licenses.update(parser.parse(report_file))
+ dep_to_license = parser.parse(report_file)
+ reported_dep_to_licenses.update(dep_to_license)
+ for key in dep_to_license:
+ reported_modules.setdefault(key,
set()).add(druid_module_name)
if len(reported_dep_to_licenses) == 0:
raise Exception("No dependency reports are found")
@@ -415,6 +440,7 @@ def check_licenses(license_yaml, dependency_reports_root):
# Build registered license dictionary.
registered_dep_to_licenses = {}
+ registered_modules = {}
skipping_licenses = {}
# Validate that every entry uses a known module tag. Core extensions use
extensions/<output-dir>
# (matching how they appear in the binary distribution's extensions/
directory), and contrib
@@ -469,14 +495,15 @@ def check_licenses(license_yaml, dependency_reports_root):
version = get_version_string(license['version'])
skipping_licenses[get_dep_key(group_id, artifact_id,
version)] = license
else:
- registered_dep_to_licenses[get_dep_key(group_id,
artifact_id, get_version_string(license['version']))] =
compatible_license_names[license['license_name']]
+ key = get_dep_key(group_id, artifact_id,
get_version_string(license['version']))
+ registered_dep_to_licenses[key] =
compatible_license_names[license['license_name']]
+ registered_modules.setdefault(key,
set()).add(license['module'])
if len(registered_dep_to_licenses) == 0:
raise Exception("No registered licenses are found")
# Compare licenses in registry and those in dependency reports.
mismatched_licenses = []
- missing_licenses = []
unchecked_licenses = []
# Iterate through registered licenses and check if its license is same
with the reported one.
for key, registered_license in registered_dep_to_licenses.items():
@@ -498,9 +525,9 @@ def check_licenses(license_yaml, dependency_reports_root):
print_log_to_stderr("")
# Let's find missing licenses, which are reported but missing in the
registry.
- for key, reported_license_druid_module in reported_dep_to_licenses.items():
- if reported_license_druid_module[0] != "-" and key not in
registered_dep_to_licenses and key not in skipping_licenses:
- missing_licenses.append((reported_license_druid_module[1], key[0],
key[1], key[2], reported_license_druid_module[0]))
+ missing_licenses = find_missing_licenses(
+ reported_dep_to_licenses, reported_modules, registered_modules,
skipping_licenses, exclude_module_prefixes
+ )
if len(missing_licenses) > 0:
print_log_to_stderr("Error: found {} missing licenses. These licenses
are reported, but missing in the registry".format(len(missing_licenses)))
@@ -534,11 +561,18 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Check and generate
license file.')
parser.add_argument('license_yaml', metavar='<path to license.yaml>',
type=str)
parser.add_argument('dependency_reports_root', metavar='<root to maven
dependency reports>', type=str)
+ parser.add_argument(
+ '--exclude-module-prefix',
+ action='append',
+ default=[],
+ help='Module prefix whose licenses.yaml entries the release
LICENSE omits. Repeatable. A dependency '
+ 'that ships in the release must be registered under at least
one module not matching these.'
+ )
args = parser.parse_args()
license_yaml = args.license_yaml
dependency_reports_root = args.dependency_reports_root
- check_licenses(license_yaml, dependency_reports_root)
+ check_licenses(license_yaml, dependency_reports_root,
args.exclude_module_prefix)
except KeyboardInterrupt:
print('Interrupted, closing.')
diff --git a/distribution/bin/generate-license-dependency-reports.py
b/distribution/bin/generate-license-dependency-reports.py
index 52577ea8111..28cc14f6184 100755
--- a/distribution/bin/generate-license-dependency-reports.py
+++ b/distribution/bin/generate-license-dependency-reports.py
@@ -59,17 +59,20 @@ def generate_reports(druid_path, tmp_path, exclude_ext,
num_threads):
if not exclude_ext:
extensions_core_path = os.path.join(druid_path, "extensions-core")
- command = "mvn -Dexec.executable='echo' -Dexec.args='${basedir}'
exec:exec -q | grep extensions-core | grep -o '[^/]*$'"
+ command = "mvn -Dexec.executable='echo' -Dexec.args='${basedir}
${project.artifactId}' exec:exec -q | grep extensions-core | grep -o '[^/]*$'"
extension_dirs = subprocess.check_output(command, cwd=druid_path,
shell=True).decode().split('\n')[:-1]
print("Found {} extensions".format(len(extension_dirs)))
- for extension_dir in extension_dirs:
- print("extension dir: {}".format(extension_dir))
+ for extension_dir_and_artifact_id in extension_dirs:
+ extension_dir, artifact_id = extension_dir_and_artifact_id.split('
')
+ print("extension dir: {}, artifactId: {}".format(extension_dir,
artifact_id))
extension_path = os.path.join(extensions_core_path, extension_dir)
if not os.path.isdir(extension_path):
print("{} is not a directory".format(extension_path))
continue
- extension_report_dir = "{}/{}".format(license_ext_path,
extension_dir)
+ # Name the report after the artifactId, which is the extension's
directory in the binary distribution
+ # and the module tag check-licenses.py matches against
licenses.yaml (extensions/<artifactId>).
+ extension_report_dir = "{}/{}".format(license_ext_path,
artifact_id)
script_args.append((extension_path, os.path.join(extension_path,
"target", "reports"), extension_report_dir))
print("Generating dependency reports")
diff --git a/distribution/bin/test_check_licenses.py
b/distribution/bin/test_check_licenses.py
new file mode 100755
index 00000000000..7583f41e1d2
--- /dev/null
+++ b/distribution/bin/test_check_licenses.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.util
+import os
+import unittest
+
+spec = importlib.util.spec_from_file_location(
+ "check_licenses", os.path.join(os.path.dirname(os.path.abspath(__file__)),
"check-licenses.py")
+)
+check_licenses = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(check_licenses)
+
+APACHE = "Apache License version 2.0"
+AZURE = "extensions/druid-azure-extensions"
+TESTCONTAINERS = "extensions/druid-testcontainers"
+RANGER = "extensions-contrib/druid-ranger-security"
+EXCLUDED = ["extensions-contrib/", TESTCONTAINERS]
+GAV = check_licenses.get_dep_key("org.example", "a", "1.0")
+
+
+def find_missing(reporting_modules, registered_modules, skipping=None):
+ reported = {GAV: (APACHE, reporting_modules[0])}
+ registered = {GAV: set(registered_modules)} if registered_modules else {}
+ return check_licenses.find_missing_licenses(reported, {GAV:
set(reporting_modules)}, registered, skipping or {}, EXCLUDED)
+
+
+class FindMissingLicensesTest(unittest.TestCase):
+ def test_registered_under_released_module(self):
+ self.assertEqual([], find_missing([AZURE], ["java-core"]))
+
+ def test_unregistered_is_missing(self):
+ self.assertEqual([(AZURE,) + GAV + (APACHE,)], find_missing([AZURE],
[]))
+
+ def
test_registered_only_under_excluded_module_is_missing_when_shipped(self):
+ self.assertEqual([(AZURE,) + GAV + (APACHE,)],
find_missing([TESTCONTAINERS, AZURE], [RANGER]))
+
+ def
test_registered_only_under_excluded_module_is_fine_when_not_shipped(self):
+ self.assertEqual([], find_missing([TESTCONTAINERS], [TESTCONTAINERS]))
+
+ def test_skipped_dependency_is_not_missing(self):
+ self.assertEqual([], find_missing([AZURE], [], skipping={GAV: {}}))
+
+
+class FindDruidModuleNameTest(unittest.TestCase):
+ def test_maps_report_dirs_to_module_tags(self):
+ self.assertEqual("java-core",
check_licenses.find_druid_module_name("target/license-reports/core/reports"))
+ self.assertEqual(AZURE,
check_licenses.find_druid_module_name("target/license-reports/ext/druid-azure-extensions/reports"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 77b8f358ab9..7202d8910f8 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -318,6 +318,16 @@
</arguments>
</configuration>
</execution>
+ <execution>
+ <id>test-check-licenses</id>
+ <phase>test</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+
<executable>${project.basedir}/bin/test_check_licenses.py</executable>
+ </configuration>
+ </execution>
<execution>
<id>check-licenses</id>
<phase>test</phase>
@@ -329,6 +339,10 @@
<arguments>
<argument>${project.parent.basedir}/licenses.yaml</argument>
<argument>${project.basedir}/target/license-reports</argument>
+ <!-- Keep these under
bundle-contrib-exts too: the check guards the Apache release
+ LICENSE, which leaves out contrib
entries. -->
+
<argument>--exclude-module-prefix=extensions-contrib/</argument>
+
<argument>--exclude-module-prefix=extensions/druid-testcontainers</argument>
</arguments>
</configuration>
</execution>
diff --git a/licenses.yaml b/licenses.yaml
index 8278a73faf8..d169fd88b60 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -5247,6 +5247,16 @@ libraries:
---
+name: net.java.dev.jna jna-platform
+license_category: binary
+version: 5.19.1
+module: extensions/druid-azure-extensions
+license_name: Apache License version 2.0
+libraries:
+ - net.java.dev.jna: jna-platform
+
+---
+
name: JOpt Simple
license_category: binary
version: 5.0.2
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
