chia7712 commented on code in PR #22266: URL: https://github.com/apache/kafka/pull/22266#discussion_r4433760474
########## committer-tools/verify_notice.py: ########## @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import os +import re +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from collections import defaultdict + +# Kafka-own jars carry the project's NOTICE already covered by the hand-written +# header of NOTICE-binary, so we exclude them from the third-party scan. +KAFKA_JAR_PATTERN = re.compile(r"(kafka|connect|trogdor)", re.IGNORECASE) + +# Finds NOTICE-like filenames anywhere in a jar, ignoring case. +# Optional prefixes are separated from NOTICE by '-', '_', or '.', e.g. +# FastDoubleParser-NOTICE. The full suffix is validated during extraction; +# unsupported candidates cause an error. +NOTICE_FILE_PATTERN = re.compile( + r"(?:^|/)(?:[^/]*[-_.])?NOTICE(?P<suffix>[-_.][^/]*)?\Z", + re.IGNORECASE, +) + +# Marker within the final comment block of Kafka's own header in NOTICE-binary. +# The header ends at the first blank line after this marker and is excluded +# from validation against jar NOTICEs. Update this if the marker text changes. +KAFKA_HEADER_END_MARKER = ( + "// NOTICE file corresponding to the section 4d of The Apache License," +) + +# Sentinel placed where a region has already been consumed. +_CONSUMED_SENTINEL = "\x00" + + +def run_gradlew(project_dir): + print("Running './gradlew clean releaseTarGz'") + subprocess.run(["./gradlew", "clean", "releaseTarGz"], check=True, cwd=project_dir) + + +def get_tarball_path(project_dir): + distributions_dir = os.path.join(project_dir, "core", "build", "distributions") + if not os.path.isdir(distributions_dir): + print("Error: Distributions directory not found:", distributions_dir) + sys.exit(1) + + pattern = re.compile(r"^kafka_2\.13-(?!.*docs).+\.tgz$", re.IGNORECASE) + candidates = [ + os.path.join(distributions_dir, filename) + for filename in os.listdir(distributions_dir) + if pattern.match(filename) + ] + if not candidates: + print("Error: No tarball matching 'kafka_2.13-*.tgz' found in:", distributions_dir) + sys.exit(1) + return max(candidates, key=os.path.getmtime) + + +def extract_tarball(tarball, extract_dir): + with tarfile.open(tarball, "r:gz") as tar: + try: + tar.extractall(path=extract_dir, filter=lambda tarinfo, dest: tarinfo) + except TypeError: + tar.extractall(path=extract_dir) + + +def get_third_party_jars(libs_dir): + return [ + os.path.join(libs_dir, jar_name) + for jar_name in sorted(os.listdir(libs_dir)) + if jar_name.endswith(".jar") and not KAFKA_JAR_PATTERN.search(jar_name) + ] + + +def extract_notices_from_jar(jar_path): + found = [] + with zipfile.ZipFile(jar_path, "r") as zf: + for info in zf.infolist(): + if info.is_dir(): + continue + + match = NOTICE_FILE_PATTERN.search(info.filename) + if match is None: + continue + + suffix = (match.group("suffix") or "").lower() + # Add new supported NOTICE suffixes here in lowercase. + if suffix not in ("", ".txt", ".md", ".markdown"): + print( + f"Error: unsupported NOTICE-like filename " + f"{info.filename!r} in {jar_path!r}." + ) + sys.exit(1) + + try: + text = zf.read(info).decode("utf-8").strip() + except UnicodeDecodeError as error: + print( + f"Error: cannot decode NOTICE file {info.filename!r} " + f"in {jar_path!r} as UTF-8: {error}" + ) + sys.exit(1) + if text: + found.append((info.filename, text)) + return found + + +def collect_notices(jars): + notice_to_sources = defaultdict(list) + canonical_text = {} + + for jar in jars: + jar_name = os.path.basename(jar) + for entry, text in extract_notices_from_jar(jar): + key = " ".join(text.split()) + notice_to_sources[key].append((jar_name, entry)) + canonical_text.setdefault(key, text) + + return notice_to_sources, canonical_text + + +def split_kafka_header(notice_binary_text): + marker_start = notice_binary_text.find(KAFKA_HEADER_END_MARKER) + if marker_start < 0: + print(f"Error: header marker not found in NOTICE-binary: {KAFKA_HEADER_END_MARKER!r}") + sys.exit(1) + end_of_block = notice_binary_text.find("\n\n", marker_start) + if end_of_block < 0: + print("Error: no blank line found after the header marker in NOTICE-binary.") + sys.exit(1) + return notice_binary_text[:end_of_block], notice_binary_text[end_of_block:] + + +def find_match_span(body, normalized_notice, matched_mask): + normalized_chars = [] + original_offsets = [] + previous_char = _CONSUMED_SENTINEL + for offset, char in enumerate(body): + if matched_mask[offset]: + if previous_char != _CONSUMED_SENTINEL: + normalized_chars.append(_CONSUMED_SENTINEL) + original_offsets.append(offset) + previous_char = _CONSUMED_SENTINEL + continue + if char.isspace(): + if previous_char != " " and previous_char != _CONSUMED_SENTINEL: + normalized_chars.append(" ") + original_offsets.append(offset) + previous_char = " " + else: + normalized_chars.append(char) + original_offsets.append(offset) + previous_char = char + + match_start = "".join(normalized_chars).find(normalized_notice) + if match_start < 0: + return None + return original_offsets[match_start], original_offsets[match_start + len(normalized_notice) - 1] + 1 + + +def trim_blank_edges(lines): + first_content_line, content_end_line = 0, len(lines) + while first_content_line < content_end_line and not lines[first_content_line].strip(): + first_content_line += 1 + while content_end_line > first_content_line and not lines[content_end_line - 1].strip(): + content_end_line -= 1 + return first_content_line, content_end_line + + +def find_leftover_regions(body, matched_mask, notice_binary_text, body_start): + regions = [] + region_start, body_length = 0, len(body) + while region_start < body_length: + if matched_mask[region_start]: + region_start += 1 + continue + region_end = region_start + while region_end < body_length and not matched_mask[region_end]: + region_end += 1 + region_text = body[region_start:region_end] + if region_text.strip(): + lines = region_text.split("\n") + first_content_line, content_end_line = trim_blank_edges(lines) + absolute_start = body_start + region_start + line_start = notice_binary_text.count("\n", 0, absolute_start) + 1 + first_content_line + line_end = line_start + (content_end_line - first_content_line - 1) + regions.append((line_start, line_end)) + region_start = region_end + return regions + + +def print_sources(sources): + for idx, (jar, entry) in enumerate(sources): + if idx > 0: + print() + print(f" - {jar}") + print(f" NOTICE file inside JAR: {entry}") + + +def print_section_header(title): + print("\n" + "=" * 78) + print(title) + print("=" * 78) + + +def print_update_summary(unmatched, leftover): Review Comment: I ran the script and it is a bit hard for me to follow the output to fix NOTICE-binary. Could this script generate the whole NOTICE-binary instead (for example, a --fix flag that writes the expected file, with the default mode printing the diff)? Or it could dump all NOTICE files to a folder and then we can leverage powerful AI to regenerate NOTICE-binary. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
