leaves12138 commented on code in PR #9787: URL: https://github.com/apache/paimon/pull/9787#discussion_r3999278863
########## tools/ci/plan.py: ########## @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Select CI suites from the complete git diff, without API pagination limits.""" + +import json +import os +from pathlib import Path +import re +import subprocess + + +# Keep the existing JDK, engine and Scala coverage. A suite is a test boundary, +# not just a source directory: core also owns Hive and generated-doc tests. +SUITES = [ + ("core", "Core and integrations / JDK 8", 8, "2.12", 90), + ("core", "Core and integrations / JDK 11", 11, "2.12", 90), + ("flink1-common", "Flink 1 / Common", 8, "2.12", 100), + ("flink1-connectors", "Flink 1 / Connectors and CDC", 8, "2.12", 60), + ("flink2", "Flink 2 / Common and connectors", 11, "2.12", 120), + ("spark3", "Spark 3 / Scala 2.12", 8, "2.12", 90), + ("spark3", "Spark 3 / Scala 2.13", 8, "2.13", 90), + ("spark4", "Spark 4 / Scala 2.13", 17, "2.13", 90), + ("e2e-flink1", "E2E / Flink 1 and Spark interoperability", 8, "2.12", 60), + ("e2e-flink2", "E2E / Flink 2", 11, "2.12", 60), + ("iceberg", "Iceberg GA / JDK 17", 17, "2.12", 60), + ("eslib", "ESLib / JDK 11", 11, "2.12", 30), + ("full-text", "Full text / JDK 8", 8, "2.12", 30), +] +ALL_JAVA = {suite[0] for suite in SUITES} +FLINK = {"flink1-common", "flink1-connectors", "flink2"} +SPARK = {"spark3", "spark4"} +E2E = {"e2e-flink1", "e2e-flink2"} + + +def select(paths, full=False): + selected = set() + docs = python = licensing = False + if full: + selected.update(ALL_JAVA) + docs = python = licensing = True + + for path in paths: + if path.startswith("docs/"): + docs = True + continue + if path.endswith(".md"): + continue + if path.startswith("paimon-python/") or path == ".github/workflows/ci-python.yml": + python = True + continue + if re.search(r"/src/test/.*?/JavaPy[^/]*\.java$", path): + python = True + continue + if path == ".github/workflows/ci-docs.yml": + docs = True + continue + if path == ".github/workflows/ci-licensing.yml": + licensing = True + continue + # Changes to CI machinery must validate every consumer of that machinery. + if path.startswith((".github/workflows/ci", "tools/ci/")): + selected.update(ALL_JAVA) + docs = python = licensing = True + continue + if path.startswith((".github/", ".asf.yaml")): + continue + + licensing = True + if path.startswith("paimon-spark/"): + # Core includes spark-common tests and paimon-docs completeness tests. + # SparkE2eTest is disabled on JDK 11, so only the Flink 1 E2E lane applies. + selected.update(SPARK | {"core", "e2e-flink1"}) + elif path.startswith("paimon-flink/"): + # Hive and Iceberg integration tests depend on flink-common. + selected.update(FLINK | E2E | {"core", "iceberg"}) + elif path.startswith("paimon-e2e-tests/"): + selected.update(E2E) + elif path.startswith("paimon-eslib/"): + selected.add("eslib") + elif path.startswith("paimon-iceberg/"): + selected.update({"core", "iceberg"}) + elif path.startswith("paimon-full-text/"): + selected.update({"core", "full-text"}) + else: + # Shared modules, POMs outside a known leaf, build configuration and + # new/unknown directories fall back to all Java suites. + selected.update(ALL_JAVA) + if path == "pom.xml": + python = True + + matrix = [ + dict(suite=suite, name=name, java=java, scala=scala, timeout=timeout) + for suite, name, java, scala, timeout in SUITES + if suite in selected + ] + return dict(java=bool(matrix), matrix={"include": matrix}, docs=docs, + python=python, licensing=licensing) + + +def changed_paths(event_name, event): + if event_name == "pull_request": + base = event["pull_request"]["base"]["sha"] + head = event["pull_request"]["head"]["sha"] + separator = "..." + elif event_name == "push": + base, head = event["before"], event["after"] + separator = ".." + else: + return None + for sha in (base, head): + if not re.fullmatch(r"[0-9a-f]{40}", sha) or sha == "0" * 40: + return None + try: + # Disabling rename detection includes both the old and new path. -z also + # handles whitespace, newlines and non-ASCII names without shell parsing. + output = subprocess.check_output([ + "git", "diff", "--no-renames", "--name-only", "-z", + base + separator + head, "--", + ]) + except subprocess.CalledProcessError: + print("Cannot resolve the complete diff; selecting full CI.") + return None + return [os.fsdecode(path) for path in output.split(b"\0") if path] + + +def check_file_sizes(paths, root=Path(".")): + for path in paths: + candidate = root / path + # Git symlinks contain a link target; don't follow them outside checkout. + if candidate.is_symlink() or not candidate.is_file(): + continue + if candidate.stat().st_size > 1048576: + raise ValueError("Changed file exceeds 1 MiB: " + repr(path)) + + +def main(): + event_name = os.environ["GITHUB_EVENT_NAME"] + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) + paths = changed_paths(event_name, event) + plan = select(paths or [], full=paths is None) + if event_name == "pull_request": + # Even if diff resolution fails, keep the file-size guard effective. + size_paths = paths + if size_paths is None: + size_paths = [os.fsdecode(p) for p in subprocess.check_output( + ["git", "ls-files", "-z"]).split(b"\0") if p] + check_file_sizes(size_paths) Review Comment: [P2] Do not apply the changed-file size limit to the entire fallback checkout If `git diff` cannot resolve a PR diff, this scans every tracked file rather than only changed files. The existing repository already contains oversized files, e.g. `docs/static/img/lsm-inside-bucket.png` is 2,214,166 bytes and is unchanged from the base. I reproduced `main()` with the diff command failing and the tracked-file fallback returning that file: it raises `ValueError: Changed file exceeds 1 MiB` before writing any selection outputs. Consequently the advertised full-CI fallback fails the planner and never launches the test suites. Please obtain a changed-file set through an independent fallback, or otherwise account for pre-existing oversized files instead of rejecting them as new changes. Add a PR-fallback regression test containing an unchanged oversized file. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
