This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch wiki-proposals-consistency-check in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 5ef5e040cf8fd2d600cccd389573889eb32e6a14 Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Nov 10 08:37:11 2026 -0700 Check the AIP and ADIP wiki trees daily and report problems to Slack The AIP and ADIP index pages are now generated: each list is a Page Properties Report over one folder, so an AIP sits in the folder for its state and carries a matching State badge and state label. A page in the wrong folder, a State or label that does not match it, or a stale count in a folded section silently puts an AIP in the wrong list or drops it. scripts/ci/verify_improvement_proposals_wiki.py reads the wiki anonymously and reports each such page; a scheduled workflow runs it daily and posts the list to #internal-airflow-ci-cd when anything is out of line, or a short notice when the check itself could not run. Generated-by: Claude Opus 5 --- ...scheduled-verify-improvement-proposals-wiki.yml | 64 ++++ dev/breeze/doc/ci/04_selective_checks.md | 2 +- .../src/airflow_breeze/utils/selective_checks.py | 3 +- dev/breeze/tests/test_selective_checks.py | 5 + scripts/ci/verify_improvement_proposals_wiki.py | 379 ++++++++++++++++++++ .../ci/test_verify_improvement_proposals_wiki.py | 396 +++++++++++++++++++++ 6 files changed, 847 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scheduled-verify-improvement-proposals-wiki.yml b/.github/workflows/scheduled-verify-improvement-proposals-wiki.yml new file mode 100644 index 00000000000..ed7a79b6c6a --- /dev/null +++ b/.github/workflows/scheduled-verify-improvement-proposals-wiki.yml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: "Scheduled verify improvement proposals wiki" +on: # yamllint disable-line rule:truthy + schedule: + # Daily at 06:30 UTC + - cron: '30 6 * * *' + workflow_dispatch: +permissions: + contents: read +jobs: + verify-improvement-proposals-wiki: + name: "Verify AIP and ADIP wiki trees" + if: github.event_name != 'schedule' || github.repository == 'apache/airflow' + runs-on: ["ubuntu-slim"] + timeout-minutes: 15 + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: "Verify AIP and ADIP wiki trees" + id: verify + env: + RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + run: > + python3 scripts/ci/verify_improvement_proposals_wiki.py + --slack-payload slack-message.json + --run-url "${RUN_URL}" + - name: "Notify Slack about the problems" + if: failure() && hashFiles('slack-message.json') != '' + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload-file-path: "slack-message.json" + # yamllint disable rule:line-length + - name: "Notify Slack that the check could not run" + if: failure() && hashFiles('slack-message.json') == '' + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + channel: "internal-airflow-ci-cd" + text: >- + :warning: The AIP/ADIP wiki consistency check could not run. + See: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 4c14a33e999..b2a1123994f 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -421,7 +421,7 @@ We have the following Groups of files for CI that determine which tests are run: drive prek static checks rather than the test matrix, so they only trigger the CI image build (via the `Prek files` group) — `mypy-scripts` and the image-based static checks still run, but the full test matrix does not; and **non-test workflows** (`.github/workflows/` for security scans, doc publishing, - notifications, backporting, calendar/stale bots, …) cannot change test outcomes. Workflows that run or + notifications, backporting, calendar/wiki/stale bots, …) cannot change test outcomes. Workflows that run or configure the test suite (`ci-amd`, `ci-arm`, `run-unit-tests`, `k8s-tests`, `helm-tests`, image builds, …) still force the full matrix. * `Prek files` - `scripts/ci/prek/...` static-check hooks. They do not force full tests, but they trigger diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 1f57fc93871..7c6b8cba59c 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -246,7 +246,8 @@ CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] = HashableDict( r"^\.github/workflows/(?!(" r"asf-allowlist-check|automatic-backport|backport-cli|ci-duration-monitor|ci-notification|" r"codeql-analysis|e2e-flaky-tests-report|milestone-tag-assistant|notify-uv-lock-conflicts|" - r"publish-docs-to-s3|recheck-old-bug-report|scheduled-verify-release-calendar|stale" + r"publish-docs-to-s3|recheck-old-bug-report|scheduled-verify-improvement-proposals-wiki|" + r"scheduled-verify-release-calendar|stale" r")\.yml$)", r"^dev/breeze/src", r"^dev/breeze/pyproject\.toml", diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index d5bf33d73d9..b871f515ce5 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2747,6 +2747,11 @@ def test_ui_compile_hooks_not_skipped_when_ui_openapi_spec_changes(files: tuple[ {"full-tests-needed": "false"}, id="Non-test workflow (ci-notification) does not force full tests", ), + pytest.param( + (".github/workflows/scheduled-verify-improvement-proposals-wiki.yml",), + {"full-tests-needed": "false"}, + id="Non-test workflow (improvement proposals wiki check) does not force full tests", + ), ], ) def test_non_test_workflows_do_not_force_full_tests(files: tuple[str, ...], expected_outputs: dict[str, str]): diff --git a/scripts/ci/verify_improvement_proposals_wiki.py b/scripts/ci/verify_improvement_proposals_wiki.py new file mode 100644 index 00000000000..d273ea6dee3 --- /dev/null +++ b/scripts/ci/verify_improvement_proposals_wiki.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11" +# /// +""" +Check that the AIP and ADIP pages on the Airflow wiki are laid out the way their index pages expect. + +The index pages are generated: each list is a Page Properties Report over the pages of one +folder, so a page in the wrong place, or with a State that does not match where it sits, shows +up in the wrong list or not at all. This script reads the wiki anonymously and reports every +such page. + +AIP tree (https://cwiki.apache.org/confluence/display/AIRFLOW/Airflow+Improvement+Proposals): + + * the index page has only the state folders (and the folder of ADIP stubs) as children; + * every page in a state folder has one Page Properties block whose State badge matches the + folder, the ``airflow-improvement-proposal`` label, and the state label of that folder only; + * a page carrying a state label sits in that state's folder; + * the counts in the titles of the folded sections on the index page match the folders. + +ADIP tree: every child of the ADIP page is titled ``ADIP-<n> <title>`` with a unique number, +has the ``adip`` label and one Page Properties block with a State badge and a Formerly row. + +Exits 1 when anything is out of line. With ``--slack-payload <file>`` it then also writes a +``chat.postMessage`` payload listing the problems, for the workflow to post. +""" + +from __future__ import annotations + +import argparse +import gzip +import http.client +import json +import re +import sys +import time +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path + +WIKI = "https://cwiki.apache.org/confluence"; +AIP_INDEX_ID = "89066602" +ADIP_INDEX_ID = "462816431" +ADIP_STUBS_FOLDER_ID = "462816661" +BASE_LABEL = "airflow-improvement-proposal" +ADIP_LABEL = "adip" +SLACK_CHANNEL = "internal-airflow-ci-cd" +MAX_PROBLEMS_IN_SLACK = 25 +MAX_TITLE_LENGTH = 100 +# Slack rejects a section block whose text is longer than 3000 characters. +MAX_SECTION_LENGTH = 2900 +ATTEMPTS = 4 + + +@dataclass(frozen=True) +class StateFolder: + page_id: str + label: str + badge: str + # Title of the folded section on the AIP index, with "{count}" where the count goes; + # None when the section is not folded. + folded_title: str | None = None + + +STATE_FOLDERS = ( + StateFolder("462816682", f"{BASE_LABEL}/wip", "WIP"), + StateFolder("462816536", f"{BASE_LABEL}/draft", "Under discussion"), + StateFolder("462816537", f"{BASE_LABEL}/accepted", "Accepted"), + StateFolder( + "173087665", f"{BASE_LABEL}/completed", "Completed", "{count} completed AIPs (click to expand)" + ), + StateFolder( + "173087786", + f"{BASE_LABEL}/abandoned", + "Abandoned", + "{count} abandoned or replaced AIPs (click to expand)", + ), +) +STATE_LABELS = {folder.label for folder in STATE_FOLDERS} +ADIP_BADGES = {"Draft", "Accepted", "In progress", "Completed", "Abandoned"} + +# Pages in a state folder that the people who keep the wiki in order cannot edit, with the +# reason. Their State badge and labels are not checked; their place in the tree still is. +RESTRICTED_PAGES = { + "247827224": "AIP-52 has edit restrictions", +} + + +@dataclass +class Page: + page_id: str + title: str + body: str = "" + labels: list[str] = field(default_factory=list) + + @property + def url(self) -> str: + return f"{WIKI}/pages/viewpage.action?pageId={self.page_id}" + + +def get_json(path: str, params: dict[str, str], sleep: Callable[[float], None] = time.sleep) -> dict: + url = f"{WIKI}/rest/api/{path}?{urllib.parse.urlencode(params)}" + # Uncompressed responses of more than a few tens of kilobytes get cut off mid-transfer. + request = urllib.request.Request( + url, + headers={ + "Accept": "application/json", + "Accept-Encoding": "gzip", + "User-Agent": "apache-airflow-wiki-consistency-check", + }, + ) + for attempt in range(1, ATTEMPTS): + try: + return read_json(request) + except (http.client.HTTPException, OSError): + sleep(5 * attempt) + return read_json(request) + + +def read_json(request: urllib.request.Request) -> dict: + with urllib.request.urlopen(request, timeout=60) as response: + raw = response.read() + if response.headers.get("Content-Encoding") == "gzip": + raw = gzip.decompress(raw) + return json.loads(raw) + + +def build_page(raw: dict) -> Page: + return Page( + page_id=raw["id"], + title=raw["title"], + body=raw.get("body", {}).get("storage", {}).get("value", ""), + labels=[label["name"] for label in raw.get("metadata", {}).get("labels", {}).get("results", [])], + ) + + +def fetch_all(path: str, params: dict[str, str], fetch: Callable[[str, dict[str, str]], dict]) -> list[Page]: + pages: list[Page] = [] + start = 0 + while True: + data = fetch(path, {**params, "limit": "50", "start": str(start)}) + pages.extend(build_page(raw) for raw in data["results"]) + if not data["results"] or "next" not in data.get("_links", {}): + return pages + start += len(data["results"]) + + +def fetch_page(page_id: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> Page: + return build_page(fetch(f"content/{page_id}", {"expand": "body.storage,metadata.labels"})) + + +def fetch_children(page_id: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> list[Page]: + # Bodies are fetched one page at a time: a response carrying several AIP bodies is large + # enough to be cut off mid-transfer. + listed = fetch_all(f"content/{page_id}/child/page", {}, fetch) + return [fetch_page(page.page_id, fetch) for page in listed] + + +def fetch_labelled(label: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> list[Page]: + return fetch_all("content/search", {"cql": f'label = "{label}" and type = page'}, fetch) + + +def get_text(html: str) -> str: + return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html).replace(" ", " ")).strip() + + +def find_macro_body_end(body: str, start: int) -> int: + """Return where the rich-text body opened after ``start`` closes, skipping nested macro bodies.""" + depth = 0 + for tag in re.finditer(r"<(/?)ac:rich-text-body>", body[start:]): + depth += -1 if tag.group(1) else 1 + if depth == 0: + return start + tag.start() + return len(body) + + +def find_properties(body: str) -> list[dict[str, str]]: + """Return the rows of each Page Properties block as {key: value storage HTML}.""" + blocks = [] + for match in re.finditer(r'<ac:structured-macro ac:name="details"', body): + # Only a table inside the macro's own rich-text body is read by the index report; a + # self-closing macro, or one with no body, has no properties. + opening = re.match( + r"[^>]*?(/?)>(?:\s*<ac:parameter[^>]*>.*?</ac:parameter>)*\s*<ac:rich-text-body>", + body[match.end() :], + re.S, + ) + if not opening or opening.group(1): + macro_body = "" + else: + body_start = match.end() + opening.end() - len("<ac:rich-text-body>") + macro_body = body[body_start : find_macro_body_end(body, body_start)] + table_end = macro_body.find("</table>") + rows: dict[str, str] = {} + for row in re.findall(r"<tr[^>]*>(.*?)</tr>", macro_body[: max(table_end, 0)], re.S): + cells = re.findall(r"<t([dh])[^>]*>(.*?)</t\1>", row, re.S) + if len(cells) >= 2: + rows.setdefault(get_text(cells[0][1]).rstrip(":"), cells[1][1]) + blocks.append(rows) + return blocks + + +def find_badge(cell: str) -> str | None: + match = re.search( + r'<ac:structured-macro ac:name="status".*?<ac:parameter ac:name="title">([^<]*)</ac:parameter>', + cell, + re.S, + ) + return match.group(1) if match else None + + +def check_properties(page: Page, allowed_badges: set[str]) -> tuple[list[str], dict[str, str]]: + blocks = find_properties(page.body) + if len(blocks) != 1: + return [f"has {len(blocks)} Page Properties blocks, expected 1"], blocks[0] if blocks else {} + properties = blocks[0] + if "State" not in properties: + return ["has no State row in its Page Properties block"], properties + badge = find_badge(properties["State"]) + if badge not in allowed_badges: + return [f"State badge is {badge!r}, expected one of {sorted(allowed_badges)}"], properties + return [], properties + + +def escape_for_slack(text: str) -> str: + # Wiki titles are free text: unescaped, "<!channel>" would notify the channel and a "<" or + # "|" would change the link. + return text.replace("&", "&").replace("<", "<").replace(">", ">").replace("|", "|") + + +def shorten(title: str) -> str: + return title if len(title) <= MAX_TITLE_LENGTH else title[: MAX_TITLE_LENGTH - 1] + "…" + + +def describe(page: Page, problem: str) -> str: + return f"<{page.url}|{escape_for_slack(shorten(page.title))}>: {escape_for_slack(problem)}" + + +def check_aip_tree( + index: Page, + index_children: list[Page], + folders: dict[str, list[Page]], + labelled: dict[str, list[Page]], +) -> list[str]: + problems = [] + allowed_children = {folder.page_id for folder in STATE_FOLDERS} | {ADIP_STUBS_FOLDER_ID} + for page in index_children: + if page.page_id not in allowed_children: + problems.append(describe(page, "sits directly under the AIP index; move it into a state folder")) + for missing in sorted(allowed_children - {page.page_id for page in index_children}): + problems.append( + describe(Page(missing, f"page {missing}"), "folder is no longer directly under the AIP index") + ) + for folder in STATE_FOLDERS: + pages = folders[folder.page_id] + for page in pages: + if page.page_id in RESTRICTED_PAGES: + continue + page_problems, _ = check_properties(page, {folder.badge}) + state_labels = STATE_LABELS.intersection(page.labels) + if state_labels != {folder.label}: + page_problems.append(f"state labels are {sorted(state_labels)}, expected ['{folder.label}']") + if BASE_LABEL not in page.labels: + page_problems.append(f"lacks the {BASE_LABEL!r} label") + problems.extend(describe(page, problem) for problem in page_problems) + in_folder = {page.page_id for page in pages} + for page in labelled[folder.label]: + if page.page_id not in in_folder: + problems.append( + describe(page, f"has the {folder.label!r} label but is not in that state's folder") + ) + if folder.folded_title: + expected = folder.folded_title.format(count=len(pages)) + if f'<ac:parameter ac:name="title">{expected}</ac:parameter>' not in index.body: + problems.append(describe(index, f"the folded section should be titled {expected!r}")) + return problems + + +def check_adip_tree(children: list[Page]) -> list[str]: + problems = [] + numbers: dict[int, Page] = {} + for page in children: + match = re.match(r"ADIP-(\d+) \S", page.title) + if not match: + problems.append(describe(page, "title does not start with 'ADIP-' and its number")) + elif (number := int(match.group(1))) in numbers: + problems.append(describe(page, f"reuses the number of {shorten(numbers[number].title)!r}")) + else: + numbers[number] = page + if "(formerly" in page.title: + problems.append( + describe(page, "title repeats the former AIP number; it is the Formerly property") + ) + if ADIP_LABEL not in page.labels: + problems.append(describe(page, f"lacks the {ADIP_LABEL!r} label")) + page_problems, properties = check_properties(page, ADIP_BADGES) + if not page_problems and not re.fullmatch(r"AIP-\d+", get_text(properties.get("Formerly", ""))): + page_problems.append( + "has no Formerly row with the former AIP number in its Page Properties block" + ) + problems.extend(describe(page, problem) for problem in page_problems) + return problems + + +def collect_problems(fetch: Callable[[str, dict[str, str]], dict] = get_json) -> list[str]: + index = fetch_page(AIP_INDEX_ID, fetch) + folders = {folder.page_id: fetch_children(folder.page_id, fetch) for folder in STATE_FOLDERS} + labelled = {folder.label: fetch_labelled(folder.label, fetch) for folder in STATE_FOLDERS} + return check_aip_tree(index, fetch_children(AIP_INDEX_ID, fetch), folders, labelled) + check_adip_tree( + fetch_children(ADIP_INDEX_ID, fetch) + ) + + +def build_slack_payload(problems: list[str], run_url: str | None) -> dict: + shown = problems[:MAX_PROBLEMS_IN_SLACK] + lines = [ + f":warning: *AIP/ADIP wiki consistency check found {len(problems)} problem(s)*", + "Move each AIP page to the folder for its state and keep its State badge and labels in line " + "with that folder.", + *(f"• {problem}" for problem in shown), + ] + if len(problems) > len(shown): + lines.append(f"… and {len(problems) - len(shown)} more") + if run_url: + lines.append(f"<{run_url}|View the run>") + sections: list[str] = [] + for line in lines: + if sections and len(sections[-1]) + 1 + len(line) <= MAX_SECTION_LENGTH: + sections[-1] += "\n" + line + else: + sections.append(line) + return { + "channel": SLACK_CHANNEL, + "text": f":warning: AIP/ADIP wiki consistency check found {len(problems)} problem(s)", + "blocks": [{"type": "section", "text": {"type": "mrkdwn", "text": text}} for text in sections], + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--slack-payload", type=Path, help="write a Slack payload here when problems are found" + ) + parser.add_argument("--run-url", help="link to the CI run, added to the Slack message") + args = parser.parse_args(argv) + problems = collect_problems() + if not problems: + print("The AIP and ADIP wiki trees are consistent.") + return 0 + print(f"Found {len(problems)} problem(s):") + for problem in problems: + print(f" - {problem}") + if args.slack_payload: + args.slack_payload.write_text(json.dumps(build_slack_payload(problems, args.run_url), indent=2)) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/ci/test_verify_improvement_proposals_wiki.py b/scripts/tests/ci/test_verify_improvement_proposals_wiki.py new file mode 100644 index 00000000000..26bf4d8af92 --- /dev/null +++ b/scripts/tests/ci/test_verify_improvement_proposals_wiki.py @@ -0,0 +1,396 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import gzip +import http.client +import importlib.util +import json +import sys +from pathlib import Path +from unittest import mock + +import pytest + +MODULE_PATH = Path(__file__).resolve().parents[3] / "scripts" / "ci" / "verify_improvement_proposals_wiki.py" + + [email protected] +def wiki(): + module_name = "test_verify_improvement_proposals_wiki_module" + sys.modules.pop(module_name, None) + spec = importlib.util.spec_from_file_location(module_name, MODULE_PATH) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +def badge(title, colour="Grey"): + return ( + '<ac:structured-macro ac:name="status" ac:schema-version="1" ac:macro-id="x">' + f'<ac:parameter ac:name="colour">{colour}</ac:parameter>' + f'<ac:parameter ac:name="title">{title}</ac:parameter></ac:structured-macro>' + ) + + +def properties(rows): + cells = "".join( + f"<tr><td><strong>{key}</strong></td><td>{value}</td></tr>" for key, value in rows.items() + ) + return ( + '<ac:structured-macro ac:name="details" ac:schema-version="1" ac:macro-id="y">' + f"<ac:rich-text-body><table><tbody>{cells}</tbody></table></ac:rich-text-body></ac:structured-macro>" + ) + + +def folded(count, kind): + return ( + '<ac:structured-macro ac:name="expand" ac:schema-version="1"><ac:parameter ac:name="title">' + f"{count} {kind} AIPs (click to expand)</ac:parameter></ac:structured-macro>" + ) + + +def aip(wiki, folder, page_id="1", title="AIP-1 Something", state=None, labels=None): + return wiki.Page( + page_id, + title, + properties({"State": badge(state or folder.badge) + " extra detail"}), + [wiki.BASE_LABEL, folder.label] if labels is None else labels, + ) + + +def build_tree(wiki, pages=None, index_children=None, labelled=None, index_body=None): + """A consistent AIP tree with one AIP in each state folder, with overrides.""" + folders = {f.page_id: [aip(wiki, f, page_id=f"p{f.page_id}")] for f in wiki.STATE_FOLDERS} + folders.update(pages or {}) + children = [wiki.Page(f.page_id, f.badge) for f in wiki.STATE_FOLDERS] + [ + wiki.Page(wiki.ADIP_STUBS_FOLDER_ID, "AIPs moved to ADIP") + ] + by_label = {f.label: folders[f.page_id] for f in wiki.STATE_FOLDERS} + by_label.update(labelled or {}) + body = ( + index_body if index_body is not None else folded(1, "completed") + folded(1, "abandoned or replaced") + ) + return ( + wiki.Page(wiki.AIP_INDEX_ID, "AIP index", body), + children + (index_children or []), + folders, + by_label, + ) + + +def folder_of(wiki, badge_title): + return next(f for f in wiki.STATE_FOLDERS if f.badge == badge_title) + + +def test_consistent_aip_tree_has_no_problems(wiki): + assert wiki.check_aip_tree(*build_tree(wiki)) == [] + + +def test_page_directly_under_the_aip_index_is_reported(wiki): + stray = wiki.Page("99", "AIP-200 New") + problems = wiki.check_aip_tree(*build_tree(wiki, index_children=[stray])) + assert problems == [ + f"<{stray.url}|AIP-200 New>: sits directly under the AIP index; move it into a state folder" + ] + + [email protected]( + ("body", "expected"), + [ + pytest.param("", "has 0 Page Properties blocks, expected 1", id="no-block"), + pytest.param( + properties({"State": badge("Completed")}) * 2, + "has 2 Page Properties blocks, expected 1", + id="two-blocks", + ), + pytest.param( + properties({"Discussion Thread": "x"}), + "has no State row in its Page Properties block", + id="no-state", + ), + pytest.param( + properties({"State": "Completed"}), + "State badge is None, expected one of ['Completed']", + id="plain-text-state", + ), + pytest.param( + properties({"State": badge("Accepted")}), + "State badge is 'Accepted', expected one of ['Completed']", + id="badge-of-another-state", + ), + ], +) +def test_aip_properties_must_match_the_folder(wiki, body, expected): + completed = folder_of(wiki, "Completed") + page = wiki.Page("5", "AIP-5 X", body, [wiki.BASE_LABEL, completed.label]) + problems = wiki.check_aip_tree(*build_tree(wiki, pages={completed.page_id: [page]})) + assert problems == [f"<{page.url}|AIP-5 X>: {expected}"] + + [email protected]( + ("labels", "expected"), + [ + pytest.param( + ["airflow-improvement-proposal"], + ["state labels are [], expected ['airflow-improvement-proposal/completed']"], + id="no-state-label", + ), + pytest.param( + [ + "airflow-improvement-proposal", + "airflow-improvement-proposal/completed", + "airflow-improvement-proposal/draft", + ], + [ + "state labels are ['airflow-improvement-proposal/completed', 'airflow-improvement-proposal/draft']," + " expected ['airflow-improvement-proposal/completed']" + ], + id="two-state-labels", + ), + pytest.param( + ["airflow-improvement-proposal/completed"], + ["lacks the 'airflow-improvement-proposal' label"], + id="no-base-label", + ), + ], +) +def test_aip_labels_must_match_the_folder(wiki, labels, expected): + completed = folder_of(wiki, "Completed") + page = aip(wiki, completed, page_id="5", title="AIP-5 X", labels=labels) + problems = wiki.check_aip_tree(*build_tree(wiki, pages={completed.page_id: [page]})) + assert problems == [f"<{page.url}|AIP-5 X>: {problem}" for problem in expected] + + +def test_restricted_page_is_not_checked_for_badge_or_labels(wiki): + completed = folder_of(wiki, "Completed") + page = wiki.Page(next(iter(wiki.RESTRICTED_PAGES)), "AIP-52 X", properties({"State": "Completed"}), []) + assert wiki.check_aip_tree(*build_tree(wiki, pages={completed.page_id: [page]})) == [] + + +def test_titles_and_problems_are_escaped_and_shortened_for_slack(wiki): + page = wiki.Page("9", "AIP-9 <!channel> & a|b") + assert ( + wiki.describe(page, "see <x>") == f"<{page.url}|AIP-9 <!channel> & a|b>: see <x>" + ) + long_page = wiki.Page("10", "A" * 300) + assert wiki.describe(long_page, "x") == f"<{long_page.url}|{'A' * 99}…>: x" + + [email protected]( + "macro", + [ + pytest.param('<ac:structured-macro ac:name="details" ac:schema-version="1" />', id="self-closing"), + pytest.param( + '<ac:structured-macro ac:name="details" ac:schema-version="1"></ac:structured-macro>', + id="no-body", + ), + ], +) +def test_macro_without_a_body_has_no_properties_even_before_a_table(wiki, macro): + body = macro + f"<table><tbody><tr><td>State</td><td>{badge('Completed')}</td></tr></tbody></table>" + assert wiki.find_properties(body) == [{}] + + +def test_folder_moved_out_of_the_aip_index_is_reported(wiki): + index, children, folders, labelled = build_tree(wiki) + moved = children[0] + problems = wiki.check_aip_tree(index, children[1:], folders, labelled) + assert problems == [ + f"<{moved.url}|page {moved.page_id}>: folder is no longer directly under the AIP index" + ] + + +def test_properties_with_a_nested_macro_in_a_cell_are_read(wiki): + nested = ( + '<ac:structured-macro ac:name="info"><ac:rich-text-body><p>note</p></ac:rich-text-body>' + "</ac:structured-macro>" + ) + body = properties({"Discussion Thread": nested, "State": badge("Completed")}) + assert wiki.find_badge(wiki.find_properties(body)[0]["State"]) == "Completed" + + +def test_properties_table_outside_the_macro_is_not_read(wiki): + body = ( + '<ac:structured-macro ac:name="details" ac:schema-version="1"><ac:rich-text-body><p></p>' + "</ac:rich-text-body></ac:structured-macro>" + f"<table><tbody><tr><td>State</td><td>{badge('Completed')}</td></tr></tbody></table>" + ) + assert wiki.find_properties(body) == [{}] + + +def test_state_label_outside_its_folder_is_reported(wiki): + draft = folder_of(wiki, "Under discussion") + elsewhere = wiki.Page("77", "AIP-101: Appendix", labels=[draft.label]) + tree = build_tree(wiki) + tree[3][draft.label] = [*tree[2][draft.page_id], elsewhere] + problems = wiki.check_aip_tree(*tree) + assert problems == [ + f"<{elsewhere.url}|AIP-101: Appendix>: has the '{draft.label}' label but is not in that state's folder" + ] + + +def test_stale_count_in_a_folded_section_is_reported(wiki): + tree = build_tree(wiki, index_body=folded(7, "completed") + folded(1, "abandoned or replaced")) + problems = wiki.check_aip_tree(*tree) + assert problems == [ + f"<{tree[0].url}|AIP index>: the folded section should be titled '1 completed AIPs (click to expand)'" + ] + + +def adip(wiki, number=1, title=None, state="Completed", formerly="AIP-4", labels=("adip",)): + rows = {"State": badge(state, "Green")} + if formerly is not None: + rows["Formerly"] = f'<span style="color: rgb(151,160,175);">{formerly}</span>' + return wiki.Page(str(number), title or f"ADIP-{number} Something", properties(rows), list(labels)) + + +def test_consistent_adip_tree_has_no_problems(wiki): + assert wiki.check_adip_tree([adip(wiki, 1), adip(wiki, 2, formerly="AIP-6")]) == [] + + [email protected]( + ("page_kwargs", "expected"), + [ + pytest.param( + {"title": "Automation of tests"}, ["title does not start with 'ADIP-' and its number"], id="title" + ), + pytest.param( + {"title": "ADIP-2 Something (formerly AIP-6)"}, + ["title repeats the former AIP number; it is the Formerly property"], + id="formerly-in-title", + ), + pytest.param({"labels": ()}, ["lacks the 'adip' label"], id="no-label"), + pytest.param( + {"state": "Done"}, + [ + "State badge is 'Done', expected one of ['Abandoned', 'Accepted', 'Completed', 'Draft', 'In progress']" + ], + id="unknown-badge", + ), + pytest.param( + {"formerly": None}, + ["has no Formerly row with the former AIP number in its Page Properties block"], + id="no-formerly", + ), + ], +) +def test_adip_page_problems(wiki, page_kwargs, expected): + page = adip(wiki, 2, **page_kwargs) + assert wiki.check_adip_tree([adip(wiki, 1), page]) == [ + f"<{page.url}|{page.title}>: {p}" for p in expected + ] + + +def test_duplicate_adip_number_is_reported_with_the_other_title_escaped(wiki): + first, second = adip(wiki, 3, title="ADIP-3 <!channel>"), adip(wiki, 4, title="ADIP-3 Two") + assert wiki.check_adip_tree([first, second]) == [ + f"<{second.url}|ADIP-3 Two>: reuses the number of 'ADIP-3 <!channel>'" + ] + + +def test_fetch_children_pages_through_the_listing_and_reads_each_body(wiki): + calls = [] + + def fetch(path, params): + calls.append((path, params.get("start"))) + if path.endswith("/child/page"): + if params["start"] == "0": + return {"results": [{"id": "1", "title": "A"}], "_links": {"next": "more"}} + return {"results": [{"id": "2", "title": "B"}], "_links": {}} + page_id = path.rsplit("/", 1)[1] + return { + "id": page_id, + "title": f"T{page_id}", + "body": {"storage": {"value": f"<p>{page_id}</p>"}}, + "metadata": {"labels": {"results": [{"name": "adip"}]}}, + } + + pages = wiki.fetch_children("10", fetch) + assert [(p.page_id, p.title, p.body, p.labels) for p in pages] == [ + ("1", "T1", "<p>1</p>", ["adip"]), + ("2", "T2", "<p>2</p>", ["adip"]), + ] + assert calls == [ + ("content/10/child/page", "0"), + ("content/10/child/page", "1"), + ("content/1", None), + ("content/2", None), + ] + + +def test_fetch_labelled_searches_by_label(wiki): + fetch = mock.Mock(return_value={"results": [{"id": "3", "title": "AIP-3"}], "_links": {}}) + assert [p.page_id for p in wiki.fetch_labelled("airflow-improvement-proposal/wip", fetch)] == ["3"] + assert fetch.call_args.args[1]["cql"] == 'label = "airflow-improvement-proposal/wip" and type = page' + + [email protected]("urllib.request.urlopen") +def test_get_json_retries_and_decompresses_gzip(urlopen, wiki): + response = mock.MagicMock() + response.__enter__.return_value = response + response.read.return_value = gzip.compress(json.dumps({"ok": True}).encode()) + response.headers = {"Content-Encoding": "gzip"} + urlopen.side_effect = [http.client.RemoteDisconnected("gone"), response] + sleep = mock.Mock() + assert wiki.get_json("content/1", {}, sleep=sleep) == {"ok": True} + sleep.assert_called_once_with(5) + assert urlopen.call_args.args[0].get_header("Accept-encoding") == "gzip" + + [email protected]("urllib.request.urlopen", side_effect=http.client.IncompleteRead(b"")) +def test_get_json_gives_up_after_the_last_attempt(urlopen, wiki): + with pytest.raises(http.client.IncompleteRead): + wiki.get_json("content/1", {}, sleep=mock.Mock()) + assert urlopen.call_count == wiki.ATTEMPTS + + +def test_slack_payload_lists_problems_and_truncates(wiki): + problems = [f"problem {i}" for i in range(wiki.MAX_PROBLEMS_IN_SLACK + 2)] + payload = wiki.build_slack_payload(problems, "https://run";) + text = payload["blocks"][0]["text"]["text"] + assert payload["channel"] == wiki.SLACK_CHANNEL + assert f"found {len(problems)} problem(s)" in payload["text"] + assert "• problem 0" in text + assert f"• problem {wiki.MAX_PROBLEMS_IN_SLACK}" not in text + assert "… and 2 more" in text + assert text.endswith("<https://run|View the run>") + + +def test_main_exits_0_and_writes_no_payload_when_consistent(wiki, tmp_path): + payload = tmp_path / "slack.json" + with mock.patch.object(wiki, "collect_problems", return_value=[]): + assert wiki.main(["--slack-payload", str(payload)]) == 0 + assert not payload.exists() + + +def test_main_exits_1_and_writes_the_payload_on_problems(wiki, tmp_path): + payload = tmp_path / "slack.json" + with mock.patch.object(wiki, "collect_problems", return_value=["bad page"]): + assert wiki.main(["--slack-payload", str(payload), "--run-url", "https://run";]) == 1 + assert "• bad page" in json.loads(payload.read_text())["blocks"][0]["text"]["text"] + + +def test_slack_payload_splits_long_lists_into_sections_within_the_limit(wiki): + problems = [f"problem {i} " + "x" * 400 for i in range(wiki.MAX_PROBLEMS_IN_SLACK)] + sections = [block["text"]["text"] for block in wiki.build_slack_payload(problems, None)["blocks"]] + assert len(sections) > 1 + assert all(len(text) <= wiki.MAX_SECTION_LENGTH for text in sections) + assert "\n".join(sections).count("• problem") == wiki.MAX_PROBLEMS_IN_SLACK
