o-nikolas commented on code in PR #76845: URL: https://github.com/apache/airflow/pull/76845#discussion_r4571063607
########## scripts/ci/verify_improvement_proposals_wiki.py: ########## @@ -0,0 +1,416 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11" +# /// +""" +Check that the AIP and ADIP pages on the Airflow wiki are laid out the way their index pages expect. + +The index pages are generated: each list is a Page Properties Report over the pages of one +folder, so a page in the wrong place, or with a State that does not match where it sits, shows +up in the wrong list or not at all. This script reads the wiki anonymously and reports every +such page. + +The AIP tree (https://cwiki.apache.org/confluence/display/AIRFLOW/AIP+-+Airflow+Improvement+Proposals) +and the ADIP tree are laid out the same way, and for each of them: + + * the index page has only the state folders (and, for AIPs, the folder of ADIP stubs) as + children; + * every page in a state folder has one Page Properties block whose State badge matches the + folder, the tree's base label, and the state label of that folder only; + * a page carrying a state label sits in that state's folder; + * the counts in the titles of the folded sections on the index page match the folders. + +Every ADIP is also titled ``ADIP-<n> <title>`` with a unique number; an ADIP that started as an AIP +has its old number in a Formerly row. + +Exits 1 when anything is out of line. With ``--slack-payload <file>`` it then also writes a +``chat.postMessage`` payload listing the problems, for the workflow to post. +""" + +from __future__ import annotations + +import argparse +import gzip +import http.client +import json +import re +import sys +import time +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path + +WIKI = "https://cwiki.apache.org/confluence" +SLACK_CHANNEL = "internal-airflow-ci-cd" +MAX_PROBLEMS_IN_SLACK = 25 +MAX_TITLE_LENGTH = 100 +MAX_PROBLEM_LENGTH = 300 +# Slack rejects a section block whose text is longer than 3000 characters. +MAX_SECTION_LENGTH = 2900 +ATTEMPTS = 4 + + +@dataclass(frozen=True) +class StateFolder: + page_id: str + state: str + badge: str + # How the folded section of this folder on the index page names its pages, e.g. + # "completed" for "8 completed ADIPs (click to expand)"; None when it is not folded. + folded: str | None = None + + +@dataclass(frozen=True) +class ProposalTree: + name: str + index_id: str + base_label: str + folders: tuple[StateFolder, ...] + other_children: frozenset[str] = frozenset() + + def get_label(self, folder: StateFolder) -> str: + return f"{self.base_label}/{folder.state}" + + def get_state_labels(self) -> set[str]: + return {self.get_label(folder) for folder in self.folders} + + def get_folded_title(self, folder: StateFolder, count: int) -> str: + return f"{count} {folder.folded} {self.name}s (click to expand)" + + +def build_state_folders(wip: str, draft: str, accepted: str, completed: str, abandoned: str): + return ( + StateFolder(wip, "wip", "WIP"), + StateFolder(draft, "draft", "Under discussion"), + StateFolder(accepted, "accepted", "Accepted"), + StateFolder(completed, "completed", "Completed", "completed"), + StateFolder(abandoned, "abandoned", "Abandoned", "abandoned or replaced"), + ) + + +AIP_TREE = ProposalTree( + "AIP", + "89066602", + "airflow-improvement-proposal", + build_state_folders("462816682", "462816536", "462816537", "173087665", "173087786"), + # Pages under the old titles of the AIPs that became ADIPs. + other_children=frozenset({"462816661"}), +) +ADIP_TREE = ProposalTree( + "ADIP", + "462816431", + "adip", + build_state_folders("462816703", "462816705", "462816707", "462816709", "462816711"), +) +TREES = (AIP_TREE, ADIP_TREE) + +# Pages in a state folder that the people who keep the wiki in order cannot edit, with the +# reason. Their State badge and labels are not checked; their place in the tree still is. +RESTRICTED_PAGES = { + "247827224": "AIP-52 has edit restrictions", +} + + +@dataclass +class Page: + page_id: str + title: str + body: str = "" + labels: list[str] = field(default_factory=list) + + @property + def url(self) -> str: + return f"{WIKI}/pages/viewpage.action?pageId={self.page_id}" + + +def get_json(path: str, params: dict[str, str], sleep: Callable[[float], None] = time.sleep) -> dict: + url = f"{WIKI}/rest/api/{path}?{urllib.parse.urlencode(params)}" + # Uncompressed responses of more than a few tens of kilobytes get cut off mid-transfer. + request = urllib.request.Request( + url, + headers={ + "Accept": "application/json", + "Accept-Encoding": "gzip", + "User-Agent": "apache-airflow-wiki-consistency-check", + }, + ) + for attempt in range(1, ATTEMPTS): + try: + return read_json(request) + except (http.client.HTTPException, OSError): + sleep(5 * attempt) + return read_json(request) + + +def read_json(request: urllib.request.Request) -> dict: + with urllib.request.urlopen(request, timeout=60) as response: + raw = response.read() + if response.headers.get("Content-Encoding") == "gzip": + raw = gzip.decompress(raw) + return json.loads(raw) + + +def build_page(raw: dict) -> Page: + return Page( + page_id=raw["id"], + title=raw["title"], + body=raw.get("body", {}).get("storage", {}).get("value", ""), + labels=[label["name"] for label in raw.get("metadata", {}).get("labels", {}).get("results", [])], + ) + + +def fetch_all(path: str, params: dict[str, str], fetch: Callable[[str, dict[str, str]], dict]) -> list[Page]: + pages: list[Page] = [] + start = 0 + while True: + data = fetch(path, {**params, "limit": "50", "start": str(start)}) + pages.extend(build_page(raw) for raw in data["results"]) + if not data["results"] or "next" not in data.get("_links", {}): + return pages + start += len(data["results"]) + + +def fetch_page(page_id: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> Page: + return build_page(fetch(f"content/{page_id}", {"expand": "body.storage,metadata.labels"})) + + +def fetch_children(page_id: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> list[Page]: + # Bodies are fetched one page at a time: a response carrying several AIP bodies is large + # enough to be cut off mid-transfer. + listed = fetch_all(f"content/{page_id}/child/page", {}, fetch) + return [fetch_page(page.page_id, fetch) for page in listed] + + +def fetch_labelled(label: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> list[Page]: + return fetch_all("content/search", {"cql": f'label = "{label}" and type = page'}, fetch) + + +def get_text(html: str) -> str: + return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html).replace(" ", " ")).strip() + + +def find_macro_body_end(body: str, start: int) -> int: + """Return where the rich-text body opened after ``start`` closes, skipping nested macro bodies.""" + depth = 0 + for tag in re.finditer(r"<(/?)ac:rich-text-body>", body[start:]): + depth += -1 if tag.group(1) else 1 + if depth == 0: + return start + tag.start() + return len(body) + + +def find_properties(body: str) -> list[dict[str, str]]: + """Return the rows of each Page Properties block as {key: value storage HTML}.""" + blocks = [] + for match in re.finditer(r'<ac:structured-macro ac:name="details"', body): + # Only a table inside the macro's own rich-text body is read by the index report; a + # self-closing macro, or one with no body, has no properties. + opening = re.match( + r"[^>]*?(/?)>(?:\s*<ac:parameter[^>]*>.*?</ac:parameter>)*\s*<ac:rich-text-body>", Review Comment: The `.*?` in the parameter group isn't bounded to one `<ac:parameter>` (with `re.S` it can run past `</ac:parameter>`), which causes the CodeQL failure and also a wrong match: a details macro with a parameter but no body picks up the next macro's `<ac:rich-text-body>` further down the page and reads that table as its properties. Locally, a body-less `details` macro followed by an `expand` macro holding a table gave `[{'State': 'WRONG TABLE'}]`, and 20 parameters with no body took ~0.5 s, about 4x more per 2 extra parameters. Using `[^<]*` for the value fixes both (gives `[{}]` and runs instantly): r"[^>]*?(/?)>(?:\s*<ac:parameter[^>]*>[^<]*</ac:parameter>)*\s*<ac:rich-text-body>", Could you also add a case for it to `test_macro_without_a_body_has_no_properties_even_before_a_table`, e.g. a body-less macro with an `id` parameter followed by an `expand` macro containing the table? --- Drafted-by: Kiro (claude-opus-5.5); reviewed by @o-nikolas before posting ########## scripts/ci/verify_improvement_proposals_wiki.py: ########## @@ -0,0 +1,416 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11" +# /// +""" +Check that the AIP and ADIP pages on the Airflow wiki are laid out the way their index pages expect. + +The index pages are generated: each list is a Page Properties Report over the pages of one +folder, so a page in the wrong place, or with a State that does not match where it sits, shows +up in the wrong list or not at all. This script reads the wiki anonymously and reports every +such page. + +The AIP tree (https://cwiki.apache.org/confluence/display/AIRFLOW/AIP+-+Airflow+Improvement+Proposals) +and the ADIP tree are laid out the same way, and for each of them: + + * the index page has only the state folders (and, for AIPs, the folder of ADIP stubs) as + children; + * every page in a state folder has one Page Properties block whose State badge matches the + folder, the tree's base label, and the state label of that folder only; + * a page carrying a state label sits in that state's folder; + * the counts in the titles of the folded sections on the index page match the folders. + +Every ADIP is also titled ``ADIP-<n> <title>`` with a unique number; an ADIP that started as an AIP +has its old number in a Formerly row. + +Exits 1 when anything is out of line. With ``--slack-payload <file>`` it then also writes a +``chat.postMessage`` payload listing the problems, for the workflow to post. +""" + +from __future__ import annotations + +import argparse +import gzip +import http.client +import json +import re +import sys +import time +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path + +WIKI = "https://cwiki.apache.org/confluence" +SLACK_CHANNEL = "internal-airflow-ci-cd" +MAX_PROBLEMS_IN_SLACK = 25 +MAX_TITLE_LENGTH = 100 +MAX_PROBLEM_LENGTH = 300 +# Slack rejects a section block whose text is longer than 3000 characters. +MAX_SECTION_LENGTH = 2900 +ATTEMPTS = 4 + + +@dataclass(frozen=True) +class StateFolder: + page_id: str + state: str + badge: str + # How the folded section of this folder on the index page names its pages, e.g. + # "completed" for "8 completed ADIPs (click to expand)"; None when it is not folded. + folded: str | None = None + + +@dataclass(frozen=True) +class ProposalTree: + name: str + index_id: str + base_label: str + folders: tuple[StateFolder, ...] + other_children: frozenset[str] = frozenset() + + def get_label(self, folder: StateFolder) -> str: + return f"{self.base_label}/{folder.state}" + + def get_state_labels(self) -> set[str]: + return {self.get_label(folder) for folder in self.folders} + + def get_folded_title(self, folder: StateFolder, count: int) -> str: + return f"{count} {folder.folded} {self.name}s (click to expand)" + + +def build_state_folders(wip: str, draft: str, accepted: str, completed: str, abandoned: str): + return ( + StateFolder(wip, "wip", "WIP"), + StateFolder(draft, "draft", "Under discussion"), + StateFolder(accepted, "accepted", "Accepted"), + StateFolder(completed, "completed", "Completed", "completed"), + StateFolder(abandoned, "abandoned", "Abandoned", "abandoned or replaced"), + ) + + +AIP_TREE = ProposalTree( + "AIP", + "89066602", + "airflow-improvement-proposal", + build_state_folders("462816682", "462816536", "462816537", "173087665", "173087786"), + # Pages under the old titles of the AIPs that became ADIPs. + other_children=frozenset({"462816661"}), +) +ADIP_TREE = ProposalTree( + "ADIP", + "462816431", + "adip", + build_state_folders("462816703", "462816705", "462816707", "462816709", "462816711"), +) +TREES = (AIP_TREE, ADIP_TREE) + +# Pages in a state folder that the people who keep the wiki in order cannot edit, with the +# reason. Their State badge and labels are not checked; their place in the tree still is. +RESTRICTED_PAGES = { + "247827224": "AIP-52 has edit restrictions", +} + + +@dataclass +class Page: + page_id: str + title: str + body: str = "" + labels: list[str] = field(default_factory=list) + + @property + def url(self) -> str: + return f"{WIKI}/pages/viewpage.action?pageId={self.page_id}" + + +def get_json(path: str, params: dict[str, str], sleep: Callable[[float], None] = time.sleep) -> dict: + url = f"{WIKI}/rest/api/{path}?{urllib.parse.urlencode(params)}" + # Uncompressed responses of more than a few tens of kilobytes get cut off mid-transfer. + request = urllib.request.Request( + url, + headers={ + "Accept": "application/json", + "Accept-Encoding": "gzip", + "User-Agent": "apache-airflow-wiki-consistency-check", + }, + ) + for attempt in range(1, ATTEMPTS): + try: + return read_json(request) + except (http.client.HTTPException, OSError): + sleep(5 * attempt) + return read_json(request) + + +def read_json(request: urllib.request.Request) -> dict: + with urllib.request.urlopen(request, timeout=60) as response: + raw = response.read() + if response.headers.get("Content-Encoding") == "gzip": + raw = gzip.decompress(raw) + return json.loads(raw) + + +def build_page(raw: dict) -> Page: + return Page( + page_id=raw["id"], + title=raw["title"], + body=raw.get("body", {}).get("storage", {}).get("value", ""), + labels=[label["name"] for label in raw.get("metadata", {}).get("labels", {}).get("results", [])], + ) + + +def fetch_all(path: str, params: dict[str, str], fetch: Callable[[str, dict[str, str]], dict]) -> list[Page]: + pages: list[Page] = [] + start = 0 + while True: + data = fetch(path, {**params, "limit": "50", "start": str(start)}) + pages.extend(build_page(raw) for raw in data["results"]) + if not data["results"] or "next" not in data.get("_links", {}): + return pages + start += len(data["results"]) + + +def fetch_page(page_id: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> Page: + return build_page(fetch(f"content/{page_id}", {"expand": "body.storage,metadata.labels"})) + + +def fetch_children(page_id: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> list[Page]: + # Bodies are fetched one page at a time: a response carrying several AIP bodies is large + # enough to be cut off mid-transfer. + listed = fetch_all(f"content/{page_id}/child/page", {}, fetch) + return [fetch_page(page.page_id, fetch) for page in listed] + + +def fetch_labelled(label: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> list[Page]: + return fetch_all("content/search", {"cql": f'label = "{label}" and type = page'}, fetch) Review Comment: Nit: the CQL search covers every cwiki space. Adding `and space = AIRFLOW` would stop a page elsewhere with the same label from being reported as outside its folder. --- Drafted-by: Kiro (claude-opus-5.5); reviewed by @o-nikolas before posting ########## scripts/tests/ci/test_verify_improvement_proposals_wiki.py: ########## @@ -0,0 +1,431 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import gzip +import http.client +import importlib.util +import json +import sys +from pathlib import Path +from unittest import mock + +import pytest + +MODULE_PATH = Path(__file__).resolve().parents[3] / "scripts" / "ci" / "verify_improvement_proposals_wiki.py" + + [email protected] +def wiki(): + module_name = "test_verify_improvement_proposals_wiki_module" + sys.modules.pop(module_name, None) + spec = importlib.util.spec_from_file_location(module_name, MODULE_PATH) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +def badge(title, colour="Grey"): + return ( + '<ac:structured-macro ac:name="status" ac:schema-version="1" ac:macro-id="x">' + f'<ac:parameter ac:name="colour">{colour}</ac:parameter>' + f'<ac:parameter ac:name="title">{title}</ac:parameter></ac:structured-macro>' + ) + + +def properties(rows): + cells = "".join( + f"<tr><td><strong>{key}</strong></td><td>{value}</td></tr>" for key, value in rows.items() + ) + return ( + '<ac:structured-macro ac:name="details" ac:schema-version="1" ac:macro-id="y">' + f"<ac:rich-text-body><table><tbody>{cells}</tbody></table></ac:rich-text-body></ac:structured-macro>" + ) + + +def folded(count, kind, name="AIP"): + return ( + '<ac:structured-macro ac:name="expand" ac:schema-version="1"><ac:parameter ac:name="title">' + f"{count} {kind} {name}s (click to expand)</ac:parameter></ac:structured-macro>" + ) + + +def aip(wiki, folder, page_id="1", title="AIP-1 Something", state=None, labels=None, tree=None): + tree = tree or wiki.AIP_TREE + return wiki.Page( + page_id, + title, + properties({"State": badge(state or folder.badge) + " extra detail"}), + [tree.base_label, tree.get_label(folder)] if labels is None else labels, + ) + + +def build_tree(wiki, tree=None, pages=None, index_children=None, labelled=None, index_body=None): + """A consistent tree with one proposal in each state folder, with overrides.""" + tree = tree or wiki.AIP_TREE + folders = {f.page_id: [aip(wiki, f, page_id=f"p{f.page_id}", tree=tree)] for f in tree.folders} + folders.update(pages or {}) + children = [wiki.Page(f.page_id, f.badge) for f in tree.folders] + [ + wiki.Page(page_id, "other") for page_id in tree.other_children + ] + by_label = {tree.get_label(f): folders[f.page_id] for f in tree.folders} + by_label.update(labelled or {}) + body = ( + index_body + if index_body is not None + else folded(1, "completed", tree.name) + folded(1, "abandoned or replaced", tree.name) + ) + return ( + tree, + wiki.Page(tree.index_id, f"{tree.name} index", body), + children + (index_children or []), + folders, + by_label, + ) + + +def folder_of(wiki, badge_title, tree=None): + return next(f for f in (tree or wiki.AIP_TREE).folders if f.badge == badge_title) + + [email protected]("tree_name", ["AIP_TREE", "ADIP_TREE"]) +def test_consistent_trees_have_no_problems(wiki, tree_name): + assert wiki.check_tree(*build_tree(wiki, tree=getattr(wiki, tree_name))) == [] + + +def test_page_directly_under_the_aip_index_is_reported(wiki): + stray = wiki.Page("99", "AIP-200 New") + problems = wiki.check_tree(*build_tree(wiki, index_children=[stray])) + assert problems == [ + f"<{stray.url}|AIP-200 New>: sits directly under the AIP index; move it into a state folder" + ] + + [email protected]( + ("body", "expected"), + [ + pytest.param("", "has 0 Page Properties blocks, expected 1", id="no-block"), + pytest.param( + properties({"State": badge("Completed")}) * 2, + "has 2 Page Properties blocks, expected 1", + id="two-blocks", + ), + pytest.param( + properties({"Discussion Thread": "x"}), + "has no State row in its Page Properties block", + id="no-state", + ), + pytest.param( + properties({"State": "Completed"}), + "State badge is None, expected one of ['Completed']", + id="plain-text-state", + ), + pytest.param( + properties({"State": badge("Accepted")}), + "State badge is 'Accepted', expected one of ['Completed']", + id="badge-of-another-state", + ), + ], +) +def test_aip_properties_must_match_the_folder(wiki, body, expected): + completed = folder_of(wiki, "Completed") + page = wiki.Page("5", "AIP-5 X", body, [wiki.AIP_TREE.base_label, wiki.AIP_TREE.get_label(completed)]) + problems = wiki.check_tree(*build_tree(wiki, pages={completed.page_id: [page]})) + assert problems == [f"<{page.url}|AIP-5 X>: {expected}"] + + [email protected]( + ("labels", "expected"), + [ + pytest.param( + ["airflow-improvement-proposal"], + ["state labels are [], expected ['airflow-improvement-proposal/completed']"], + id="no-state-label", + ), + pytest.param( + [ + "airflow-improvement-proposal", + "airflow-improvement-proposal/completed", + "airflow-improvement-proposal/draft", + ], + [ + "state labels are ['airflow-improvement-proposal/completed', 'airflow-improvement-proposal/draft']," + " expected ['airflow-improvement-proposal/completed']" + ], + id="two-state-labels", + ), + pytest.param( + ["airflow-improvement-proposal/completed"], + ["lacks the 'airflow-improvement-proposal' label"], + id="no-base-label", + ), + ], +) +def test_aip_labels_must_match_the_folder(wiki, labels, expected): + completed = folder_of(wiki, "Completed") + page = aip(wiki, completed, page_id="5", title="AIP-5 X", labels=labels) + problems = wiki.check_tree(*build_tree(wiki, pages={completed.page_id: [page]})) + assert problems == [f"<{page.url}|AIP-5 X>: {problem}" for problem in expected] + + +def test_restricted_page_is_not_checked_for_badge_or_labels(wiki): + completed = folder_of(wiki, "Completed") + page = wiki.Page(next(iter(wiki.RESTRICTED_PAGES)), "AIP-52 X", properties({"State": "Completed"}), []) + assert wiki.check_tree(*build_tree(wiki, pages={completed.page_id: [page]})) == [] + + +def test_titles_and_problems_are_escaped_and_shortened_for_slack(wiki): + page = wiki.Page("9", "AIP-9 <!channel> & a|b") + assert ( + wiki.describe(page, "see <x>") == f"<{page.url}|AIP-9 <!channel> & a|b>: see <x>" + ) + long_page = wiki.Page("10", "A" * 300) + assert wiki.describe(long_page, "x") == f"<{long_page.url}|{'A' * 99}…>: x" + + [email protected]( + "macro", + [ + pytest.param('<ac:structured-macro ac:name="details" ac:schema-version="1" />', id="self-closing"), + pytest.param( + '<ac:structured-macro ac:name="details" ac:schema-version="1"></ac:structured-macro>', + id="no-body", + ), + ], +) +def test_macro_without_a_body_has_no_properties_even_before_a_table(wiki, macro): + body = macro + f"<table><tbody><tr><td>State</td><td>{badge('Completed')}</td></tr></tbody></table>" + assert wiki.find_properties(body) == [{}] + + [email protected]("tree_name", ["AIP_TREE", "ADIP_TREE"]) +def test_folder_moved_out_of_the_index_is_reported(wiki, tree_name): + tree, index, children, folders, labelled = build_tree(wiki, tree=getattr(wiki, tree_name)) + moved = children[0] + problems = wiki.check_tree(tree, index, children[1:], folders, labelled) + assert problems == [ + f"<{moved.url}|page {moved.page_id}>: folder is no longer directly under the {tree.name} index" + ] + + +def test_properties_with_a_nested_macro_in_a_cell_are_read(wiki): + nested = ( + '<ac:structured-macro ac:name="info"><ac:rich-text-body><p>note</p></ac:rich-text-body>' + "</ac:structured-macro>" + ) + body = properties({"Discussion Thread": nested, "State": badge("Completed")}) + assert wiki.find_badge(wiki.find_properties(body)[0]["State"]) == "Completed" + + +def test_properties_table_outside_the_macro_is_not_read(wiki): + body = ( + '<ac:structured-macro ac:name="details" ac:schema-version="1"><ac:rich-text-body><p></p>' + "</ac:rich-text-body></ac:structured-macro>" + f"<table><tbody><tr><td>State</td><td>{badge('Completed')}</td></tr></tbody></table>" + ) + assert wiki.find_properties(body) == [{}] + + +def test_state_label_outside_its_folder_is_reported(wiki): + draft = folder_of(wiki, "Under discussion") + label = wiki.AIP_TREE.get_label(draft) + elsewhere = wiki.Page("77", "AIP-101: Appendix", labels=[label]) + tree = build_tree(wiki) + tree[4][label] = [*tree[3][draft.page_id], elsewhere] + problems = wiki.check_tree(*tree) + assert problems == [ + f"<{elsewhere.url}|AIP-101: Appendix>: has the '{label}' label but is not in that state's folder" + ] + + [email protected]("tree_name", ["AIP_TREE", "ADIP_TREE"]) +def test_stale_count_in_a_folded_section_is_reported(wiki, tree_name): + name = getattr(wiki, tree_name).name + tree = build_tree( + wiki, + tree=getattr(wiki, tree_name), + index_body=folded(7, "completed", name) + folded(1, "abandoned or replaced", name), + ) + problems = wiki.check_tree(*tree) + assert problems == [ + f"<{tree[1].url}|{name} index>: the folded section should be titled " + f"'1 completed {name}s (click to expand)'" + ] + + +def adip(wiki, number=1, title=None, formerly="AIP-4"): + rows = {"State": badge("Completed", "Green")} + if formerly is not None: + rows["Formerly"] = f'<span style="color: rgb(151,160,175);">{formerly}</span>' + return wiki.Page( + str(number), title or f"ADIP-{number} Something", properties(rows), ["adip", "adip/completed"] + ) + + +def test_consistent_adip_pages_have_no_problems(wiki): + assert wiki.check_adip_pages([adip(wiki, 1), adip(wiki, 2, formerly="AIP-6")]) == [] + + [email protected]( + ("page_kwargs", "expected"), + [ + pytest.param( + {"title": "Automation of tests"}, ["title does not start with 'ADIP-' and its number"], id="title" + ), + pytest.param( + {"title": "ADIP-2 Something (formerly AIP-6)"}, + ["title repeats the former AIP number; it is the Formerly property"], + id="formerly-in-title", + ), + pytest.param({"formerly": None}, [], id="new-adip-without-formerly"), + pytest.param( + {"formerly": "AIP 6"}, ["Formerly is 'AIP 6', expected 'AIP-' and its number"], id="bad-formerly" + ), + ], +) +def test_adip_page_problems(wiki, page_kwargs, expected): + page = adip(wiki, 2, **page_kwargs) + assert wiki.check_adip_pages([adip(wiki, 1), page]) == [ + f"<{page.url}|{page.title}>: {p}" for p in expected + ] + + +def test_duplicate_adip_number_is_reported_with_the_other_title_escaped(wiki): + first, second = adip(wiki, 3, title="ADIP-3 <!channel>"), adip(wiki, 4, title="ADIP-3 Two") + assert wiki.check_adip_pages([first, second]) == [ + f"<{second.url}|ADIP-3 Two>: reuses the number of 'ADIP-3 <!channel>'" + ] + + +def test_fetch_children_pages_through_the_listing_and_reads_each_body(wiki): + calls = [] + + def fetch(path, params): + calls.append((path, params.get("start"))) + if path.endswith("/child/page"): + if params["start"] == "0": + return {"results": [{"id": "1", "title": "A"}], "_links": {"next": "more"}} + return {"results": [{"id": "2", "title": "B"}], "_links": {}} + page_id = path.rsplit("/", 1)[1] + return { + "id": page_id, + "title": f"T{page_id}", + "body": {"storage": {"value": f"<p>{page_id}</p>"}}, + "metadata": {"labels": {"results": [{"name": "adip"}]}}, + } + + pages = wiki.fetch_children("10", fetch) + assert [(p.page_id, p.title, p.body, p.labels) for p in pages] == [ + ("1", "T1", "<p>1</p>", ["adip"]), + ("2", "T2", "<p>2</p>", ["adip"]), + ] + assert calls == [ + ("content/10/child/page", "0"), + ("content/10/child/page", "1"), + ("content/1", None), + ("content/2", None), + ] + + +def test_fetch_labelled_searches_by_label(wiki): + fetch = mock.Mock(return_value={"results": [{"id": "3", "title": "AIP-3"}], "_links": {}}) + assert [p.page_id for p in wiki.fetch_labelled("airflow-improvement-proposal/wip", fetch)] == ["3"] + assert fetch.call_args.args[1]["cql"] == 'label = "airflow-improvement-proposal/wip" and type = page' + + [email protected]("urllib.request.urlopen") +def test_get_json_retries_and_decompresses_gzip(urlopen, wiki): + response = mock.MagicMock() Review Comment: Nit: AGENTS.md asks for `spec`/`autospec` on mocks. This one and the bare `mock.Mock()` fetch/sleep mocks (L348, L360, L369, L419) could take a `spec`, e.g. `mock.MagicMock(spec=http.client.HTTPResponse)` here. --- Drafted-by: Kiro (claude-opus-5.5); reviewed by @o-nikolas before posting -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
