github-advanced-security[bot] commented on code in PR #76845: URL: https://github.com/apache/airflow/pull/76845#discussion_r4529171072
########## scripts/ci/verify_improvement_proposals_wiki.py: ########## @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11" +# /// +""" +Check that the AIP and ADIP pages on the Airflow wiki are laid out the way their index pages expect. + +The index pages are generated: each list is a Page Properties Report over the pages of one +folder, so a page in the wrong place, or with a State that does not match where it sits, shows +up in the wrong list or not at all. This script reads the wiki anonymously and reports every +such page. + +The AIP tree (https://cwiki.apache.org/confluence/display/AIRFLOW/AIP+-+Airflow+Improvement+Proposals) +and the ADIP tree are laid out the same way, and for each of them: + + * the index page has only the state folders (and, for AIPs, the folder of ADIP stubs) as + children; + * every page in a state folder has one Page Properties block whose State badge matches the + folder, the tree's base label, and the state label of that folder only; + * a page carrying a state label sits in that state's folder; + * the counts in the titles of the folded sections on the index page match the folders. + +Every ADIP is also titled ``ADIP-<n> <title>`` with a unique number and has a Formerly row. + +Exits 1 when anything is out of line. With ``--slack-payload <file>`` it then also writes a +``chat.postMessage`` payload listing the problems, for the workflow to post. +""" + +from __future__ import annotations + +import argparse +import gzip +import http.client +import json +import re +import sys +import time +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path + +WIKI = "https://cwiki.apache.org/confluence"; +SLACK_CHANNEL = "internal-airflow-ci-cd" +MAX_PROBLEMS_IN_SLACK = 25 +MAX_TITLE_LENGTH = 100 +# Slack rejects a section block whose text is longer than 3000 characters. +MAX_SECTION_LENGTH = 2900 +ATTEMPTS = 4 + + +@dataclass(frozen=True) +class StateFolder: + page_id: str + state: str + badge: str + # How the folded section of this folder on the index page names its pages, e.g. + # "completed" for "8 completed ADIPs (click to expand)"; None when it is not folded. + folded: str | None = None + + +@dataclass(frozen=True) +class ProposalTree: + name: str + index_id: str + base_label: str + folders: tuple[StateFolder, ...] + other_children: frozenset[str] = frozenset() + + def get_label(self, folder: StateFolder) -> str: + return f"{self.base_label}/{folder.state}" + + def get_state_labels(self) -> set[str]: + return {self.get_label(folder) for folder in self.folders} + + def get_folded_title(self, folder: StateFolder, count: int) -> str: + return f"{count} {folder.folded} {self.name}s (click to expand)" + + +def build_state_folders(wip: str, draft: str, accepted: str, completed: str, abandoned: str): + return ( + StateFolder(wip, "wip", "WIP"), + StateFolder(draft, "draft", "Under discussion"), + StateFolder(accepted, "accepted", "Accepted"), + StateFolder(completed, "completed", "Completed", "completed"), + StateFolder(abandoned, "abandoned", "Abandoned", "abandoned or replaced"), + ) + + +AIP_TREE = ProposalTree( + "AIP", + "89066602", + "airflow-improvement-proposal", + build_state_folders("462816682", "462816536", "462816537", "173087665", "173087786"), + # Pages under the old titles of the AIPs that became ADIPs. + other_children=frozenset({"462816661"}), +) +ADIP_TREE = ProposalTree( + "ADIP", + "462816431", + "adip", + build_state_folders("462816703", "462816705", "462816707", "462816709", "462816711"), +) +TREES = (AIP_TREE, ADIP_TREE) + +# Pages in a state folder that the people who keep the wiki in order cannot edit, with the +# reason. Their State badge and labels are not checked; their place in the tree still is. +RESTRICTED_PAGES = { + "247827224": "AIP-52 has edit restrictions", +} + + +@dataclass +class Page: + page_id: str + title: str + body: str = "" + labels: list[str] = field(default_factory=list) + + @property + def url(self) -> str: + return f"{WIKI}/pages/viewpage.action?pageId={self.page_id}" + + +def get_json(path: str, params: dict[str, str], sleep: Callable[[float], None] = time.sleep) -> dict: + url = f"{WIKI}/rest/api/{path}?{urllib.parse.urlencode(params)}" + # Uncompressed responses of more than a few tens of kilobytes get cut off mid-transfer. + request = urllib.request.Request( + url, + headers={ + "Accept": "application/json", + "Accept-Encoding": "gzip", + "User-Agent": "apache-airflow-wiki-consistency-check", + }, + ) + for attempt in range(1, ATTEMPTS): + try: + return read_json(request) + except (http.client.HTTPException, OSError): + sleep(5 * attempt) + return read_json(request) + + +def read_json(request: urllib.request.Request) -> dict: + with urllib.request.urlopen(request, timeout=60) as response: + raw = response.read() + if response.headers.get("Content-Encoding") == "gzip": + raw = gzip.decompress(raw) + return json.loads(raw) + + +def build_page(raw: dict) -> Page: + return Page( + page_id=raw["id"], + title=raw["title"], + body=raw.get("body", {}).get("storage", {}).get("value", ""), + labels=[label["name"] for label in raw.get("metadata", {}).get("labels", {}).get("results", [])], + ) + + +def fetch_all(path: str, params: dict[str, str], fetch: Callable[[str, dict[str, str]], dict]) -> list[Page]: + pages: list[Page] = [] + start = 0 + while True: + data = fetch(path, {**params, "limit": "50", "start": str(start)}) + pages.extend(build_page(raw) for raw in data["results"]) + if not data["results"] or "next" not in data.get("_links", {}): + return pages + start += len(data["results"]) + + +def fetch_page(page_id: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> Page: + return build_page(fetch(f"content/{page_id}", {"expand": "body.storage,metadata.labels"})) + + +def fetch_children(page_id: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> list[Page]: + # Bodies are fetched one page at a time: a response carrying several AIP bodies is large + # enough to be cut off mid-transfer. + listed = fetch_all(f"content/{page_id}/child/page", {}, fetch) + return [fetch_page(page.page_id, fetch) for page in listed] + + +def fetch_labelled(label: str, fetch: Callable[[str, dict[str, str]], dict] = get_json) -> list[Page]: + return fetch_all("content/search", {"cql": f'label = "{label}" and type = page'}, fetch) + + +def get_text(html: str) -> str: + return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html).replace(" ", " ")).strip() + + +def find_macro_body_end(body: str, start: int) -> int: + """Return where the rich-text body opened after ``start`` closes, skipping nested macro bodies.""" + depth = 0 + for tag in re.finditer(r"<(/?)ac:rich-text-body>", body[start:]): + depth += -1 if tag.group(1) else 1 + if depth == 0: + return start + tag.start() + return len(body) + + +def find_properties(body: str) -> list[dict[str, str]]: + """Return the rows of each Page Properties block as {key: value storage HTML}.""" + blocks = [] + for match in re.finditer(r'<ac:structured-macro ac:name="details"', body): + # Only a table inside the macro's own rich-text body is read by the index report; a + # self-closing macro, or one with no body, has no properties. + opening = re.match( + r"[^>]*?(/?)>(?:\s*<ac:parameter[^>]*>.*?</ac:parameter>)*\s*<ac:rich-text-body>", Review Comment: ## CodeQL / Inefficient regular expression This part of the regular expression may cause exponential backtracking on strings starting with '><ac:parameter>' and containing many repetitions of '</ac:parameter><ac:parameter>'. [Show more details](https://github.com/apache/airflow/security/code-scanning/703) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
