codeant-ai-for-open-source[bot] commented on code in PR #41133: URL: https://github.com/apache/superset/pull/41133#discussion_r3507549339
########## superset/utils/excel_streaming.py: ########## @@ -0,0 +1,232 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Streaming XLSX writer for multi-sheet dashboard exports. + +Unlike :mod:`superset.utils.excel`, which builds an in-memory DataFrame per +sheet and hands the whole thing to ``xlsxwriter`` at once, this writer opens the +workbook in ``constant_memory`` mode and writes rows one at a time, so +``xlsxwriter`` keeps at most one row per sheet buffered on the writer side. The +source records may still be materialized upstream (e.g. by the chart query +response); this bounds only the writer's own footprint, not the caller's. +""" + +from __future__ import annotations + +import math +import numbers +import re +from collections.abc import Iterable, Sequence +from datetime import date, datetime +from decimal import Decimal +from typing import Any + +import xlsxwriter + +from superset.utils.excel import NEUTRAL_DOCUMENT_PROPERTIES + +# Excel limits a sheet name to 31 characters and forbids these characters. +MAX_SHEET_NAME_LEN = 31 +_INVALID_SHEET_CHARS_RE = re.compile(r"[\[\]:*?/\\]") +# Excel reserves the sheet name "History" (case-insensitive). +_RESERVED_SHEET_NAME = "history" + +# A worksheet holds at most 1,048,576 rows; one is reserved for the header. +MAX_DATA_ROWS_PER_SHEET = 1_048_576 - 1 + +# Leading characters that turn a cell into a formula in spreadsheet apps. Mirrors +# superset.utils.excel.quote_formulas so streamed exports get the same guard. +_FORMULA_PREFIXES = {"=", "+", "-", "@"} + +# Excel cannot represent integers beyond 10**15 without precision loss. +_MAX_EXCEL_INT = 10**15 + + +def _quote_if_formula(text: str) -> str: + """ + Prefix formula-like text with an apostrophe so spreadsheet apps treat it as + literal text (defense against formula injection). + + Leading whitespace is ignored when detecting a formula, because spreadsheet + apps still evaluate a cell whose formula prefix is preceded by spaces or + tabs (e.g. ``" =cmd"`` or ``"\\t=cmd"``). + """ + stripped = text.lstrip() + return f"'{text}" if stripped and stripped[0] in _FORMULA_PREFIXES else text + + +def _coerce_float_cell(value: Any) -> Any: + """ + Convert a ``Decimal``/real value to something ``xlsxwriter`` accepts. + + ``float()`` on a non-finite ``Decimal`` ("NaN"/"Infinity") yields a value + xlsxwriter rejects, and an over-large value can raise ``OverflowError``; + blank the former and stringify the latter, and stringify magnitudes Excel + cannot represent precisely. + """ + try: + number = float(value) + except (OverflowError, ValueError): + return str(value) + if not math.isfinite(number): + return "" + return str(number) if abs(number) > _MAX_EXCEL_INT else number + + +def sanitize_sheet_name(raw: str, used: set[str]) -> str: + """ + Produce a valid, unique Excel sheet name from ``raw``. + + Replaces forbidden characters, strips surrounding apostrophes/whitespace, + avoids the reserved name "History", truncates to 31 characters, and + disambiguates case-insensitive collisions with ``~2``/``~3`` suffixes. + The chosen name (lower-cased) is added to ``used``. + + :param raw: The desired sheet name (e.g. ``"42 - Sales by Region"``) + :param used: Lower-cased names already taken; mutated with the result + :returns: A sanitized, unique sheet name no longer than 31 characters + """ + name = _INVALID_SHEET_CHARS_RE.sub("_", raw or "") + name = name.strip().strip("'").strip() + if not name: + name = "Sheet" + if name.lower() == _RESERVED_SHEET_NAME: + name = f"{name}_" + name = name[:MAX_SHEET_NAME_LEN] + + if name.lower() not in used: + used.add(name.lower()) + return name + + suffix = 2 + while True: + marker = f"~{suffix}" + candidate = name[: MAX_SHEET_NAME_LEN - len(marker)] + marker + if candidate.lower() not in used: + used.add(candidate.lower()) + return candidate + suffix += 1 + + +def _sanitize_cell(value: Any) -> Any: + """ + Coerce a single cell value into something safe for ``xlsxwriter``. + + Quotes formula-like strings (defense against formula injection), stringifies + integers/floats Excel cannot represent precisely, renders temporal values as + ISO strings (timezones are not natively supported), and blanks out ``None`` + and non-finite floats. + """ + if value is None: + return "" + # bool is a subclass of int; preserve it before the numeric branches. + if isinstance(value, bool): + return value + if isinstance(value, str): + return _quote_if_formula(value) + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, Decimal): + return _coerce_float_cell(value) + if isinstance(value, numbers.Integral): + number = int(value) + return str(number) if abs(number) > _MAX_EXCEL_INT else number + if isinstance(value, numbers.Real): + return _coerce_float_cell(value) + # Anything else (lists, dicts, custom objects) is stringified, still guarding + # against formula injection on the resulting text. + return _quote_if_formula(str(value)) + + +class StreamingXlsxWriter: + """ + A thin wrapper over ``xlsxwriter`` in constant-memory mode that writes one + sheet per chart, row by row. + + Sheet names are sanitized and de-duplicated, cell values are sanitized for + safety/compatibility, and per-sheet row counts are capped at Excel's limit. + Always call :meth:`close` (e.g. in a ``finally`` block) to finalize the file. + """ + + def __init__(self, path: str) -> None: + self._workbook = xlsxwriter.Workbook(path, {"constant_memory": True}) + # Reset document properties so the file carries no identifying details. + self._workbook.set_properties(NEUTRAL_DOCUMENT_PROPERTIES) + self._used_sheet_names: set[str] = set() + self.sheet_count = 0 Review Comment: **Suggestion:** Add an explicit type annotation for this instance attribute during initialization to satisfy the type-hint requirement for relevant variables. [custom_rule] **Severity Level:** Minor ⚠️ <details> <summary><b>Why it matters? 🤔 </b></summary> The new class initializes a clearly typeable instance variable with a bare assignment, and the custom rule requires type hints for relevant variables in modified Python code. This is a real omission that can be annotated as an integer attribute. </details> [](https://app.codeant.ai/fix-in-ide?tool=cursor&prompt_id=580cfcf37aa245e58adf31abacb080cc&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) [](https://app.codeant.ai/fix-in-ide?tool=vscode-claude&prompt_id=580cfcf37aa245e58adf31abacb080cc&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) *(Use Cmd/Ctrl + Click for best experience)* <details> <summary><b>Prompt for AI Agent 🤖 </b></summary> ```mdx This is a comment left during a code review. **Path:** superset/utils/excel_streaming.py **Line:** 170:170 **Comment:** *Custom Rule: Add an explicit type annotation for this instance attribute during initialization to satisfy the type-hint requirement for relevant variables. Validate the correctness of the flagged issue. If correct, How can I resolve this? If you propose a fix, implement it and please make it concise. Once fix is implemented, also check other comments on the same PR, and ask user if the user wants to fix the rest of the comments as well. if said yes, then fetch all the comments validate the correctness and implement a minimal fix ``` </details> <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=0cfd95af9274fb70a8594072832eabb7e57131679cb118defecb0be64be58e58&reaction=like'>👍</a> | <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=0cfd95af9274fb70a8594072832eabb7e57131679cb118defecb0be64be58e58&reaction=dislike'>👎</a> ########## superset/config.py: ########## @@ -1369,6 +1369,23 @@ def sync_theme_logo_href( # note: index option should not be overridden EXCEL_EXPORT: dict[str, Any] = {} +# --------------------------------------------------- +# Dashboard "Export Data to Excel" (async, S3-backed) +# --------------------------------------------------- +# Destination S3 bucket for generated dashboard .xlsx exports. The feature is +# disabled until this is set: the export endpoint returns 501 when it is None. +EXCEL_EXPORT_S3_BUCKET: str | None = None +# Key prefix for export objects: {prefix}{dashboard_id}/{job_id}.xlsx +EXCEL_EXPORT_S3_KEY_PREFIX = "dashboard-exports/" Review Comment: **Suggestion:** Add an explicit type annotation to this new configuration constant to satisfy the type-hint requirement for new variables. [custom_rule] **Severity Level:** Minor ⚠️ <details> <summary><b>Why it matters? 🤔 </b></summary> This is a newly added Python variable that can be annotated, but it is introduced without a type hint. That matches the rule requiring type hints on new or modified Python code where applicable. </details> [](https://app.codeant.ai/fix-in-ide?tool=cursor&prompt_id=ceacec5296874e9ebe7c74f1d514ea9b&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) [](https://app.codeant.ai/fix-in-ide?tool=vscode-claude&prompt_id=ceacec5296874e9ebe7c74f1d514ea9b&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) *(Use Cmd/Ctrl + Click for best experience)* <details> <summary><b>Prompt for AI Agent 🤖 </b></summary> ```mdx This is a comment left during a code review. **Path:** superset/config.py **Line:** 1379:1379 **Comment:** *Custom Rule: Add an explicit type annotation to this new configuration constant to satisfy the type-hint requirement for new variables. Validate the correctness of the flagged issue. If correct, How can I resolve this? If you propose a fix, implement it and please make it concise. Once fix is implemented, also check other comments on the same PR, and ask user if the user wants to fix the rest of the comments as well. if said yes, then fetch all the comments validate the correctness and implement a minimal fix ``` </details> <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=9af3fe32254564b1674278b6d4d3f14ee06cfd5374fc3f303bc7e02ed32492cd&reaction=like'>👍</a> | <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=9af3fe32254564b1674278b6d4d3f14ee06cfd5374fc3f303bc7e02ed32492cd&reaction=dislike'>👎</a> ########## superset/dashboards/excel_export/layout.py: ########## @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Determine the order in which a dashboard's charts appear in its layout.""" + +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from superset.models.dashboard import Dashboard + from superset.models.slice import Slice + +CHART_TYPE = "CHART" +ROOT_ID = "ROOT_ID" + + +def _walk_chart_ids(position: dict[str, Any]) -> list[int]: + """ + Depth-first walk of a dashboard ``position_json`` returning chart ids in + visual (layout) order, including tab-nested charts. Each chart id appears + once (first occurrence wins); cycles are guarded against. + """ + if ROOT_ID not in position: + return [] + + ordered: list[int] = [] + seen_charts: set[int] = set() + visited_nodes: set[str] = set() + stack: list[str] = [ROOT_ID] + + while stack: + node_id = stack.pop() + if node_id in visited_nodes: + continue + visited_nodes.add(node_id) + + node = position.get(node_id) + if not isinstance(node, dict): + continue + + if node.get("type") == CHART_TYPE: + chart_id = node.get("meta", {}).get("chartId") + if isinstance(chart_id, int) and chart_id not in seen_charts: + seen_charts.add(chart_id) + ordered.append(chart_id) + + # Push children in reverse so they are popped in their declared order. + children = node.get("children", []) Review Comment: **Suggestion:** Add an explicit type annotation to this local variable since it is a newly introduced variable with an annotatable collection type. [custom_rule] **Severity Level:** Minor ⚠️ <details> <summary><b>Why it matters? 🤔 </b></summary> This local variable is introduced in new Python code and could be annotated with a concrete collection type. Since the rule flags new or modified code that omits type hints on annotatable variables, this is a valid violation. </details> [](https://app.codeant.ai/fix-in-ide?tool=cursor&prompt_id=739b3f6464954ec2a59c9ec4f6bf33a8&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) [](https://app.codeant.ai/fix-in-ide?tool=vscode-claude&prompt_id=739b3f6464954ec2a59c9ec4f6bf33a8&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) *(Use Cmd/Ctrl + Click for best experience)* <details> <summary><b>Prompt for AI Agent 🤖 </b></summary> ```mdx This is a comment left during a code review. **Path:** superset/dashboards/excel_export/layout.py **Line:** 62:62 **Comment:** *Custom Rule: Add an explicit type annotation to this local variable since it is a newly introduced variable with an annotatable collection type. Validate the correctness of the flagged issue. If correct, How can I resolve this? If you propose a fix, implement it and please make it concise. Once fix is implemented, also check other comments on the same PR, and ask user if the user wants to fix the rest of the comments as well. if said yes, then fetch all the comments validate the correctness and implement a minimal fix ``` </details> <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=bfe417b2d74c4e04652ed2c06d0f7e30b5dd6a7970f5698d8ee069edbb4959dd&reaction=like'>👍</a> | <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=bfe417b2d74c4e04652ed2c06d0f7e30b5dd6a7970f5698d8ee069edbb4959dd&reaction=dislike'>👎</a> ########## superset/tasks/export_dashboard_excel.py: ########## @@ -0,0 +1,246 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Celery task that exports every chart on a dashboard to a single multi-sheet +``.xlsx`` file, uploads it to S3, and emails the requesting user a pre-signed +download link. + +The task re-runs each chart's saved query context under the requesting user, +applies the live dashboard filter state, and streams the results row-by-row into +a constant-memory workbook so large dashboards never load all data at once. +""" + +from __future__ import annotations + +import logging +import os +import tempfile +from datetime import datetime, timedelta +from typing import Any + +from celery.exceptions import SoftTimeLimitExceeded +from flask import current_app, g + +from superset import db, security_manager +from superset.charts.data.dashboard_filter_context import ( + apply_dashboard_filter_context, + get_dashboard_filter_context, +) +from superset.charts.schemas import ChartDataQueryContextSchema +from superset.commands.chart.data.get_data_command import ChartDataCommand +from superset.common.chart_data import ChartDataResultFormat, ChartDataResultType +from superset.dashboards.excel_export import email +from superset.dashboards.excel_export.layout import get_charts_in_layout_order +from superset.extensions import celery_app +from superset.utils import json, s3 +from superset.utils.core import override_user +from superset.utils.excel_streaming import StreamingXlsxWriter + +logger = logging.getLogger(__name__) Review Comment: **Suggestion:** Add an explicit type annotation for this module-level variable to satisfy the type-hint requirement for relevant variables. [custom_rule] **Severity Level:** Minor ⚠️ <details> <summary><b>Why it matters? 🤔 </b></summary> This new Python module defines a module-level variable without an explicit type hint. Under the type-hint rule, this is a relevant variable that can be annotated, so the suggestion identifies a real violation. </details> [](https://app.codeant.ai/fix-in-ide?tool=cursor&prompt_id=56dbbfc66ac141189eb97d93dc5e0084&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) [](https://app.codeant.ai/fix-in-ide?tool=vscode-claude&prompt_id=56dbbfc66ac141189eb97d93dc5e0084&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) *(Use Cmd/Ctrl + Click for best experience)* <details> <summary><b>Prompt for AI Agent 🤖 </b></summary> ```mdx This is a comment left during a code review. **Path:** superset/tasks/export_dashboard_excel.py **Line:** 53:53 **Comment:** *Custom Rule: Add an explicit type annotation for this module-level variable to satisfy the type-hint requirement for relevant variables. Validate the correctness of the flagged issue. If correct, How can I resolve this? If you propose a fix, implement it and please make it concise. Once fix is implemented, also check other comments on the same PR, and ask user if the user wants to fix the rest of the comments as well. if said yes, then fetch all the comments validate the correctness and implement a minimal fix ``` </details> <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=3ddeecd138a8f3ddc1d7cfbcea227b0f34c054be2d43e228d82981ff1ae2dd07&reaction=like'>👍</a> | <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=3ddeecd138a8f3ddc1d7cfbcea227b0f34c054be2d43e228d82981ff1ae2dd07&reaction=dislike'>👎</a> ########## superset/dashboards/excel_export/layout.py: ########## @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Determine the order in which a dashboard's charts appear in its layout.""" + +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from superset.models.dashboard import Dashboard + from superset.models.slice import Slice + +CHART_TYPE = "CHART" +ROOT_ID = "ROOT_ID" Review Comment: **Suggestion:** Add explicit type annotations for these new module-level constants to satisfy the type-hint requirement for annotatable variables. [custom_rule] **Severity Level:** Minor ⚠️ <details> <summary><b>Why it matters? 🤔 </b></summary> These are new module-level string constants in a Python file and can be explicitly annotated as `str`. The custom rule requires type hints for annotatable variables, so omitting annotations here is a real violation. </details> [](https://app.codeant.ai/fix-in-ide?tool=cursor&prompt_id=16bd500e702f4946b19c7c16089f3334&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) [](https://app.codeant.ai/fix-in-ide?tool=vscode-claude&prompt_id=16bd500e702f4946b19c7c16089f3334&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) *(Use Cmd/Ctrl + Click for best experience)* <details> <summary><b>Prompt for AI Agent 🤖 </b></summary> ```mdx This is a comment left during a code review. **Path:** superset/dashboards/excel_export/layout.py **Line:** 27:28 **Comment:** *Custom Rule: Add explicit type annotations for these new module-level constants to satisfy the type-hint requirement for annotatable variables. Validate the correctness of the flagged issue. If correct, How can I resolve this? If you propose a fix, implement it and please make it concise. Once fix is implemented, also check other comments on the same PR, and ask user if the user wants to fix the rest of the comments as well. if said yes, then fetch all the comments validate the correctness and implement a minimal fix ``` </details> <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=bf0fcbd8e6ae724687b911cf5566a0d8c5d3713d19384fec3d4aac12767aef4a&reaction=like'>👍</a> | <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=bf0fcbd8e6ae724687b911cf5566a0d8c5d3713d19384fec3d4aac12767aef4a&reaction=dislike'>👎</a> -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
