codeant-ai-for-open-source[bot] commented on code in PR #41133: URL: https://github.com/apache/superset/pull/41133#discussion_r3507593464
########## superset/tasks/export_dashboard_excel.py: ########## @@ -0,0 +1,246 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Celery task that exports every chart on a dashboard to a single multi-sheet +``.xlsx`` file, uploads it to S3, and emails the requesting user a pre-signed +download link. + +The task re-runs each chart's saved query context under the requesting user, +applies the live dashboard filter state, and streams the results row-by-row into +a constant-memory workbook so large dashboards never load all data at once. +""" + +from __future__ import annotations + +import logging +import os +import tempfile +from datetime import datetime, timedelta +from typing import Any + +from celery.exceptions import SoftTimeLimitExceeded +from flask import current_app, g + +from superset import db, security_manager +from superset.charts.data.dashboard_filter_context import ( + apply_dashboard_filter_context, + get_dashboard_filter_context, +) +from superset.charts.schemas import ChartDataQueryContextSchema +from superset.commands.chart.data.get_data_command import ChartDataCommand +from superset.common.chart_data import ChartDataResultFormat, ChartDataResultType +from superset.dashboards.excel_export import email +from superset.dashboards.excel_export.layout import get_charts_in_layout_order +from superset.extensions import celery_app +from superset.utils import json, s3 +from superset.utils.core import override_user +from superset.utils.excel_streaming import StreamingXlsxWriter + +logger = logging.getLogger(__name__) + + +def _chart_label(chart: Any) -> str: + """Human-readable label for a chart in the skipped-charts list.""" + return f"{chart.id} - {chart.slice_name or ''}".strip() + + +def _record_to_row(record: dict[str, Any], colnames: list[str]) -> list[Any]: + return [record.get(col) for col in colnames] + + +def _write_chart_sheets( + writer: StreamingXlsxWriter, + chart: Any, + dashboard_id: int, + active_data_mask: dict[str, Any], +) -> None: + """ + Run a single chart's query and stream its result(s) into the workbook. + + Charts may yield more than one query (e.g. mixed-series charts); each becomes + its own sheet. Raises if the chart cannot be exported, so the caller can skip + it and note it in the email. + """ + json_body = json.loads(chart.query_context) + # Override any stale saved values: we always want full JSON results. + json_body["result_format"] = ChartDataResultFormat.JSON + json_body["result_type"] = ChartDataResultType.FULL + json_body.pop("force", None) + + filter_context = get_dashboard_filter_context( + dashboard_id=dashboard_id, + chart_id=chart.id, + active_data_mask=active_data_mask, + ) + if filter_context.extra_form_data: + apply_dashboard_filter_context(json_body, filter_context.extra_form_data) + + # Jinja macros resolve form data from g.form_data; expose the saved context. + g.form_data = json_body + + query_context = ChartDataQueryContextSchema().load(json_body) + command = ChartDataCommand(query_context) + command.validate() + result = command.run() + + for index, query in enumerate(result["queries"]): + colnames = query.get("colnames") or [] + data = query.get("data") or [] + if index == 0: + name = f"{chart.id} - {chart.slice_name or ''}" + else: + name = f"{chart.id}.{index} - {chart.slice_name or ''}" + writer.add_sheet( + name, + colnames, + (_record_to_row(record, colnames) for record in data), + ) + + +def _build_workbook( + path: str, + dashboard: Any, + active_data_mask: dict[str, Any], + job_id: str, +) -> list[str]: + """Build the workbook on disk; return the list of skipped chart labels.""" + skipped: list[str] = [] + writer = StreamingXlsxWriter(path) + try: + for chart in get_charts_in_layout_order(dashboard): + if not chart.query_context: + skipped.append(_chart_label(chart)) + continue + try: + _write_chart_sheets(writer, chart, dashboard.id, active_data_mask) + except Exception: # pylint: disable=broad-except + logger.exception( + "Skipping chart %s in dashboard export %s", chart.id, job_id + ) + skipped.append(_chart_label(chart)) + + if writer.sheet_count == 0: + writer.add_summary_sheet( + "Export Summary", + ["No chart data could be exported.", *skipped], + ) + finally: + writer.close() + return skipped + + +def _send_failure_email( + user: Any, dashboard_title: str, requested_at: datetime +) -> None: + if not (user and getattr(user, "email", None)): + return + try: + email.send_export_email( + user.email, + email.build_subject(dashboard_title, success=False), + email.build_failure_email(dashboard_title, requested_at), + ) + except Exception: # pylint: disable=broad-except + logger.exception("Failed to send export failure email") + + +@celery_app.task( + name="export_dashboard_excel", + bind=True, + soft_time_limit=600, + time_limit=660, + max_retries=0, +) +def export_dashboard_excel( + self: Any, # pylint: disable=unused-argument + dashboard_id: int, + user_id: int, + active_data_mask: dict[str, Any], + job_id: str, +) -> None: + """ + Export a dashboard's chart data to an ``.xlsx`` and email a download link. + + :param dashboard_id: The dashboard to export + :param user_id: The requesting user (the task runs with their permissions) + :param active_data_mask: Live dashboard filter state keyed by native filter id + :param job_id: Correlation id, also the Celery task id and S3 object name + """ + # pylint: disable=import-outside-toplevel + from superset.models.dashboard import Dashboard + + requested_at = datetime.utcnow() + user = security_manager.get_user_by_id(user_id) + dashboard_title = "" + tmp_path: str | None = None + + try: + with override_user(user, force=False): + dashboard = ( + db.session.query(Dashboard).filter_by(id=dashboard_id).one_or_none() + ) + if dashboard is None: + raise ValueError(f"Dashboard {dashboard_id} not found") + dashboard_title = dashboard.dashboard_title or f"Dashboard {dashboard_id}" + + file_descriptor, tmp_path = tempfile.mkstemp( + suffix=".xlsx", prefix=f"dash-export-{job_id}-" + ) + os.close(file_descriptor) + + skipped = _build_workbook(tmp_path, dashboard, active_data_mask, job_id) + + bucket = current_app.config["EXCEL_EXPORT_S3_BUCKET"] + key = ( + f"{current_app.config['EXCEL_EXPORT_S3_KEY_PREFIX']}" + f"{dashboard_id}/{job_id}.xlsx" + ) + ttl = current_app.config["EXCEL_EXPORT_LINK_TTL_SECONDS"] + + s3.upload_file_to_s3(tmp_path, bucket, key) + download_url = s3.generate_presigned_url(bucket, key, ttl) Review Comment: **Suggestion:** The presigned URL TTL is passed straight through without validating S3 limits; values above the provider cap (notably AWS 7 days) will make URL generation fail and the entire job error out. Clamp or validate the configured TTL before calling URL generation. [logic error] <details> <summary><b>Severity Level:</b> Critical 🚨</summary> ```mdx ❌ Misconfigured TTL makes all Excel exports fail at signing. ⚠️ Operators may not notice configuration cause without explicit validation. ``` </details> <details> <summary><b>Steps of Reproduction ✅ </b></summary> ```mdx 1. Configure `EXCEL_EXPORT_LINK_TTL_SECONDS` to exceed the S3 pre-signed URL maximum (for AWS S3 this is 7 days, e.g. set it to `int(timedelta(days=8).total_seconds())`) in the Superset configuration so the value is available via `current_app.config`. 2. Trigger a dashboard "Export to Excel"; `export_dashboard_excel()` in `superset/tasks/export_dashboard_excel.py:168-244` reads `ttl = current_app.config["EXCEL_EXPORT_LINK_TTL_SECONDS"]` at line 212, uploads the file with `s3.upload_file_to_s3(tmp_path, bucket, key)` at line 214, and then calls `s3.generate_presigned_url(bucket, key, ttl)` at line 215. 3. `generate_presigned_url()` in `superset/utils/s3.py:63-76` forwards `expires_in` directly to `boto3` via `_get_s3_client().generate_presigned_url(..., ExpiresIn=expires_in)` at lines 72-75; when `expires_in` exceeds the backend limit, the boto3 client raises a `ParamValidationError` (or similar) instead of returning a URL. 4. That exception propagates back into `export_dashboard_excel()`, is caught by the broad `except Exception` block at lines 240-242, which logs "Dashboard excel export %s failed", calls `_send_failure_email()` at lines 238-242, and re-raises, causing every Excel export to fail deterministically until the TTL configuration is corrected. ``` </details> [](https://app.codeant.ai/fix-in-ide?tool=cursor&prompt_id=06867507ea4a4ff29ac76d76d83e9e48&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) [](https://app.codeant.ai/fix-in-ide?tool=vscode-claude&prompt_id=06867507ea4a4ff29ac76d76d83e9e48&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) *(Use Cmd/Ctrl + Click for best experience)* <details> <summary><b>Prompt for AI Agent 🤖 </b></summary> ```mdx This is a comment left during a code review. **Path:** superset/tasks/export_dashboard_excel.py **Line:** 212:215 **Comment:** *Logic Error: The presigned URL TTL is passed straight through without validating S3 limits; values above the provider cap (notably AWS 7 days) will make URL generation fail and the entire job error out. Clamp or validate the configured TTL before calling URL generation. Validate the correctness of the flagged issue. If correct, How can I resolve this? If you propose a fix, implement it and please make it concise. Once fix is implemented, also check other comments on the same PR, and ask user if the user wants to fix the rest of the comments as well. if said yes, then fetch all the comments validate the correctness and implement a minimal fix ``` </details> <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=71d92f4904eddb888202dac1818fc1e759952d7e04db6bb7eccfd12747fef777&reaction=like'>👍</a> | <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=71d92f4904eddb888202dac1818fc1e759952d7e04db6bb7eccfd12747fef777&reaction=dislike'>👎</a> ########## superset/dashboards/excel_export/layout.py: ########## @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Determine the order in which a dashboard's charts appear in its layout.""" + +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from superset.models.dashboard import Dashboard + from superset.models.slice import Slice + +CHART_TYPE = "CHART" +ROOT_ID = "ROOT_ID" + + +def _walk_chart_ids(position: dict[str, Any]) -> list[int]: + """ + Depth-first walk of a dashboard ``position_json`` returning chart ids in + visual (layout) order, including tab-nested charts. Each chart id appears + once (first occurrence wins); cycles are guarded against. + """ + if ROOT_ID not in position: + return [] + + ordered: list[int] = [] + seen_charts: set[int] = set() + visited_nodes: set[str] = set() + stack: list[str] = [ROOT_ID] + + while stack: + node_id = stack.pop() + if node_id in visited_nodes: + continue + visited_nodes.add(node_id) + + node = position.get(node_id) + if not isinstance(node, dict): + continue + + if node.get("type") == CHART_TYPE: + chart_id = node.get("meta", {}).get("chartId") Review Comment: **Suggestion:** This assumes `meta` is always a dict; if persisted layout JSON has `meta: null`, calling `.get` on it raises and stops export. Guard `meta` with an `isinstance(..., dict)` check before reading `chartId`. [type error] <details> <summary><b>Severity Level:</b> Critical 🚨</summary> ```mdx ❌ Null meta entries can abort layout traversal during export. ⚠️ Single corrupted layout node breaks all downstream chart processing. ``` </details> <details> <summary><b>Steps of Reproduction ✅ </b></summary> ```mdx 1. Persist or import a dashboard layout where a chart node in `position_json` has `"type": "CHART"` but `"meta": null` (or another non-dict value) for that layout item. 2. Trigger "Export to Excel" for the dashboard so the Celery worker runs `export_dashboard_excel()` in `superset/tasks/export_dashboard_excel.py:168-244`, queries the `Dashboard` at lines 193-195, and calls `_build_workbook()` at line 205. 3. `_build_workbook()` invokes `get_charts_in_layout_order(dashboard)` at `export_dashboard_excel.py:124`, which calls `_walk_chart_ids(dashboard.position)` in `superset/dashboards/excel_export/layout.py:31-66` to derive chart IDs from the layout. 4. When `_walk_chart_ids()` processes the malformed chart node, the condition `if node.get("type") == CHART_TYPE:` at line 55 passes, then `chart_id = node.get("meta", {}).get("chartId")` at line 56 evaluates `node.get("meta", {})` to `None` (since `"meta"` is present but null) and attempts `.get("chartId")` on `None`, raising an `AttributeError` that aborts traversal and causes the export job to fail instead of continuing or skipping the bad node. ``` </details> [](https://app.codeant.ai/fix-in-ide?tool=cursor&prompt_id=5e2d40b99cbc4ea699a75892bcfd9d19&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) [](https://app.codeant.ai/fix-in-ide?tool=vscode-claude&prompt_id=5e2d40b99cbc4ea699a75892bcfd9d19&service=github&base_url=https%3A%2F%2Fgithub.com&org=apache&repo=apache%2Fsuperset) *(Use Cmd/Ctrl + Click for best experience)* <details> <summary><b>Prompt for AI Agent 🤖 </b></summary> ```mdx This is a comment left during a code review. **Path:** superset/dashboards/excel_export/layout.py **Line:** 56:56 **Comment:** *Type Error: This assumes `meta` is always a dict; if persisted layout JSON has `meta: null`, calling `.get` on it raises and stops export. Guard `meta` with an `isinstance(..., dict)` check before reading `chartId`. Validate the correctness of the flagged issue. If correct, How can I resolve this? If you propose a fix, implement it and please make it concise. Once fix is implemented, also check other comments on the same PR, and ask user if the user wants to fix the rest of the comments as well. if said yes, then fetch all the comments validate the correctness and implement a minimal fix ``` </details> <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=39d5a57c4926a182436115f55966d6956254a3586db2dee5f10c1992e528b38f&reaction=like'>👍</a> | <a href='https://app.codeant.ai/feedback?pr_url=https%3A%2F%2Fgithub.com%2Fapache%2Fsuperset%2Fpull%2F41133&comment_hash=39d5a57c4926a182436115f55966d6956254a3586db2dee5f10c1992e528b38f&reaction=dislike'>👎</a> -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
