aminghadersohi commented on code in PR #45348:
URL: https://github.com/apache/superset/pull/45348#discussion_r4286276750
##########
superset/sql/execution/executor.py:
##########
@@ -404,6 +405,94 @@ def execute_sql_with_cursor(
return results
+def _decode_async_results_payload(blob: bytes) -> dict[str, Any]:
+ """
+ Decode a results backend blob written by the async SQL execution task.
+
+ The task stores a zlib-compressed payload serialized either as MessagePack
+ or as UTF-8 JSON, depending on ``RESULTS_BACKEND_USE_MSGPACK``. The format
+ is detected from the payload itself (a JSON object starts with ``{``, which
+ is never the first byte of a MessagePack map), so results stay readable if
+ the setting differs between the writing worker and the reading process.
+
+ :param blob: Compressed payload read from the results backend
+ :returns: The deserialized payload
+ """
+ import msgpack
+
+ from superset.utils import json
+
+ decompressed = cast(bytes, utils.zlib_decompress(blob, decode=False))
+ if decompressed.lstrip()[:1] == b"{":
+ return json.loads(decompressed)
+ return msgpack.loads(decompressed, raw=False)
+
+
+def _decode_async_statement_data(stmt_data: dict[str, Any]) -> pd.DataFrame |
None:
+ """
+ Convert one stored statement's data into a DataFrame.
+
+ SELECT results are stored as Arrow IPC bytes (MessagePack payloads) or as
+ a list of records (JSON payloads); DML statements store ``None``. A
+ SELECT returning no rows yields an empty DataFrame that keeps its columns,
+ matching the synchronous execution path.
+
+ :param stmt_data: Statement entry from the stored payload
+ :returns: DataFrame of the statement rows, or ``None`` for DML
+ """
+ import pyarrow as pa
+
+ data = stmt_data.get("data")
+ if data is None:
+ return None
+
+ if isinstance(data, (bytes, bytearray, memoryview)):
+ from superset.result_set import SupersetResultSet
+
+ table = pa.ipc.open_stream(pa.BufferReader(data)).read_all()
+ return SupersetResultSet.convert_table_to_df(table)
+
+ columns = [
+ column.get("column_name", column.get("name", ""))
+ for column in stmt_data.get("columns", [])
+ ]
+ return _records_to_dataframe(data, columns)
+
+
+def _records_to_dataframe(records: list[Any], columns: list[Any]) ->
pd.DataFrame:
+ """
+ Build a DataFrame from JSON-style records.
+
+ Values are converted through Arrow, as SELECT results are on the
+ synchronous path, so integer columns with nulls keep ``None`` instead of
+ becoming ``NaN`` floats; mixed or nested values are kept as Python
+ objects. The given columns are kept even when there are no records.
+
+ :param records: Rows as dicts keyed by column name
+ :param columns: Column names, in order
+ :returns: DataFrame of the records
+ """
+ import pandas as pd
+ import pyarrow as pa
+
+ if not columns or not all(isinstance(row, dict) for row in records):
+ return pd.DataFrame(records, columns=columns or None)
+
+ def to_series(values: list[Any]) -> pd.Series:
+ try:
+ array = pa.array(values)
+ except (pa.ArrowInvalid, pa.ArrowTypeError):
+ return pd.Series(values, dtype=object)
+ if pa.types.is_nested(array.type):
+ return pd.Series(values, dtype=object)
+ return array.to_pandas(integer_object_nulls=True)
Review Comment:
I need to correct my earlier reply: it covered dates/timestamps but omitted
TIME/INTERVAL, large integers, Decimal, binary values, and timezone-aware cache
dtypes. Those cases are fixed for newly written results in
c310e0254be44e31cb44d5b398273319b253b829. The writer retains the complete Arrow
schema, stores binary values as Base64 and intervals as integer ticks, and the
cache retains original pandas dtypes.
Regression tests:
`tests/unit_tests/sql/execution/test_async_results.py::test_async_scalar_types_match_sync`
and `::test_timezone_cache_hit_dtype_matches_sync`. They compare values and
dtypes against synchronous execution through the real Celery writer and handle
reader, including nulls, negative intervals, non-UTF-8 bytes, both codecs, and
codec-setting changes. With the two source files restored to
b522f99c25bf85e7f639d95db73aa1b90bff7855, `pytest
tests/unit_tests/sql/execution/test_async_results.py -q -m 'not manual and not
llm_compliance'` produced **33 failed, 59 passed**; at
c310e0254be44e31cb44d5b398273319b253b829, it produced **92 passed**.
Compatibility tests cover blobs without schema, the older temporal-only schema,
caches without dtype metadata, and nested JSON.
With the source at master, the same tests produced **89 failed, 3 passed**.
The three `test_async_result_non_success_does_not_read_backend` cases are
useful guards; they pass on master and do not pin this fix. `pytest
tests/unit_tests/sql/ -q -m 'not manual and not llm_compliance'`: **2398
passed**. Staged-file `pre-commit run`: passed, including mypy.
Siblings checked:
- superset/sql/execution/celery_task.py:148 — fixed in
c310e0254be44e31cb44d5b398273319b253b829
- superset/sql/execution/celery_task.py:329 — not affected: its records are
enriched by the sole caller, _finalize_successful_query, before storage; the
round-trip regressions exercise both functions
- superset/sql/execution/celery_task.py:250 — not affected: the outer
backend serializer/compressor transports the schema and encoded values unchanged
- superset/sql/execution/celery_task.py:524 — not affected: task success
delegates to the tested finalizer; DML/empty/multi-statement guards also pass
- superset/sql/execution/executor.py:409 — not affected: outer codec
detection transports metadata unchanged; all four writer/reader codec-setting
combinations pass
- superset/sql/execution/executor.py:432 — fixed in
c310e0254be44e31cb44d5b398273319b253b829
- superset/sql/execution/executor.py:469 — fixed in
c310e0254be44e31cb44d5b398273319b253b829
- superset/sql/execution/executor.py:500 — fixed in
c310e0254be44e31cb44d5b398273319b253b829
- superset/sql/execution/executor.py:1212 — fixed in
c310e0254be44e31cb44d5b398273319b253b829
- superset/sql/execution/executor.py:1261 — fixed in
c310e0254be44e31cb44d5b398273319b253b829
- superset/sql/execution/executor.py:1433 — not affected: async handles
delegate to the same result reader exercised by the regressions
- superset/sql/execution/executor.py:1445 — not affected: cached handles
return the reconstructed result; the timezone regression exercises execute and
execute_async cache hits
- superset/models/core.py:1689 — not affected: Database.execute and
execute_async directly delegate to SQLExecutor
- superset/result_set.py:334 — not affected: synchronous and restored Arrow
tables use the same DataFrame conversion
- superset/sql_lab.py:392 — not affected: shared outer serialization does
not interpret statement schema metadata
- superset/commands/sql_lab/results.py:158 — not affected: SQL Lab consumes
the separate top-level data/columns payload, not Python StatementResult
DataFrames
- superset/commands/sql_lab/export.py:106 — not affected: the export path
consumes that separate SQL Lab payload and CSV representation
- superset/views/utils.py:650 — not affected: the legacy SQL Lab reader
intentionally returns JSON records, not typed statement DataFrames
Remaining: older JSON blobs without full schema metadata retain previously
serialized strings or lossy binary representations; rerun those queries to
recover typed results. Older cache entries without dtype metadata still load
using inferred dtypes until refreshed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
