aminghadersohi commented on code in PR #45348:
URL: https://github.com/apache/superset/pull/45348#discussion_r4285610059
##########
superset/sql/execution/executor.py:
##########
@@ -404,6 +405,94 @@ def execute_sql_with_cursor(
return results
+def _decode_async_results_payload(blob: bytes) -> dict[str, Any]:
+ """
+ Decode a results backend blob written by the async SQL execution task.
+
+ The task stores a zlib-compressed payload serialized either as MessagePack
+ or as UTF-8 JSON, depending on ``RESULTS_BACKEND_USE_MSGPACK``. The format
+ is detected from the payload itself (a JSON object starts with ``{``, which
+ is never the first byte of a MessagePack map), so results stay readable if
+ the setting differs between the writing worker and the reading process.
+
+ :param blob: Compressed payload read from the results backend
+ :returns: The deserialized payload
+ """
+ import msgpack
+
+ from superset.utils import json
+
+ decompressed = cast(bytes, utils.zlib_decompress(blob, decode=False))
+ if decompressed.lstrip()[:1] == b"{":
+ return json.loads(decompressed)
+ return msgpack.loads(decompressed, raw=False)
+
+
+def _decode_async_statement_data(stmt_data: dict[str, Any]) -> pd.DataFrame |
None:
+ """
+ Convert one stored statement's data into a DataFrame.
+
+ SELECT results are stored as Arrow IPC bytes (MessagePack payloads) or as
+ a list of records (JSON payloads); DML statements store ``None``. A
+ SELECT returning no rows yields an empty DataFrame that keeps its columns,
+ matching the synchronous execution path.
+
+ :param stmt_data: Statement entry from the stored payload
+ :returns: DataFrame of the statement rows, or ``None`` for DML
+ """
+ import pyarrow as pa
+
+ data = stmt_data.get("data")
+ if data is None:
+ return None
+
+ if isinstance(data, (bytes, bytearray, memoryview)):
+ from superset.result_set import SupersetResultSet
+
+ table = pa.ipc.open_stream(pa.BufferReader(data)).read_all()
+ return SupersetResultSet.convert_table_to_df(table)
+
+ columns = [
+ column.get("column_name", column.get("name", ""))
+ for column in stmt_data.get("columns", [])
+ ]
+ return _records_to_dataframe(data, columns)
+
+
+def _records_to_dataframe(records: list[Any], columns: list[Any]) ->
pd.DataFrame:
+ """
+ Build a DataFrame from JSON-style records.
+
+ Values are converted through Arrow, as SELECT results are on the
+ synchronous path, so integer columns with nulls keep ``None`` instead of
+ becoming ``NaN`` floats; mixed or nested values are kept as Python
+ objects. The given columns are kept even when there are no records.
+
+ :param records: Rows as dicts keyed by column name
+ :param columns: Column names, in order
+ :returns: DataFrame of the records
+ """
+ import pandas as pd
+ import pyarrow as pa
+
+ if not columns or not all(isinstance(row, dict) for row in records):
+ return pd.DataFrame(records, columns=columns or None)
+
+ def to_series(values: list[Any]) -> pd.Series:
+ try:
+ array = pa.array(values)
+ except (pa.ArrowInvalid, pa.ArrowTypeError):
+ return pd.Series(values, dtype=object)
+ if pa.types.is_nested(array.type):
+ return pd.Series(values, dtype=object)
+ return array.to_pandas(integer_object_nulls=True)
Review Comment:
Confirmed and fixed for newly written results in
b522f99c25bf85e7f639d95db73aa1b90bff7855. The writer retains the Arrow
date/timestamp schema and the reader restores those types without converting
date-looking text. I also normalized cached NaT values before Arrow conversion.
Regression tests:
`tests/unit_tests/sql/execution/test_async_results.py::test_async_temporal_results_match_sync`
and `::test_temporal_cache_hit_matches_sync`. With the source reverted to
f1013eb00bb30ea3d6de0972c575cd9ca6c47eaa, `pytest
tests/unit_tests/sql/execution/test_async_results.py -q -k temporal` produced
**3 failed, 2 passed** (both JSON cases: object versus datetime64[us]; cache:
NaTType does not support utcoffset). With
b522f99c25bf85e7f639d95db73aa1b90bff7855, the same command produced **5
passed**. Both MessagePack cases match synchronous execution, including
timezone-aware timestamps, dates, nulls, and date-looking text. `pytest
tests/unit_tests/sql/ -q -m 'not manual and not llm_compliance'`: **2328
passed**; staged-file pre-commit: passed.
Siblings checked:
- superset/sql/execution/celery_task.py:148 — fixed in
b522f99c25bf85e7f639d95db73aa1b90bff7855
- superset/sql/execution/executor.py:432 — fixed in
b522f99c25bf85e7f639d95db73aa1b90bff7855
- superset/sql/execution/executor.py:490 — fixed in
b522f99c25bf85e7f639d95db73aa1b90bff7855
- superset/sql/execution/executor.py:1180 — fixed in
b522f99c25bf85e7f639d95db73aa1b90bff7855
- superset/sql/execution/executor.py:1370 — not affected: the async handle
delegates to the same result reader covered by the regression
- superset/sql/execution/executor.py:1410 — not affected: cached handles
return the reconstructed cached result, covered by the cache regression
- superset/sql/execution/executor.py:448 — not affected: MessagePack uses
Arrow IPC and already retains temporal types; both codec-setting combinations
pass
- superset/commands/sql_lab/results.py — not affected: SQL Lab returns JSON
via superset/views/utils.py::_deserialize_results_payload, not the Python
statement DataFrame contract
- superset/commands/sql_lab/export.py — not affected: uses the separate SQL
Lab top-level data/columns payload and CSV representation, not this statement
reader
- superset/views/utils.py:650 — not affected: the legacy SQL Lab JSON reader
intentionally returns serialized strings and does not consume statement
temporal_schema metadata
Remaining: older JSON blobs without temporal schema metadata retain strings
until their queries are rerun; this compatibility limitation is documented in
docs/admin_docs/configuration/async-queries-celery.mdx.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
