aminghadersohi commented on code in PR #45348:
URL: https://github.com/apache/superset/pull/45348#discussion_r4292419673
##########
superset/sql/execution/executor.py:
##########
@@ -404,6 +410,129 @@ def execute_sql_with_cursor(
return results
+def _decode_async_results_payload(blob: bytes) -> dict[str, Any]:
+ """
+ Decode a results backend blob written by the async SQL execution task.
+
+ The task stores a zlib-compressed payload serialized either as MessagePack
+ or as UTF-8 JSON, depending on ``RESULTS_BACKEND_USE_MSGPACK``. The format
+ is detected from the payload itself (a JSON object starts with ``{``, which
+ is never the first byte of a MessagePack map), so results stay readable if
+ the setting differs between the writing worker and the reading process.
+
+ :param blob: Compressed payload read from the results backend
+ :returns: The deserialized payload
+ """
+ decompressed = cast(bytes, utils.zlib_decompress(blob, decode=False))
+ if decompressed.lstrip()[:1] == b"{":
+ return json.loads(decompressed)
+ return msgpack.loads(decompressed, raw=False)
+
+
+def _decode_async_statement_data(stmt_data: dict[str, Any]) -> pd.DataFrame |
None:
+ """
+ Convert one stored statement's data into a DataFrame.
+
+ SELECT results are Arrow IPC bytes (MessagePack) or Base64 strings (JSON).
+ Older JSON payloads contain records; DML statements store ``None``. A
+ SELECT returning no rows yields an empty DataFrame that keeps its columns,
+ matching the synchronous execution path.
+
+ :param stmt_data: Statement entry from the stored payload
+ :returns: DataFrame of the statement rows, or ``None`` for DML
+ """
+ data = stmt_data.get("data")
+ if data is None:
+ return None
+
+ if isinstance(data, str):
+ data = base64.b64decode(data)
+ if isinstance(data, (bytes, bytearray, memoryview)):
+ from superset.result_set import SupersetResultSet
+
+ table = pa.ipc.open_stream(pa.BufferReader(data)).read_all()
+ df = SupersetResultSet.convert_table_to_df(table)
+ for name, data in stmt_data.get("nested_columns_msgpack", {}).items():
+ try:
+ values = deserialize_nested_columns({name: data})[name]
+ if name not in df.columns:
+ raise ValueError("Nested sidecar has no Arrow column")
+ if not isinstance(values, list) or len(values) != len(df):
+ raise ValueError("Nested sidecar does not match Arrow row
count")
+ df[name] = values
+ except (
+ ValueError,
+ TypeError,
+ DecimalException,
+ msgpack.UnpackException,
+ OverflowError,
+ RecursionError,
+ ):
+ # Readers in a rolling deployment may not know newer type tags.
+ logger.warning(
+ "Unable to decode nested SQL column %s; using Arrow
strings",
+ name,
+ exc_info=True,
+ )
+ return df
+
+ columns = [
+ column.get("column_name", column.get("name", ""))
+ for column in stmt_data.get("columns", [])
+ ]
+ return _records_to_dataframe(data, columns)
+
+
+def _records_to_dataframe(
+ records: list[Any],
+ columns: list[Any],
+ dtypes: list[str] | None = None,
+) -> pd.DataFrame:
+ """
+ Build a DataFrame from JSON-style records.
+
+ Values are converted through Arrow, as SELECT results are on the
+ synchronous path, so integer columns with nulls keep ``None`` instead of
+ becoming ``NaN`` floats; mixed or nested values are kept as Python
+ objects. The given columns are kept even when there are no records.
+
+ :param records: Rows as dicts keyed by column name
+ :param columns: Column names, in order
+ :param dtypes: Original pandas dtypes for cached DataFrames
+ :returns: DataFrame of the records
+ """
+ if not columns or not all(isinstance(row, dict) for row in records):
+ return pd.DataFrame(records, columns=columns or None)
Review Comment:
I reproduced this with the old writer's actual cache shape (no `columns` or
`dtypes`): `1152921504606846977` became `1152921504606846976`, and `None`
became `NaN`.
Fixed in de6e8f226997599d1d4ee43864ab6d3ff8a0401f: infer column names from
record keys, then use the existing Arrow conversion. Regression:
`tests/unit_tests/sql/execution/test_async_results.py::test_legacy_cache_without_column_metadata_preserves_values`,
parameterized for `execute()` and `execute_async().get_result()`, fails with
the original helper and passes here, asserting the exact integer and `None`.
`test_decode_legacy_records_without_column_metadata` also fails before and
passes here for the shared decoder.
Siblings checked:
- superset/sql/execution/executor.py:486 — fixed in
de6e8f226997599d1d4ee43864ab6d3ff8a0401f
- superset/sql/execution/executor.py:1237 — fixed in
de6e8f226997599d1d4ee43864ab6d3ff8a0401f
- superset/sql/execution/executor.py:622 — fixed in
de6e8f226997599d1d4ee43864ab6d3ff8a0401f
- superset/sql/execution/executor.py:754 — fixed in
de6e8f226997599d1d4ee43864ab6d3ff8a0401f
- superset/sql/execution/executor.py:1448 — fixed in
de6e8f226997599d1d4ee43864ab6d3ff8a0401f
- superset/sql/execution/executor.py:483 — fixed in
de6e8f226997599d1d4ee43864ab6d3ff8a0401f
- superset/sql/execution/executor.py:1548 — fixed in
de6e8f226997599d1d4ee43864ab6d3ff8a0401f
- superset/models/core.py:1703 — fixed in
de6e8f226997599d1d4ee43864ab6d3ff8a0401f
- superset/models/core.py:1719 — fixed in
de6e8f226997599d1d4ee43864ab6d3ff8a0401f
- superset/sql/execution/executor.py:751 — not affected: dry-run handles
contain no cached records.
- superset/commands/sql_lab/results.py:126 — not affected: independent SQL
Lab JSON reader does not reconstruct these statement records.
- superset/commands/sql_lab/export.py:115 — not affected: independent SQL
Lab export reader already constructs its DataFrame with `dtype=object`.
- superset/datasource/api.py:145 — not affected: cached column values are
returned directly without pandas inference.
- superset/datasource/api.py:675 — not affected: cached compatibility
metadata is returned directly.
- superset/common/query_context_processor.py:220 — not affected: reads a
force-refresh marker, not result rows.
Search: `rg -n
'_get_from_cache|_records_to_dataframe|_decode_async_statement_data|_get_async_query_result|_create_cached_handle'
superset superset-core` and `rg -n 'results_backend\.get\(|data_cache\.get\('
superset`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
