rusackas commented on code in PR #41946:
URL: https://github.com/apache/superset/pull/41946#discussion_r3565175824
##########
tests/unit_tests/connectors/sqla/models_test.py:
##########
@@ -542,6 +542,89 @@ def
test_fetch_metadata_with_comment_field_existing_columns(
assert columns_by_name["name"].description == "Updated name description"
+def test_fetch_metadata_sets_expression_for_expanded_nested_columns(
+ mocker: MockerFixture,
+) -> None:
+ """
+ Test that fetch_metadata uses the `expression` hint provided by the db
engine
+ spec (e.g. Trino's expansion of nested `ROW` columns via `expand_rows`) to
set
+ the physical `TableColumn.expression`.
+
+ Without this, a nested column like `metadata.uuid` would have no
expression,
+ causing SQLAlchemy to render the whole dotted `column_name` as a single
quoted
+ identifier (`"metadata.uuid"`), which Trino rejects, instead of the correct
+ per-segment quoting (`"metadata"."uuid"`).
+
+ See: https://github.com/apache/superset/issues/27034
+ """
+ # Mock database
+ database = mocker.MagicMock()
+ database.get_metrics.return_value = []
+
+ # Mock db_engine_spec
+ mock_db_engine_spec = mocker.MagicMock()
+ mock_db_engine_spec.alter_new_orm_column = mocker.MagicMock()
+ database.db_engine_spec = mock_db_engine_spec
+
+ # Create table with a pre-existing (already synced) expanded column, to
also
+ # cover the "sync columns from source" (re-fetch) code path
+ table = SqlaTable(table_name="test_table_nested", database=database)
+ table.id = 1
+
+ existing_col = TableColumn(
+ column_name="metadata.uuid",
+ type="VARCHAR",
+ table=table,
+ expression="",
+ )
+ table.columns = [existing_col]
+
+ mock_columns: list[dict[str, str]] = [
+ {
+ "column_name": "metadata",
+ "type": "ROW",
+ },
+ {
+ "column_name": "metadata.uuid",
+ "type": "VARCHAR",
+ "expression": '"metadata"."uuid"',
+ "query_as": '"metadata"."uuid" AS "metadata.uuid"',
+ },
+ ]
+
+ mock_session = mocker.patch("superset.connectors.sqla.models.db.session")
+ mock_session.query.return_value.filter.return_value.all.return_value = [
+ existing_col
+ ]
+ mocker.patch.object(table, "external_metadata", return_value=mock_columns)
+ mocker.patch(
+ "superset.connectors.sqla.models.config", {"SQLA_TABLE_MUTATOR":
lambda x: None}
+ )
+
+ table.fetch_metadata()
+
+ columns_by_name = {col.column_name: col for col in table.columns}
+ assert len(table.columns) == len(mock_columns)
+ assert not columns_by_name["metadata"].expression
+ assert columns_by_name["metadata.uuid"].expression == '"metadata"."uuid"'
+
+ # Re-run fetch_metadata a second time to simulate re-syncing columns from
+ # source. The previously synced physical column now carries a truthy
+ # `expression`, which must not cause it to be duplicated in `self.columns`.
+ mock_session_2 = mocker.patch("superset.connectors.sqla.models.db.session")
+ mock_session_2.query.return_value.filter.return_value.all.return_value =
list(
+ table.columns
+ )
+ mocker.patch.object(table, "external_metadata", return_value=mock_columns)
+
+ table.fetch_metadata()
+
+ assert len(table.columns) == len(mock_columns)
+ columns_by_name = {col.column_name: col for col in table.columns}
Review Comment:
Added the type annotation.
##########
superset/connectors/sqla/models.py:
##########
@@ -2032,8 +2039,13 @@ def fetch_metadata(self) -> MetadataResult:
if not any_date_col and new_column.is_temporal:
any_date_col = col["column_name"]
- # add back calculated (virtual) columns
- columns.extend([col for col in old_columns if col.expression])
+ # Add back calculated (virtual) columns, i.e. those that weren't
matched
+ # against `new_columns` above and are thus still present in
+ # `old_columns_by_name`. Columns that were matched are already
appended to
+ # `columns` in the loop above, and re-adding them here (e.g. via
`old_columns`)
+ # would duplicate any synced physical column that also carries a truthy
+ # `expression`, such as Trino's expanded nested `ROW` fields.
+ columns.extend([col for col in old_columns_by_name.values() if
col.expression])
Review Comment:
This heuristic (truthy `.expression` means virtual column) predates this PR
though. Any physical column with a manually-added expression already has the
same stale-retention issue if it disappears from source, since there's no
persisted flag distinguishing physical vs. virtual columns. Fixing that
properly needs a schema change, bigger than this quoting fix.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
