This is an automated email from the ASF dual-hosted git repository. rusackas pushed a commit to branch fix/databricks-spark-boolean-coercion-dry in repository https://gitbox.apache.org/repos/asf/superset.git
commit 75348c73c6f21d0f226ad009f64ae05fda261193 Author: Evan Rusackas <[email protected]> AuthorDate: Sun Oct 11 22:45:40 2026 +0100 refactor(db_engine_specs): dedupe boolean-coercion logic for Databricks/Spark coerce_boolean_for_sql and use_equality_for_boolean_filters were defined identically three times (DatabricksBaseEngineSpec, DatabricksHiveEngineSpec, SparkEngineSpec) after #44355. Hoist them into a shared DatabricksBooleanEqualityMixin in hive.py so a future change to the coercion can't drift across copies. Co-Authored-By: Evan Rusackas <[email protected]> Co-Authored-By: Claude Sonnet 5 <[email protected]> --- superset/db_engine_specs/databricks.py | 41 ++++------------------------------ superset/db_engine_specs/hive.py | 18 ++++++++++++++- superset/db_engine_specs/spark.py | 20 ++--------------- 3 files changed, 23 insertions(+), 56 deletions(-) diff --git a/superset/db_engine_specs/databricks.py b/superset/db_engine_specs/databricks.py index 1fa207e8ba4..e14e2152ccc 100644 --- a/superset/db_engine_specs/databricks.py +++ b/superset/db_engine_specs/databricks.py @@ -28,7 +28,7 @@ from flask import g from flask_babel import gettext as __ from marshmallow import fields, Schema from marshmallow.validate import Range -from sqlalchemy import false, text, true, types +from sqlalchemy import text, types from sqlalchemy.engine.default import DefaultDialect from sqlalchemy.engine.reflection import Inspector from sqlalchemy.engine.url import URL @@ -40,7 +40,7 @@ from superset.db_engine_specs.base import ( BasicParametersMixin, DatabaseCategory, ) -from superset.db_engine_specs.hive import HiveEngineSpec +from superset.db_engine_specs.hive import DatabricksBooleanEqualityMixin, HiveEngineSpec from superset.errors import ErrorLevel, SupersetError, SupersetErrorType from superset.exceptions import OAuth2Error from superset.utils import json @@ -250,7 +250,7 @@ time_grain_expressions: dict[str | None, str] = { } -class DatabricksBaseEngineSpec(BaseEngineSpec): +class DatabricksBaseEngineSpec(DatabricksBooleanEqualityMixin, BaseEngineSpec): """Base engine specification for Databricks flavors and connection methods.""" _time_grain_expressions = time_grain_expressions @@ -260,27 +260,6 @@ class DatabricksBaseEngineSpec(BaseEngineSpec): identifier_quote_start: str = "`" identifier_quote_end: str = "`" - # Databricks SQL applies strict type checking: comparing a boolean column - # against an integer literal (or a bound Python boolean that the - # Hive/Databricks driver renders as 0/1) fails with - # DATATYPE_MISMATCH.DATA_DIFF_TYPES (see #36765). Enabling equality - # operators ensures boolean filters compile as 'col = true' / 'col = false'. - use_equality_for_boolean_filters: bool = True - - @classmethod - def coerce_boolean_for_sql(cls, value: bool | None) -> Any: - """ - Render Python booleans as explicit boolean keywords. - - The Hive/Databricks SQLAlchemy dialects render Python booleans as - integer literals (0/1) in some binding paths, so emit sqlalchemy - true()/false() instead: they always compile to the TRUE/FALSE keywords - and never reach the driver as bind parameters (#36765). - """ - if value is None: - return None - return true() if value else false() - @classmethod def convert_dttm( cls, target_type: str, dttm: datetime, db_extra: dict[str, Any] | None = None @@ -1167,7 +1146,7 @@ class DatabricksODBCEngineSpec(DatabricksBaseEngineSpec): # ambiguous-driver fallback in `get_engine_spec` picks the first backend-matching # spec by definition order, and this spec's inherited `HiveEngineSpec.execute()` # crashes on the `databricks.sql.client.Cursor` used by unrecognized-driver URIs. -class DatabricksHiveEngineSpec(HiveEngineSpec): +class DatabricksHiveEngineSpec(DatabricksBooleanEqualityMixin, HiveEngineSpec): """Databricks engine spec using Hive connector for Interactive Clusters.""" engine_name = "Databricks Interactive Cluster" @@ -1205,18 +1184,6 @@ class DatabricksHiveEngineSpec(HiveEngineSpec): # consolidates all Databricks connection methods. This spec exists for # backwards compatibility with Interactive Cluster connections. - # Databricks Interactive Clusters apply the same strict type checking as - # Databricks SQL, so boolean filters need equality against boolean keywords. - use_equality_for_boolean_filters: bool = True - - @classmethod - def coerce_boolean_for_sql(cls, value: bool | None) -> Any: - # Same rationale as DatabricksBaseEngineSpec: the Hive driver renders - # Python booleans as 0/1 in some binding paths (#36765). - if value is None: - return None - return true() if value else false() - _show_functions_column = "function" _time_grain_expressions = time_grain_expressions diff --git a/superset/db_engine_specs/hive.py b/superset/db_engine_specs/hive.py index be93d0de3d7..ebc8ff7854c 100644 --- a/superset/db_engine_specs/hive.py +++ b/superset/db_engine_specs/hive.py @@ -30,7 +30,7 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq from flask import current_app as app, g -from sqlalchemy import Column, text, types +from sqlalchemy import Column, false, text, true, types from sqlalchemy.engine.interfaces import Dialect from sqlalchemy.engine.reflection import Inspector from sqlalchemy.engine.url import URL @@ -85,6 +85,22 @@ def upload_to_s3(filename: str, upload_prefix: str, table: Table) -> str: return location +class DatabricksBooleanEqualityMixin: + """ + Shared boolean-filter coercion for the Databricks and Spark SQL specs, + whose dialects render bound Python booleans as integer literals in some + binding paths and apply strict type checking to the result (#36765). + """ + + use_equality_for_boolean_filters: bool = True + + @classmethod + def coerce_boolean_for_sql(cls, value: bool | None) -> Any: + if value is None: + return None + return true() if value else false() + + class HiveEngineSpec(PrestoEngineSpec): """Reuses PrestoEngineSpec functionality.""" diff --git a/superset/db_engine_specs/spark.py b/superset/db_engine_specs/spark.py index 1fa46b359c5..4b72bd62985 100644 --- a/superset/db_engine_specs/spark.py +++ b/superset/db_engine_specs/spark.py @@ -16,14 +16,11 @@ # under the License. from __future__ import annotations -from typing import Any - -from sqlalchemy import false, true from sqlalchemy.dialects import registry from superset.constants import TimeGrain from superset.db_engine_specs.base import DatabaseCategory -from superset.db_engine_specs.hive import HiveEngineSpec +from superset.db_engine_specs.hive import DatabricksBooleanEqualityMixin, HiveEngineSpec time_grain_expressions: dict[str | None, str] = { None: "{col}", @@ -44,7 +41,7 @@ time_grain_expressions: dict[str | None, str] = { } -class SparkEngineSpec(HiveEngineSpec): +class SparkEngineSpec(DatabricksBooleanEqualityMixin, HiveEngineSpec): """Engine specification for Apache Spark SQL.""" engine = "spark" @@ -52,19 +49,6 @@ class SparkEngineSpec(HiveEngineSpec): _time_grain_expressions = time_grain_expressions engine_name = "Apache Spark SQL" - # Spark SQL shares syntax rules with Databricks SQL; equality comparisons - # are required for boolean filters to avoid type mismatch errors (see #36765). - use_equality_for_boolean_filters: bool = True - - @classmethod - def coerce_boolean_for_sql(cls, value: bool | None) -> Any: - # The pyhive driver renders Python booleans as 0/1 in some binding - # paths, and Spark SQL applies the same strict type checking as - # Databricks SQL (#36765). - if value is None: - return None - return true() if value else false() - metadata = { "description": "Apache Spark SQL is a module for structured data processing.", "logo": "apache-spark.png",
