This is an automated email from the ASF dual-hosted git repository. rusackas pushed a commit to branch chore/sqla2-driver-compat-prep in repository https://gitbox.apache.org/repos/asf/superset.git
commit 72a7e06dd84a02f04e9f1cb683dc7e92f3ccc0a8 Author: Claude Code <[email protected]> AuthorDate: Tue Jul 28 10:11:56 2026 -0700 chore(sqla2): bump SQLAlchemy-dependent drivers to dual-compat versions Prep work for the SQLAlchemy 1.4 -> 2.0 core bump (discussion #40273, step 6). Every SQLAlchemy-dependent package Superset ships or offers as an optional extra gets bumped to the newest release that supports both SQLAlchemy 1.4 and 2.0, so each bump can be merged and tested independently under today's SQLAlchemy 1.4 pin before the core bump lands as its own final PR. - flask-sqlalchemy: pin explicitly (was purely transitive via flask-appbuilder, resolving to 2.5.1, 1.4-only) at >=3.0.5,<3.1, the last release before 3.1.0 drops 1.4 support. flask-appbuilder and marshmallow-sqlalchemy are already dual-compatible as pinned, no change needed. - sqlalchemy-bigquery: 1.17.0 -> 1.17.1 (patch; the project was archived 2026-05-16, likely its final release). - sqlalchemy-cratedb: 0.41.0 -> 0.43.1. - sqlalchemy-drill: drop an artificial <2 SQLAlchemy-version ceiling upstream never had (1.1.10 already supports 2.0 via import_dbapi). - firebolt-sqlalchemy: 1.0.0 -> 1.1.2. - sqlalchemy-ocient: 1.0.0 -> 3.0.0 (lower confidence: closed-source vendor package, no public changelog to verify against). - snowflake-sqlalchemy floor was already bumped independently by dependabot (#42418) to the same target this research landed on. Five extras (risingwave, exasol, firebird, redshift, dremio) have NO release supporting both SQLAlchemy versions -- version numbers cut straight from 1.4-only to 2.0-only with no overlap. Pinned each to its last 1.4-only version with a comment noting the 2.0 target to bump to once the core sqlalchemy bump lands, not before. Four extras have no SQLAlchemy 2.0 support anywhere in their ecosystem today and are flagged via a new `known_incompatibilities` metadata field (surfaced through the generated database docs): kusto, aurora-data-api (including our own dormant preset-io fork), solr, and d1. sqlalchemy-vertica-python was initially suspected to be a fifth, but turned out to already be dual-compatible (v0.6.3 already ships `import_dbapi`/`text()` wrapping) once the actual merged PR was checked instead of relying on PyPI metadata alone. Verified locally: full dependency resolution succeeds, the app boots and creates all tables cleanly under SQLAlchemy 1.4.54 + Flask-SQLAlchemy 3.0.5, 483/487 relevant unit tests pass (the 4 failures are a pre-existing local mysqlclient linkage issue, unrelated to this change), and the new metadata flows through generate_yaml_docs() end-to-end for all four flagged engines. --- pyproject.toml | 64 +++++++++++++++++++++++++++++++----- requirements/base.txt | 3 +- requirements/development.txt | 5 +-- superset/db_engine_specs/base.py | 15 +++++++++ superset/db_engine_specs/d1.py | 11 +++++++ superset/db_engine_specs/kusto.py | 11 +++++++ superset/db_engine_specs/mysql.py | 16 +++++++++ superset/db_engine_specs/postgres.py | 16 +++++++++ superset/db_engine_specs/solr.py | 11 +++++++ 9 files changed, 140 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 37590322a9d..428f88cc4b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,11 @@ dependencies = [ "flask-login>=0.6.0, < 1.0", "flask-migrate>=4.1.0, <5.0", "flask-session>=0.4.0, <1.0", + # Transitive via flask-appbuilder, pinned explicitly to the last release + # that supports both SQLAlchemy 1.4 and 2.0 (3.1.0 drops 1.4). Prep for + # the SQLAlchemy 2.0 migration (see discussion #40273); step 6 will lift + # this pin once the core sqlalchemy bump lands. + "flask-sqlalchemy>=3.0.5, <3.1", "flask-wtf>=1.3.0, <2.0", "geopy", "greenlet<=3.5.3, >=3.5.3", @@ -123,15 +128,25 @@ dependencies = [ [project.optional-dependencies] athena = ["pyathena[pandas]>=2, <4"] +# No SQLAlchemy 2.0 support anywhere in this dialect's ecosystem today: our +# own preset-io/sqlalchemy-aurora-data-api fork is dead since 2021, and the +# more active community fork (cloud-utils/sqlalchemy-aurora-data-api) has an +# unresolved SQLAlchemy 2.0 break (upstream issue #43). See +# superset/db_engine_specs/aurora.py's known_incompatibilities metadata. aurora-data-api = ["preset-sqlalchemy-aurora-data-api>=0.2.8,<0.3"] bigquery = [ "pandas-gbq>=0.35.0", - "sqlalchemy-bigquery>=1.17.0", + # 1.17.1 is likely the final release: googleapis/python-bigquery-sqlalchemy + # was archived 2026-05-16. Both 1.17.0 and 1.17.1 support SQLAlchemy 1.4/2.0. + "sqlalchemy-bigquery>=1.17.1", "google-cloud-bigquery>=3.42.2", ] clickhouse = ["clickhouse-connect>=1.4.2, <2.0"] cockroachdb = ["cockroachdb>=0.3.5, <0.4"] -crate = ["sqlalchemy-cratedb>=0.41.0, <1"] +crate = ["sqlalchemy-cratedb>=0.43.1, <1"] +# sqlalchemy-d1 is very young (single release, Nov 2025); no SQLAlchemy 2.0 +# signal either way. See superset/db_engine_specs/d1.py's +# known_incompatibilities metadata. d1 = [ "superset-engine-d1>=0.1.0", "sqlalchemy-d1>=0.1.0", @@ -145,14 +160,27 @@ databricks = [ datafusion = ["flightsql-dbapi>=0.2.2, <0.3"] db2 = ["ibm-db-sa<=0.4.4, >=0.4.4"] denodo = ["denodo-sqlalchemy>=2.0.5,<2.1.0"] -dremio = ["sqlalchemy-dremio>=1.2.1, <4"] -drill = ["sqlalchemy-drill>=1.1.10, <2"] +# sqlalchemy-dremio 3.0.5+ hard-pins sqlalchemy~=2.0.41, dropping 1.4; 3.0.4 +# is the last dual-compat release. Capped below 3.0.5 for now; widen back to +# <4 in lockstep with Superset's own SQLAlchemy 2.0 core bump (discussion +# #40273), not before. +dremio = ["sqlalchemy-dremio>=1.2.1, <3.0.5"] +# <2 was an artificial ceiling; upstream has no SQLAlchemy version cap and +# 1.1.10 already supports SQLAlchemy 2.0 (added `import_dbapi` in 1.1.7). +drill = ["sqlalchemy-drill>=1.1.10, <3"] druid = ["pydruid>=0.6.5,<0.7"] duckdb = ["duckdb>=1.5.4,<2", "duckdb-engine>=0.17.0"] dynamodb = ["pydynamodb>=0.8.2"] +# Effectively unmaintained (only dependabot bumps since 2024); hard-pinned to +# SQLAlchemy ~1.4.7 upstream, no SQLAlchemy 2.0 work. See +# superset/db_engine_specs/solr.py's known_incompatibilities metadata. solr = ["sqlalchemy-solr >= 0.2.4.3"] elasticsearch = ["elasticsearch-dbapi>=0.2.13, <0.3.0"] -exasol = ["sqlalchemy-exasol>=2.4.0, <8.0"] +# sqlalchemy-exasol cuts hard from SQLAlchemy 1.4-only (<6.0.0) to 2.0-only +# (>=6.0.0) with no dual-compat release. Capped below 6.0.0 for now; bump to +# >=6.0.0,<8.0 in lockstep with Superset's own SQLAlchemy 2.0 core bump +# (discussion #40273), not before. +exasol = ["sqlalchemy-exasol>=2.4.0, <6.0.0"] excel = ["xlrd>=2.0.2, <2.1"] # Async dashboard "Export Data/Images to Excel": uploads the workbook to S3 and # emails a pre-signed link. boto3 is imported lazily by superset.utils.s3, so @@ -165,8 +193,12 @@ fastmcp = [ # heuristic that under-counts JSON-heavy MCP responses. "tiktoken>=0.13.0,<1.0", ] -firebird = ["sqlalchemy-firebird>=0.8.0, <2.2"] -firebolt = ["firebolt-sqlalchemy>=1.0.0, <2"] +# sqlalchemy-firebird >=2.0.0 unconditionally requires SQLAlchemy 2.0 on +# Python >=3.8 (which covers Superset's >=3.11 floor), with no dual-compat +# release. Capped below 2.0.0 for now; bump to >=2.2.0 in lockstep with +# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before. +firebird = ["sqlalchemy-firebird>=0.8.0, <2.0.0"] +firebolt = ["firebolt-sqlalchemy>=1.1.2, <2"] gevent = ["gevent>=26.4.0"] gsheets = ["shillelagh[gsheetsapi]>=1.4.4, <2"] hana = ["hdbcli==2.29.25", "sqlalchemy_hana==3.0.3"] @@ -177,6 +209,9 @@ hive = [ "thrift_sasl>=0.4.3, < 1.0.0", ] impala = ["impyla>=0.24.0, <0.25"] +# Actively maintained upstream, but setup.py on main hard-pins +# sqlalchemy==1.4.*, no SQLAlchemy 2.0 work yet. See +# superset/db_engine_specs/kusto.py's known_incompatibilities metadata. kusto = ["sqlalchemy-kusto>=3.1.2, <4"] kylin = ["kylinpy>=2.8.4, <2.9"] mssql = ["pymssql>=2.3.13, <3"] @@ -184,7 +219,10 @@ mssql = ["pymssql>=2.3.13, <3"] motherduck = ["apache-superset[duckdb]"] mysql = ["mysqlclient>=2.2.8, <3"] ocient = [ - "sqlalchemy-ocient>=1.0.0, <4", + # Closed-source vendor package with no public changelog; permissive + # unpinned sqlalchemy>=1.4 declared, but SQLAlchemy 2.0 support is + # unverified. Lower confidence than the other bumps in this PR. + "sqlalchemy-ocient>=3.0.0, <4", "pyocient>=1.0.15, <4", "shapely", "geojson", @@ -197,8 +235,16 @@ postgres = ["psycopg2-binary==2.9.12"] presto = ["pyhive[presto]>=0.6.5"] trino = ["trino>=0.338.0"] prophet = ["prophet>=1.1.6, <2"] +# sqlalchemy-redshift cuts hard from SQLAlchemy 1.4-only (0.8.x) to 2.0-only +# (>=1.0.0) with no dual-compat release; the existing <0.9 ceiling already +# keeps this on the 1.4-only line. Bump to >=1.0.0 in lockstep with +# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before. redshift = ["sqlalchemy-redshift>=0.8.1, <0.9"] -risingwave = ["sqlalchemy-risingwave"] +# No release of sqlalchemy-risingwave has ever supported both SQLAlchemy 1.4 +# and 2.0 (version numbers don't track SQLAlchemy compat monotonically); pin +# to the newest 1.4-only release for now. Bump to >=2.0.0 in lockstep with +# Superset's own SQLAlchemy 2.0 core bump (discussion #40273), not before. +risingwave = ["sqlalchemy-risingwave>=1.4.1, <2.0.0"] shillelagh = ["shillelagh[all]>=1.4.4, <2"] singlestore = ["sqlalchemy-singlestoredb>=1.2.1, <2"] snowflake = ["snowflake-sqlalchemy>=1.11.0, <2"] diff --git a/requirements/base.txt b/requirements/base.txt index cbd57f5fa27..51056200784 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -148,8 +148,9 @@ flask-migrate==4.1.0 # via apache-superset (pyproject.toml) flask-session==0.8.0 # via apache-superset (pyproject.toml) -flask-sqlalchemy==2.5.1 +flask-sqlalchemy==3.0.5 # via + # apache-superset (pyproject.toml) # flask-appbuilder # flask-migrate flask-talisman==1.1.0 diff --git a/requirements/development.txt b/requirements/development.txt index 42563fb8814..34992847511 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -309,9 +309,10 @@ flask-session==0.8.0 # via # -c requirements/base-constraint.txt # apache-superset -flask-sqlalchemy==2.5.1 +flask-sqlalchemy==3.0.5 # via # -c requirements/base-constraint.txt + # apache-superset # flask-appbuilder # flask-migrate flask-talisman==1.1.0 @@ -989,7 +990,7 @@ sqlalchemy==1.4.54 # sqlalchemy-bigquery # sqlalchemy-continuum # sqlalchemy-utils -sqlalchemy-bigquery==1.17.0 +sqlalchemy-bigquery==1.17.1 # via apache-superset sqlalchemy-continuum==1.7.0 # via diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index eee5635160a..32dcf303097 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -275,6 +275,16 @@ class CompatibleDatabase(TypedDict, total=False): notes: str docs_url: str categories: list[str] # Override parent categories (e.g., for HOSTED_OPEN_SOURCE) + known_incompatibilities: list[KnownIncompatibility] + + +class KnownIncompatibility(TypedDict, total=False): + """A known, currently-unresolved incompatibility with a Superset dependency.""" + + dependency: str # e.g. "SQLAlchemy 2.0" + reason: str + tracking_url: str # upstream issue/PR tracking a fix, if one exists + since: str # ISO date this was last confirmed still broken class DBEngineSpecMetadata(TypedDict, total=False): @@ -317,6 +327,11 @@ class DBEngineSpecMetadata(TypedDict, total=False): install_instructions: str version_requirements: str + # Known, currently-unresolved incompatibilities with a Superset + # dependency (e.g. a driver that doesn't yet support SQLAlchemy 2.0). + # Hopefully temporary; remove the entry once resolved upstream. + known_incompatibilities: list[KnownIncompatibility] + # Related databases (e.g., PostgreSQL-compatible databases) compatible_databases: list[CompatibleDatabase] diff --git a/superset/db_engine_specs/d1.py b/superset/db_engine_specs/d1.py index d49c0db2c62..40214bf606a 100644 --- a/superset/db_engine_specs/d1.py +++ b/superset/db_engine_specs/d1.py @@ -48,4 +48,15 @@ class CloudflareD1EngineSpec(SqliteEngineSpec): "cloudflare_d1_database_id": "D1 database ID", }, "install_instructions": "pip install superset-engine-d1", + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "sqlalchemy-d1 is very young (single release, Nov 2025); " + "no confirmed SQLAlchemy 2.0 support either way yet." + ), + "tracking_url": ("https://github.com/sqlalchemy-cf-d1/sqlalchemy-d1";), + "since": "2026-07-28", + } + ], } diff --git a/superset/db_engine_specs/kusto.py b/superset/db_engine_specs/kusto.py index 85b563262bd..179a891a9cc 100644 --- a/superset/db_engine_specs/kusto.py +++ b/superset/db_engine_specs/kusto.py @@ -131,6 +131,17 @@ class KustoSqlEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method "notes": "Use native Kusto Query Language for advanced analytics.", }, ], + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "setup.py on the sqlalchemy-kusto main branch hard-pins " + "sqlalchemy==1.4.*; no SQLAlchemy 2.0 work has started." + ), + "tracking_url": "https://github.com/dodopizza/sqlalchemy-kusto";, + "since": "2026-07-28", + } + ], } _time_grain_expressions = { diff --git a/superset/db_engine_specs/mysql.py b/superset/db_engine_specs/mysql.py index 39d1b550cce..766107e3dc7 100644 --- a/superset/db_engine_specs/mysql.py +++ b/superset/db_engine_specs/mysql.py @@ -183,6 +183,22 @@ class MySQLEngineSpec(BasicParametersMixin, BaseEngineSpec): DatabaseCategory.CLOUD_AWS, DatabaseCategory.HOSTED_OPEN_SOURCE, ], + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "Neither our fork (preset-io/sqlalchemy-aurora-data-api, " + "dormant since 2021) nor the more active community fork " + "(cloud-utils/sqlalchemy-aurora-data-api) has resolved " + "SQLAlchemy 2.0 compatibility." + ), + "tracking_url": ( + "https://github.com/cloud-utils/"; + "sqlalchemy-aurora-data-api/issues/43" + ), + "since": "2026-07-28", + } + ], }, ], } diff --git a/superset/db_engine_specs/postgres.py b/superset/db_engine_specs/postgres.py index 1945fa3f303..2e5ab0143d7 100644 --- a/superset/db_engine_specs/postgres.py +++ b/superset/db_engine_specs/postgres.py @@ -545,6 +545,22 @@ class PostgresEngineSpec(BasicParametersMixin, PostgresBaseEngineSpec): DatabaseCategory.CLOUD_AWS, DatabaseCategory.HOSTED_OPEN_SOURCE, ], + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "Neither our fork (preset-io/sqlalchemy-aurora-data-api, " + "dormant since 2021) nor the more active community fork " + "(cloud-utils/sqlalchemy-aurora-data-api) has resolved " + "SQLAlchemy 2.0 compatibility." + ), + "tracking_url": ( + "https://github.com/cloud-utils/"; + "sqlalchemy-aurora-data-api/issues/43" + ), + "since": "2026-07-28", + } + ], }, ], } diff --git a/superset/db_engine_specs/solr.py b/superset/db_engine_specs/solr.py index 03dca6c2b66..eceec0efba7 100644 --- a/superset/db_engine_specs/solr.py +++ b/superset/db_engine_specs/solr.py @@ -43,6 +43,17 @@ class SolrEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method "[/?use_ssl=true|false]" ), "default_port": 8983, + "known_incompatibilities": [ + { + "dependency": "SQLAlchemy 2.0", + "reason": ( + "sqlalchemy-solr hard-pins sqlalchemy~=1.4.7 and has seen no " + "activity beyond dependabot bumps since 2024." + ), + "tracking_url": "https://github.com/aadel/sqlalchemy-solr";, + "since": "2026-07-28", + } + ], } _time_grain_expressions = {
