This is an automated email from the ASF dual-hosted git repository.
villebro pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 7eceebce1ce feat(dbspec): Support apache datafusion dbspec (#42374)
7eceebce1ce is described below
commit 7eceebce1ceaca78b548e705ab3630fa1e9dd687
Author: Bhargava Vadlamani <[email protected]>
AuthorDate: Fri Jul 24 11:56:30 2026 -0700
feat(dbspec): Support apache datafusion dbspec (#42374)
---
docs/src/data/databases.json | 108 +++++++++++++++++++--
docs/static/img/databases/datafusion.png | Bin 0 -> 14784 bytes
pyproject.toml | 1 +
superset/db_engine_specs/datafusion.py | 92 ++++++++++++++++++
superset_text.yml | 1 +
.../unit_tests/db_engine_specs/test_datafusion.py | 101 +++++++++++++++++++
6 files changed, 296 insertions(+), 7 deletions(-)
diff --git a/docs/src/data/databases.json b/docs/src/data/databases.json
index c2209b079ef..33662b38ea1 100644
--- a/docs/src/data/databases.json
+++ b/docs/src/data/databases.json
@@ -1,16 +1,16 @@
{
"generated": "2026-04-17T17:08:35.132Z",
"statistics": {
- "totalDatabases": 80,
- "withDocumentation": 80,
- "withConnectionString": 80,
- "withDrivers": 28,
+ "totalDatabases": 81,
+ "withDocumentation": 81,
+ "withConnectionString": 81,
+ "withDrivers": 29,
"withAuthMethods": 5,
- "supportsJoins": 74,
- "supportsSubqueries": 77,
+ "supportsJoins": 75,
+ "supportsSubqueries": 78,
"supportsDynamicSchema": 29,
"supportsCatalog": 20,
- "averageScore": 54,
+ "averageScore": 53,
"maxScore": 201,
"byCategory": {
"Cloud - AWS": [
@@ -20,6 +20,7 @@
],
"Query Engines": [
"Amazon Athena",
+ "Apache DataFusion",
"Apache Drill",
"Apache Hive",
"Apache Impala",
@@ -94,6 +95,7 @@
"Vertica"
],
"Apache Projects": [
+ "Apache DataFusion",
"Apache Doris",
"Apache Drill",
"Apache Druid",
@@ -108,6 +110,7 @@
"Databricks Interactive Cluster"
],
"Open Source": [
+ "Apache DataFusion",
"Apache Doris",
"Apache Drill",
"Apache Druid",
@@ -9698,6 +9701,97 @@
"supports_dynamic_schema": true,
"supports_catalog": true,
"supports_dynamic_catalog": true
+ },
+ "Apache DataFusion": {
+ "time_grains": {
+ "SECOND": true,
+ "FIVE_SECONDS": true,
+ "THIRTY_SECONDS": true,
+ "MINUTE": true,
+ "FIVE_MINUTES": true,
+ "TEN_MINUTES": true,
+ "FIFTEEN_MINUTES": true,
+ "THIRTY_MINUTES": true,
+ "HALF_HOUR": false,
+ "HOUR": true,
+ "SIX_HOURS": false,
+ "DAY": true,
+ "WEEK": true,
+ "WEEK_STARTING_SUNDAY": false,
+ "WEEK_STARTING_MONDAY": false,
+ "WEEK_ENDING_SATURDAY": false,
+ "WEEK_ENDING_SUNDAY": false,
+ "MONTH": true,
+ "QUARTER": true,
+ "QUARTER_YEAR": false,
+ "YEAR": true
+ },
+ "module": "superset.db_engine_specs.datafusion",
+ "limit_method": 1,
+ "limit_clause": true,
+ "joins": true,
+ "subqueries": true,
+ "alias_in_select": true,
+ "alias_in_orderby": true,
+ "time_groupby_inline": false,
+ "alias_to_source_column": false,
+ "order_by_not_in_select": true,
+ "expressions_in_orderby": false,
+ "cte_in_subquery": true,
+ "max_column_name": null,
+ "sql_comments": true,
+ "escaped_colons": true,
+ "masked_encrypted_extra": false,
+ "column_type_mapping": false,
+ "function_names": false,
+ "user_impersonation": false,
+ "file_upload": false,
+ "get_extra_table_metadata": false,
+ "dbapi_exception_mapping": false,
+ "custom_errors": false,
+ "dynamic_schema": false,
+ "catalog": false,
+ "dynamic_catalog": false,
+ "ssh_tunneling": true,
+ "query_cancelation": false,
+ "get_metrics": false,
+ "where_latest_partition": false,
+ "expand_data": false,
+ "query_cost_estimation": false,
+ "sql_validation": false,
+ "score": 24,
+ "max_score": 201,
+ "documentation": {
+ "description": "DataFusion is a highly performant query engine",
+ "logo": "datafusion.png",
+ "homepage_url": "https://datafusion.apache.org/";,
+ "categories": [
+ "Query Engines",
+ "Open Source",
+ "Apache Projects"
+ ],
+ "pypi_packages": [
+ "flightsql-dbapi"
+ ],
+ "connection_string": "datafusion://host:port",
+ "drivers": [
+ {
+ "name": "Arrow Flight SQL (Recommended)",
+ "pypi_package": "flightsql-dbapi",
+ "connection_string": "datafusion://host:port",
+ "is_recommended": true
+ }
+ ],
+ "category": "Apache Projects"
+ },
+ "engine": "datafusion",
+ "engine_name": "Apache DataFusion",
+ "engine_aliases": [],
+ "default_driver": "flightsql",
+ "supports_file_upload": false,
+ "supports_dynamic_schema": false,
+ "supports_catalog": false,
+ "supports_dynamic_catalog": false
}
}
}
diff --git a/docs/static/img/databases/datafusion.png
b/docs/static/img/databases/datafusion.png
new file mode 100644
index 00000000000..381265e62d7
Binary files /dev/null and b/docs/static/img/databases/datafusion.png differ
diff --git a/pyproject.toml b/pyproject.toml
index d379670d325..ecaf600014f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -142,6 +142,7 @@ databricks = [
"databricks-sql-connector>=4.2.6, <4.4.0",
"databricks-sqlalchemy==1.0.5",
]
+datafusion = ["flightsql-dbapi>=0.2.0, <0.3"]
db2 = ["ibm-db-sa>0.3.8, <=0.4.4"]
denodo = ["denodo-sqlalchemy>=2.0.5,<2.1.0"]
dremio = ["sqlalchemy-dremio>=1.2.1, <4"]
diff --git a/superset/db_engine_specs/datafusion.py
b/superset/db_engine_specs/datafusion.py
new file mode 100644
index 00000000000..ec7a9ac4f64
--- /dev/null
+++ b/superset/db_engine_specs/datafusion.py
@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import types
+
+from superset.constants import TimeGrain
+from superset.db_engine_specs import BaseEngineSpec
+from superset.db_engine_specs.base import DatabaseCategory
+
+
+class DataFusionEngineSpec(BaseEngineSpec):
+ engine_name = "Apache DataFusion"
+ engine = "datafusion"
+ drivers = {
+ "flightsql": "Arrow Flight SQL protocol for DataFusion",
+ }
+ default_driver = "flightsql"
+ sqlalchemy_uri_placeholder = "datafusion://host:port"
+ supports_file_upload = False
+
+ metadata = {
+ "description": "DataFusion is a highly performant query engine",
+ "logo": "datafusion.png",
+ "homepage_url": "https://datafusion.apache.org/";,
+ "categories": [
+ DatabaseCategory.QUERY_ENGINES,
+ DatabaseCategory.OPEN_SOURCE,
+ DatabaseCategory.APACHE_PROJECTS,
+ ],
+ "pypi_packages": ["flightsql-dbapi"],
+ "connection_string": "datafusion://host:port",
+ "drivers": [
+ {
+ "name": "Arrow Flight SQL (Recommended)",
+ "pypi_package": "flightsql-dbapi",
+ "connection_string": "datafusion://host:port",
+ "is_recommended": True,
+ }
+ ],
+ }
+
+ _time_grain_expressions = {
+ None: "{col}",
+ TimeGrain.SECOND: "DATE_TRUNC('second', {col})",
+ TimeGrain.FIVE_SECONDS: "DATE_TRUNC('minute', {col}) + INTERVAL '5
seconds' * FLOOR(EXTRACT(SECOND FROM {col}) / 5)", # noqa: E501
+ TimeGrain.THIRTY_SECONDS: "DATE_TRUNC('minute', {col}) + INTERVAL '30
seconds' * FLOOR(EXTRACT(SECOND FROM {col}) / 30)", # noqa: E501
+ TimeGrain.MINUTE: "DATE_TRUNC('minute', {col})",
+ TimeGrain.FIVE_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '5
minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 5)", # noqa: E501
+ TimeGrain.TEN_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '10
minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 10)", # noqa: E501
+ TimeGrain.FIFTEEN_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '15
minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 15)", # noqa: E501
+ TimeGrain.THIRTY_MINUTES: "DATE_TRUNC('hour', {col}) + INTERVAL '30
minutes' * FLOOR(EXTRACT(MINUTE FROM {col}) / 30)", # noqa: E501
+ TimeGrain.HOUR: "DATE_TRUNC('hour', {col})",
+ TimeGrain.DAY: "DATE_TRUNC('day', {col})",
+ TimeGrain.WEEK: "DATE_TRUNC('week', {col})",
+ TimeGrain.MONTH: "DATE_TRUNC('month', {col})",
+ TimeGrain.QUARTER: "DATE_TRUNC('quarter', {col})",
+ TimeGrain.YEAR: "DATE_TRUNC('year', {col})",
+ }
+
+ @classmethod
+ def epoch_to_dttm(cls) -> str:
+ return "from_unixtime({col})"
+
+ @classmethod
+ def convert_dttm(
+ cls, target_type: str, dttm: datetime, db_extra: dict[str, Any] | None
= None
+ ) -> str | None:
+ sqla_type = cls.get_sqla_column_type(target_type)
+
+ if isinstance(sqla_type, types.Date):
+ return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"
+ if isinstance(sqla_type, types.DateTime):
+ dttm_formatted = dttm.isoformat(sep=" ", timespec="milliseconds")
+ return f"""TO_TIMESTAMP('{dttm_formatted}', 'YYYY-MM-DD
HH24:MI:SS.FFF')"""
+ return None
diff --git a/superset_text.yml b/superset_text.yml
index 3b3a346b54f..aeb84d631b7 100644
--- a/superset_text.yml
+++ b/superset_text.yml
@@ -19,6 +19,7 @@
# To set the images of your preferred database, you may create a mapping here
with engine and locations of the relevant images. The image can be hosted
locally inside your static/file directory or online (e.g. S3).
# DB_IMAGES:
+# datafusion: "path/to/image/datafusion.jpg"
# postgresql: "path/to/image/postgres.jpg"
# bigquery: "path/to/s3bucket/bigquery.jpg"
# snowflake: "path/to/image/snowflake.jpg"
diff --git a/tests/unit_tests/db_engine_specs/test_datafusion.py
b/tests/unit_tests/db_engine_specs/test_datafusion.py
new file mode 100644
index 00000000000..0595f885cfd
--- /dev/null
+++ b/tests/unit_tests/db_engine_specs/test_datafusion.py
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from datetime import datetime
+
+import pytest
+
+from superset.db_engine_specs.datafusion import DataFusionEngineSpec
+from tests.unit_tests.db_engine_specs.utils import assert_convert_dttm
+from tests.unit_tests.fixtures.common import dttm # noqa: F401
+
+
[email protected](
+ "target_type,expected_result",
+ [
+ ("Date", "TO_DATE('2019-01-02', 'YYYY-MM-DD')"),
+ (
+ "DateTime",
+ "TO_TIMESTAMP('2019-01-02 03:04:05.678', 'YYYY-MM-DD
HH24:MI:SS.FFF')",
+ ),
+ (
+ "TimeStamp",
+ "TO_TIMESTAMP('2019-01-02 03:04:05.678', 'YYYY-MM-DD
HH24:MI:SS.FFF')",
+ ),
+ ("UnknownType", None),
+ ],
+)
+def test_convert_dttm(
+ target_type: str,
+ expected_result: str | None,
+ dttm: datetime, # noqa: F811
+) -> None:
+ assert_convert_dttm(DataFusionEngineSpec, target_type, expected_result,
dttm)
+
+
+def test_epoch_to_dttm() -> None:
+ assert DataFusionEngineSpec.epoch_to_dttm().format(col="ts") ==
"from_unixtime(ts)"
+
+
[email protected](
+ "time_grain,expected",
+ [
+ (None, "ts"),
+ ("PT1S", "DATE_TRUNC('second', ts)"),
+ (
+ "PT5S",
+ "DATE_TRUNC('minute', ts) + INTERVAL '5 seconds' * "
+ "FLOOR(EXTRACT(SECOND FROM ts) / 5)",
+ ),
+ (
+ "PT30S",
+ "DATE_TRUNC('minute', ts) + INTERVAL '30 seconds' * "
+ "FLOOR(EXTRACT(SECOND FROM ts) / 30)",
+ ),
+ ("PT1M", "DATE_TRUNC('minute', ts)"),
+ (
+ "PT5M",
+ "DATE_TRUNC('hour', ts) + INTERVAL '5 minutes' * "
+ "FLOOR(EXTRACT(MINUTE FROM ts) / 5)",
+ ),
+ (
+ "PT10M",
+ "DATE_TRUNC('hour', ts) + INTERVAL '10 minutes' * "
+ "FLOOR(EXTRACT(MINUTE FROM ts) / 10)",
+ ),
+ (
+ "PT15M",
+ "DATE_TRUNC('hour', ts) + INTERVAL '15 minutes' * "
+ "FLOOR(EXTRACT(MINUTE FROM ts) / 15)",
+ ),
+ (
+ "PT30M",
+ "DATE_TRUNC('hour', ts) + INTERVAL '30 minutes' * "
+ "FLOOR(EXTRACT(MINUTE FROM ts) / 30)",
+ ),
+ ("PT1H", "DATE_TRUNC('hour', ts)"),
+ ("P1D", "DATE_TRUNC('day', ts)"),
+ ("P1W", "DATE_TRUNC('week', ts)"),
+ ("P1M", "DATE_TRUNC('month', ts)"),
+ ("P3M", "DATE_TRUNC('quarter', ts)"),
+ ("P1Y", "DATE_TRUNC('year', ts)"),
+ ],
+)
+def test_time_grain_expressions(time_grain: str | None, expected: str) -> None:
+ assert (
+
DataFusionEngineSpec._time_grain_expressions[time_grain].format(col="ts")
+ == expected
+ )
