This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 0e50b64263f fix(bigquery): filter __NULL__ and __UNPARTITIONED__ from
max partition query (#44156)
0e50b64263f is described below
commit 0e50b64263f574464eafcd8211a423a09ef919b5
Author: Konstantin Khlopkov <[email protected]>
AuthorDate: Sun Oct 11 19:55:04 2026 +0300
fix(bigquery): filter __NULL__ and __UNPARTITIONED__ from max partition
query (#44156)
Signed-off-by: Konstantin Khlopkov <[email protected]>
---
superset/db_engine_specs/bigquery.py | 20 ++++--
tests/unit_tests/db_engine_specs/test_bigquery.py | 86 ++++++++++++++++++++++-
2 files changed, 99 insertions(+), 7 deletions(-)
diff --git a/superset/db_engine_specs/bigquery.py
b/superset/db_engine_specs/bigquery.py
index 397edbf5cf6..3199d210da6 100644
--- a/superset/db_engine_specs/bigquery.py
+++ b/superset/db_engine_specs/bigquery.py
@@ -556,9 +556,11 @@ class BigQueryEngineSpec(BaseEngineSpec): # pylint:
disable=too-many-public-met
) -> Select | None:
if partition_column := cls.get_time_partition_column(database, table):
max_partition_id = cls.get_max_partition_id(database, table)
- query = query.where(
- column(partition_column) == func.PARSE_DATE("%Y%m%d",
max_partition_id)
- )
+ if max_partition_id is not None:
+ query = query.where(
+ column(partition_column)
+ == func.PARSE_DATE("%Y%m%d", max_partition_id)
+ )
return query
@@ -585,9 +587,15 @@ class BigQueryEngineSpec(BaseEngineSpec): # pylint:
disable=too-many-public-met
)
# Build the query
- query = select(
- func.max(partitions_table.c.partition_id).label("max_partition_id")
- ).where(partitions_table.c.table_name == table.table)
+ query = (
+
select(func.max(partitions_table.c.partition_id).label("max_partition_id"))
+ .where(partitions_table.c.table_name == table.table)
+ .where(
+ partitions_table.c.partition_id.notin_(
+ ["__NULL__", "__UNPARTITIONED__"]
+ )
+ )
+ )
# Compile to BigQuery SQL
compiled_query = query.compile(
diff --git a/tests/unit_tests/db_engine_specs/test_bigquery.py
b/tests/unit_tests/db_engine_specs/test_bigquery.py
index 0367b916c8c..cdcc2550c09 100644
--- a/tests/unit_tests/db_engine_specs/test_bigquery.py
+++ b/tests/unit_tests/db_engine_specs/test_bigquery.py
@@ -24,7 +24,7 @@ from unittest import mock
import pytest
from pytest_mock import MockerFixture
-from sqlalchemy import select
+from sqlalchemy import column, select
from sqlalchemy.engine.url import make_url
from sqlalchemy.sql import sqltypes
from sqlalchemy_bigquery import BigQueryDialect
@@ -1219,3 +1219,87 @@ def test_identifier_quote_uses_backticks() -> None:
"end": "`",
"escape_by_doubling": False,
}
+
+
+def test_where_latest_partition_filters_null_partitions(
+ mocker: MockerFixture,
+) -> None:
+ """
+ Test that get_max_partition_id excludes __NULL__ and __UNPARTITIONED__.
+
+ When a table has NULL values in the partitioning column, BigQuery creates a
+ partition named __NULL__. If no explicit WHERE filter is applied during
+ the initial scan, BigQuery may also report an __UNPARTITIONED__ partition.
+ Both of these special IDs sort lexicographically higher than numeric
+ YYYYMMDD strings and would otherwise become max(partition_id), causing
+ PARSE_DATE('%Y%m%d', '__NULL__') to fail.
+ """
+ from superset.db_engine_specs.bigquery import BigQueryEngineSpec
+
+ database = mock.MagicMock()
+ engine = mock.MagicMock()
+ engine_ctx = mock.MagicMock()
+ engine_ctx.__enter__.return_value = engine
+ mocker.patch.object(BigQueryEngineSpec, "get_engine",
return_value=engine_ctx)
+
+ # Mock the partition metadata query: __NULL__ partitions are filtered out
+ cursor_mock = mock.MagicMock()
+ cursor_mock.fetchone.return_value = ("20251231",)
+ raw_conn_mock = mock.MagicMock()
+ raw_conn_mock.cursor.return_value = cursor_mock
+ database.get_raw_connection.return_value.__enter__.return_value =
raw_conn_mock
+ database.get_dialect.return_value = BigQueryDialect()
+
+ # Mock time_partitioning to identify the partition column
+ client = mocker.patch.object(BigQueryEngineSpec,
"_get_client").return_value
+ bq_table_mock = mock.MagicMock()
+ bq_table_mock.time_partitioning.field = "partition_date"
+ client.get_table.return_value = bq_table_mock
+
+ table = Table("test_table", "test_dataset", "test_project")
+
+ # Ensure the generated SQL filters out __NULL__ and __UNPARTITIONED__
+ max_partition = BigQueryEngineSpec.get_max_partition_id(database, table)
+ assert max_partition == "20251231"
+
+ # The returned SQL should be the result of filtering, not one of the
special IDs
+ executed_sql = cursor_mock.execute.call_args[0][0]
+ assert "__NULL__" in executed_sql or "NOT IN" in executed_sql
+ assert "__UNPARTITIONED__" in executed_sql or "NOT IN" in executed_sql
+
+
+def test_where_latest_partition_skips_where_clause_when_max_is_none(
+ mocker: MockerFixture,
+) -> None:
+ """
+ Test that where_latest_partition leaves the query untouched when the table
+ has only reserved partitions and get_max_partition_id returns None.
+
+ Without the guard, the filter would compile to
+ PARSE_DATE('%Y%m%d', None), which fails at query time.
+ """
+ from superset.db_engine_specs.bigquery import BigQueryEngineSpec
+
+ database = mock.MagicMock()
+
+ # All partitions are reserved, so the aggregate returns NULL
+ cursor_mock = mock.MagicMock()
+ cursor_mock.fetchone.return_value = (None,)
+ raw_conn_mock = mock.MagicMock()
+ raw_conn_mock.cursor.return_value = cursor_mock
+ database.get_raw_connection.return_value.__enter__.return_value =
raw_conn_mock
+ database.get_dialect.return_value = BigQueryDialect()
+
+ client = mocker.patch.object(BigQueryEngineSpec,
"_get_client").return_value
+ bq_table_mock = mock.MagicMock()
+ bq_table_mock.time_partitioning.field = "partition_date"
+ client.get_table.return_value = bq_table_mock
+
+ table = Table("test_table", "test_dataset", "test_project")
+ query = select(column("foo"))
+
+ filtered = BigQueryEngineSpec.where_latest_partition(database, table,
query)
+
+ # No WHERE clause was appended: the query is returned unchanged
+ assert filtered is query
+ assert not query.whereclause
