This is an automated email from the ASF dual-hosted git repository. eschutho pushed a commit to branch fix-bigquery-format-detect-noise in repository https://gitbox.apache.org/repos/asf/superset.git
commit cc6cf348c75b33d1dcf3966f67f8f7a6838682e2 Author: Elizabeth Thompson <[email protected]> AuthorDate: Fri Jul 24 15:31:48 2026 +0000 fix(datasets): log datetime format-detection DB query failures at WARNING (SC-115217) DatetimeFormatDetector.detect_column_format() already handles any failure during best-effort datetime format sampling by returning None -- format detection has zero user-facing impact when skipped. It used a single broad except Exception around the whole method body, so an expected, already-handled failure while actually querying the target database (bad connection config, transient outage, permission errors) was logged via logger.exception() (ERROR + full traceback), which Sentry's logging integration captures on every occurrence. The specific trigger is a customer's misconfigured BigQuery connection rejecting the sampling query with a 400 ("Cannot parse '' as CloudRegion"), flooding Sentry ~90k times across 24 users since 2025-12 with a condition that is fully recoverable and has no user impact. Wrap only the database.get_df() call in its own try/except, logging at WARNING instead -- matching the level already used for the sibling "No data returned"/"Could not detect format" cases a few lines below. The outer except Exception is unchanged, so genuine internal bugs (SQL building, quoting, etc.) still log at ERROR and stay actionable. Fixes SUPERSET-PYTHON-YDB Co-Authored-By: Claude <[email protected]> --- superset/datasets/datetime_format_detector.py | 19 +++++++++-- .../datasets/test_datetime_format_detector.py | 39 ++++++++++++++++++++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/superset/datasets/datetime_format_detector.py b/superset/datasets/datetime_format_detector.py index 4c8b036a4f3..7e6f08f1b46 100644 --- a/superset/datasets/datetime_format_detector.py +++ b/superset/datasets/datetime_format_detector.py @@ -122,8 +122,23 @@ class DatetimeFormatDetector: # This handles different SQL dialects (LIMIT, TOP, FETCH FIRST, etc.) sql = database.apply_limit_to_sql(sql, limit=self.sample_size, force=True) - # Execute query and get results - df = database.get_df(sql, dataset.schema) + # Execute query and get results. Failures here come from the + # target database itself (bad connection config, transient + # outage, permission errors, etc.), not from Superset's own + # logic. Format detection is a best-effort optimization with no + # user-facing impact when it's skipped, so log at WARNING + # instead of capturing an ERROR-level exception for every sample + # query a misconfigured/unreachable database rejects. + try: + df = database.get_df(sql, dataset.schema) + except Exception as ex: + logger.warning( + "Could not query column %s.%s for format detection: %s", + dataset.table_name, + column.column_name, + str(ex), + ) + return None if df.empty or column.column_name not in df.columns: logger.warning( diff --git a/tests/unit_tests/datasets/test_datetime_format_detector.py b/tests/unit_tests/datasets/test_datetime_format_detector.py index d66e2efe8e2..afb3a95546d 100644 --- a/tests/unit_tests/datasets/test_datetime_format_detector.py +++ b/tests/unit_tests/datasets/test_datetime_format_detector.py @@ -16,6 +16,7 @@ # under the License. """Tests for datetime format detector.""" +import logging from unittest.mock import MagicMock import pandas as pd @@ -108,16 +109,48 @@ def test_detect_column_format_empty_data( def test_detect_column_format_error_handling( - mock_dataset: MagicMock, mock_column: MagicMock + mock_dataset: MagicMock, mock_column: MagicMock, caplog: pytest.LogCaptureFixture ) -> None: - """Test error handling during format detection.""" + """Test error handling during format detection. + + A failure while querying the target database (bad connection config, + transient outage, permission errors, etc.) is expected and already + fully handled -- it must not be captured as an ERROR-level exception, + since that floods Sentry with noise for every sample query a + misconfigured/unreachable database rejects. + """ # Simulate database error mock_dataset.database.get_df.side_effect = Exception("Database error") detector = DatetimeFormatDetector() - detected_format = detector.detect_column_format(mock_dataset, mock_column) + with caplog.at_level(logging.WARNING): + detected_format = detector.detect_column_format(mock_dataset, mock_column) + + assert detected_format is None + assert not any(record.levelno >= logging.ERROR for record in caplog.records) + assert any( + record.levelno == logging.WARNING and "Could not query column" in record.message + for record in caplog.records + ) + + +def test_detect_column_format_internal_error_still_logs_at_error( + mock_dataset: MagicMock, mock_column: MagicMock, caplog: pytest.LogCaptureFixture +) -> None: + """A genuine internal bug (not a database-query failure) should still be + logged at ERROR so it remains visible/actionable, unlike the expected + database-query failure case above.""" + mock_dataset.database.get_sqla_engine.side_effect = RuntimeError( + "unexpected internal error" + ) + + detector = DatetimeFormatDetector() + with caplog.at_level(logging.WARNING): + detected_format = detector.detect_column_format(mock_dataset, mock_column) assert detected_format is None + assert any(record.levelno >= logging.ERROR for record in caplog.records) + mock_dataset.database.get_df.assert_not_called() def test_detect_all_formats(mock_dataset: MagicMock) -> None:
