This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9e258503283 Use the vertica+vertica_python scheme in
VerticaHook.sqlalchemy_url (#76804)
9e258503283 is described below
commit 9e2585032838132219b682626d06a0eafe2029dd
Author: Ianislava Chalbysheva <[email protected]>
AuthorDate: Wed Nov 18 13:35:07 2026 +0700
Use the vertica+vertica_python scheme in VerticaHook.sqlalchemy_url (#76804)
* Use the vertica+vertica_python scheme in VerticaHook.sqlalchemy_url
VerticaHook.sqlalchemy_url built the URL with the "vertica-python"
drivername, but Vertica SQLAlchemy dialects (vertica-sqlalchemy-dialect,
sqlalchemy-vertica, sqlalchemy-vertica-python) register themselves as
"vertica.vertica_python". As a result get_sqlalchemy_engine() raised
NoSuchModuleError even with a dialect installed, and get_uri() returned a
string make_url() cannot parse ("-" is not allowed in the scheme), so
dialect_name silently fell back to "default".
Use "vertica+vertica_python", replace the test that mocked create_engine
with regression tests that parse get_uri() and create the engine through
a dialect registered under the same name, and document which dialect
works with SQLAlchemy 2.x.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
* Support SQLAlchemy 1.4 in the stub Vertica dialect of the tests
SQLAlchemy < 2.0 gets the DB-API module from the dialect's dbapi()
classmethod, which SQLAlchemy 2 replaced with import_dbapi(), so the
regression test failed in the compatibility runs on Airflow 2.11 and
3.0. Define both; SQLAlchemy 2 uses import_dbapi() when it is defined,
without a deprecation warning.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
* Remove the mocked engine test and import vertica_python at module level
test_get_sqlalchemy_engine mocked create_engine and only repeated what
test_get_sqlalchemy_engine_resolves_vertica_dialect checks without a
mock; its username and password assertions move to the unmocked test.
Import vertica_python at the top of the test module instead of inside
the stub dialect.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
providers/vertica/docs/connections/vertica.rst | 20 ++++++
.../src/airflow/providers/vertica/hooks/vertica.py | 10 ++-
.../tests/unit/vertica/hooks/test_vertica_sql.py | 82 ++++++++++++++++------
3 files changed, 88 insertions(+), 24 deletions(-)
diff --git a/providers/vertica/docs/connections/vertica.rst
b/providers/vertica/docs/connections/vertica.rst
index 86f583a5481..aede93cbc45 100644
--- a/providers/vertica/docs/connections/vertica.rst
+++ b/providers/vertica/docs/connections/vertica.rst
@@ -81,3 +81,23 @@ Extra (optional)
"connection_timeout": 30,
"backup_server_node": ["bck_server_1", "bck_server_2"]
}
+
+Using SQLAlchemy
+----------------
+
+``VerticaHook.get_sqlalchemy_engine()`` and the features built on it (for
example
+``DataFrame.to_sql`` with the hook's engine, ``inspector`` or
``get_table_schema``) need the
+``sqlalchemy`` extra and a Vertica SQLAlchemy dialect. The hook builds a
+``vertica+vertica_python://`` URL from the connection, which is the name
Vertica dialects register.
+
+* With SQLAlchemy 2.x, `sqlalchemy-vertica
<https://pypi.org/project/sqlalchemy-vertica/>`_ has been
+ checked to work (connection, inspection and ``DataFrame.to_sql``).
+* The official `vertica-sqlalchemy-dialect
<https://pypi.org/project/vertica-sqlalchemy-dialect/>`_
+ requires SQLAlchemy 1.4.44 or lower, so it does not work with SQLAlchemy 2.x.
+
+.. code-block:: bash
+
+ pip install 'apache-airflow-providers-vertica[sqlalchemy]'
sqlalchemy-vertica
+
+Connection extras are passed to the dialect as URL query parameters, and the
dialect decides how to
+convert them for vertica-python.
diff --git a/providers/vertica/src/airflow/providers/vertica/hooks/vertica.py
b/providers/vertica/src/airflow/providers/vertica/hooks/vertica.py
index 1913555bd32..9f1a7e8b68e 100644
--- a/providers/vertica/src/airflow/providers/vertica/hooks/vertica.py
+++ b/providers/vertica/src/airflow/providers/vertica/hooks/vertica.py
@@ -166,7 +166,13 @@ class VerticaHook(DbApiHook):
@property
def sqlalchemy_url(self) -> URL:
- """Return a SQLAlchemy URL object with properly formatted query
parameters."""
+ """
+ Return a SQLAlchemy URL object with properly formatted query
parameters.
+
+ The URL uses the ``vertica+vertica_python`` scheme: Vertica SQLAlchemy
dialects register
+ themselves as ``vertica.vertica_python``, so a dialect package (for
SQLAlchemy 2.x, for example
+ ``sqlalchemy-vertica``) has to be installed for
``get_sqlalchemy_engine()`` to work.
+ """
try:
from sqlalchemy.engine import URL
except (ImportError, ModuleNotFoundError) as err:
@@ -185,7 +191,7 @@ class VerticaHook(DbApiHook):
}
return URL.create(
- drivername="vertica-python",
+ drivername="vertica+vertica_python",
username=conn.login,
password=conn.password or "",
host=conn.host or "localhost",
diff --git a/providers/vertica/tests/unit/vertica/hooks/test_vertica_sql.py
b/providers/vertica/tests/unit/vertica/hooks/test_vertica_sql.py
index 3618d55baad..b43361b9420 100644
--- a/providers/vertica/tests/unit/vertica/hooks/test_vertica_sql.py
+++ b/providers/vertica/tests/unit/vertica/hooks/test_vertica_sql.py
@@ -23,7 +23,10 @@ from unittest import mock
from unittest.mock import MagicMock, PropertyMock, patch
import pytest
-from sqlalchemy.engine import Engine
+import vertica_python
+from sqlalchemy.dialects import registry
+from sqlalchemy.engine import make_url
+from sqlalchemy.engine.default import DefaultDialect
from airflow.models import Connection
from airflow.providers.common.compat.sdk import AirflowException
@@ -85,7 +88,7 @@ def mock_cursor(mock_get_conn):
def test_sqlalchemy_url_property(vertica_hook):
url = vertica_hook.sqlalchemy_url.render_as_string(hide_password=False)
- expected_url =
f"vertica-python://{USER}:{PASSWORD}@{HOST}:{PORT}/{DATABASE}"
+ expected_url =
f"vertica+vertica_python://{USER}:{PASSWORD}@{HOST}:{PORT}/{DATABASE}"
assert url.startswith(expected_url)
@@ -200,34 +203,69 @@ def test_get_uri(vertica_hook):
"""
Test that the get_uri() method returns the correct connection string.
"""
- assert vertica_hook.get_uri() ==
"vertica-python://user:[email protected]:5433/test_db"
+ assert vertica_hook.get_uri() ==
"vertica+vertica_python://user:[email protected]:5433/test_db"
-def test_get_sqlalchemy_engine(vertica_hook):
- """
- Test that the get_sqlalchemy_engine() method returns a valid SQLAlchemy
engine.
- """
+class StubVerticaDialect(DefaultDialect):
+ """Stand-in for a Vertica SQLAlchemy dialect package, registered under the
same name they use."""
- with patch("airflow.providers.common.sql.hooks.sql.create_engine") as
mock_create_engine:
- mock_engine = MagicMock(spec=Engine)
- mock_create_engine.return_value = mock_engine
+ name = "vertica"
+ driver = "vertica_python"
+ supports_statement_cache = True
- engine = vertica_hook.get_sqlalchemy_engine()
+ @classmethod
+ def import_dbapi(cls):
+ return vertica_python
- assert engine is mock_engine
+ @classmethod
+ def dbapi(cls):
+ # SQLAlchemy < 2.0 calls dbapi() instead of import_dbapi(); SQLAlchemy
2 uses import_dbapi() when defined.
+ return cls.import_dbapi()
- mock_create_engine.assert_called_once()
- call_args = mock_create_engine.call_args[1]
- assert "url" in call_args
- actual_url = call_args["url"]
[email protected]
+def vertica_dialect_registered():
+ """
+ Register a dialect as ``vertica.vertica_python``, the name Vertica
dialects register.
- assert actual_url.drivername == "vertica-python"
- assert actual_url.username == "user"
- assert actual_url.password == "pass"
- assert actual_url.host == "vertica.cloud.com"
- assert actual_url.port == 5433
- assert actual_url.database == "test_db"
+ ``vertica-sqlalchemy-dialect``, ``sqlalchemy-vertica`` and
``sqlalchemy-vertica-python`` all
+ declare the ``sqlalchemy.dialects`` entry point ``vertica.vertica_python``.
+ """
+ name = "vertica.vertica_python"
+ previous = registry.impls.get(name)
+ registry.register(name, __name__, "StubVerticaDialect")
+ yield
+ if previous is None:
+ registry.impls.pop(name, None)
+ else:
+ registry.impls[name] = previous
+
+
+def test_get_uri_can_be_parsed_by_sqlalchemy(vertica_hook):
+ """Regression: the URI used to have the scheme "vertica-python", which
make_url cannot parse."""
+ url = make_url(vertica_hook.get_uri())
+ assert url.drivername == "vertica+vertica_python"
+ assert url.host == HOST
+ assert url.database == DATABASE
+
+
[email protected]("vertica_dialect_registered")
+def test_get_sqlalchemy_engine_resolves_vertica_dialect(vertica_hook):
+ """Regression: the engine is created through the registered Vertica
dialect, without mocking."""
+ engine = vertica_hook.get_sqlalchemy_engine()
+ assert isinstance(engine.dialect, StubVerticaDialect)
+ assert engine.url.drivername == "vertica+vertica_python"
+ assert engine.url.username == USER
+ assert engine.url.password == PASSWORD
+ assert engine.url.host == HOST
+ assert engine.url.port == PORT
+ assert engine.url.database == DATABASE
+
+
[email protected]("vertica_dialect_registered")
+def test_dialect_name_is_vertica(vertica_hook):
+ """Regression: dialect_name fell back to "default" because get_uri() could
not be parsed."""
+ assert vertica_hook.dialect_name == "vertica"
@pytest.mark.parametrize("sql", ["", "\n", " "])