This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e07139a Replaces the usage of postgres:// with postgresql:// (#21205)
e07139a is described below
commit e07139ab73ebf8d7ebd46655a304708f13a440ac
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Jan 30 12:50:03 2022 +0100
Replaces the usage of postgres:// with postgresql:// (#21205)
After releasing 3.4.4 we can finally migrate to SQLAlchemy 1.4,
however SQLAlchemy 1.4 removed the use of postgres:// as valid
specification for Postgres DB SQL (leaving only postgresql://)
Due to that we need to change:
* postgres provider to return postgresql:// with get_db_uri()
* fix a number of tests that expected postgres://
We cannot do much if someone uses postgres:// specification.
Technically it might be seen as breaking change, but this is not
an airflow breaking change and users could still use SQLAlchemy
1.3 to keep the old prefix, so we can introduce this change
in Airflow without raising the major version.
Details in the [SQLAlchemy
Changelog](https://docs.sqlalchemy.org/en/14/changelog/changelog_14.html#change-3687655465c25a39b968b4f5f6e9170b).
---
UPDATING.md | 19 ++++++++++++++++++-
airflow/providers/postgres/hooks/postgres.py | 3 +--
docs/apache-airflow/howto/set-up-database.rst | 18 ++++++++++++++++++
tests/models/test_base.py | 4 ++--
tests/models/test_connection.py | 24 ++++++++++++------------
tests/providers/postgres/hooks/test_postgres.py | 6 +++---
6 files changed, 54 insertions(+), 20 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index 29e9a0f..a4e2e17 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -80,7 +80,24 @@ https://developers.google.com/style/inclusive-documentation
-->
-### Passing ``execution_date`` to ``XCom.set()``, ``XCom.clear()``,
``XCom.get_one()``, and ``XCom.get_many()`` is deprecated
+### You have to use `postgresql://` instead of `postgres://` in
`sql_alchemy_conn` for SQLAlchemy 1.4.0+
+
+When you use SQLAlchemy 1.4.0+, you need ot use `postgresql://` as the
database in the `sql_alchemy_conn`.
+In the previous versions of SQLAlchemy it was possible to use `postgres://`,
but using it in
+SQLAlchemy 1.4.0+ results in:
+
+```
+> raise exc.NoSuchModuleError(
+ "Can't load plugin: %s:%s" % (self.group, name)
+ )
+E sqlalchemy.exc.NoSuchModuleError: Can't load plugin:
sqlalchemy.dialects:postgres
+```
+
+If you cannot change the prefix of your URL immediately, Airflow continues to
work with SQLAlchemy
+1.3 and you can downgrade SQLAlchemy, but we recommend to update the prefix.
+Details in the [SQLAlchemy
Changelog](https://docs.sqlalchemy.org/en/14/changelog/changelog_14.html#change-3687655465c25a39b968b4f5f6e9170b).
+
+### Passing `execution_date` to `XCom.set()`, `XCom.clear()`,
`XCom.get_one()`, and `XCom.get_many()` is deprecated
Continuing the effort to bind TaskInstance to a DagRun, XCom entries are now
also tied to a DagRun. Use the ``run_id`` argument to specify the DagRun
instead.
diff --git a/airflow/providers/postgres/hooks/postgres.py
b/airflow/providers/postgres/hooks/postgres.py
index 4b495f4..a707f9c 100644
--- a/airflow/providers/postgres/hooks/postgres.py
+++ b/airflow/providers/postgres/hooks/postgres.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
import os
from contextlib import closing
from copy import deepcopy
@@ -139,7 +138,7 @@ class PostgresHook(DbApiHook):
def get_uri(self) -> str:
conn = self.get_connection(getattr(self, self.conn_name_attr))
- uri = super().get_uri()
+ uri = super().get_uri().replace("postgres://", "postgresql://")
if conn.extra_dejson.get('client_encoding', False):
charset = conn.extra_dejson["client_encoding"]
return f"{uri}?client_encoding={charset}"
diff --git a/docs/apache-airflow/howto/set-up-database.rst
b/docs/apache-airflow/howto/set-up-database.rst
index a155880..c1e6ccb 100644
--- a/docs/apache-airflow/howto/set-up-database.rst
+++ b/docs/apache-airflow/howto/set-up-database.rst
@@ -169,6 +169,24 @@ the database configuration to load your change. See
`The pg_hba.conf File
<https://www.postgresql.org/docs/current/auth-pg-hba-conf.html>`__
in the Postgres documentation to learn more.
+.. warning::
+
+ When you use SQLAlchemy 1.4.0+, you need ot use ``postgresql://`` as the
database in the ``sql_alchemy_conn``.
+ In the previous versions of SQLAlchemy it was possible to use
``postgres://``, but using it in
+ SQLAlchemy 1.4.0+ results in:
+
+ .. code-block::
+
+ > raise exc.NoSuchModuleError(
+ "Can't load plugin: %s:%s" % (self.group, name)
+ )
+ E sqlalchemy.exc.NoSuchModuleError: Can't load plugin:
sqlalchemy.dialects:postgres
+
+ If you cannot change the prefix of your URL immediately, Airflow continues
to work with SQLAlchemy
+ 1.3 and you can downgrade SQLAlchemy, but we recommend to update the prefix.
+
+ Details in the `SQLAlchemy Changelog
<https://docs.sqlalchemy.org/en/14/changelog/changelog_14.html#change-3687655465c25a39b968b4f5f6e9170b>`_.
+
We recommend using the ``psycopg2`` driver and specifying it in your
SqlAlchemy connection string.
.. code-block:: text
diff --git a/tests/models/test_base.py b/tests/models/test_base.py
index 5579986..283fa69 100644
--- a/tests/models/test_base.py
+++ b/tests/models/test_base.py
@@ -25,7 +25,7 @@ from tests.test_utils.config import conf_vars
@pytest.mark.parametrize(
("dsn", "expected", "extra"),
[
- param("postgres://host/the_database", {}, {}, id="postgres"),
+ param("postgresql://host/the_database", {}, {}, id="postgres"),
param("mysql://host/the_database", {"collation": "utf8mb3_bin"}, {},
id="mysql"),
param("mysql+pymsql://host/the_database", {"collation":
"utf8mb3_bin"}, {}, id="mysql+pymsql"),
param(
@@ -35,7 +35,7 @@ from tests.test_utils.config import conf_vars
id="mysql with explicit config",
),
param(
- "postgres://host/the_database",
+ "postgresql://host/the_database",
{"collation": "ascii"},
{('core', 'sql_engine_collation_for_ids'): 'ascii'},
id="postgres with explicit config",
diff --git a/tests/models/test_connection.py b/tests/models/test_connection.py
index 117e078..a527381 100644
--- a/tests/models/test_connection.py
+++ b/tests/models/test_connection.py
@@ -501,7 +501,7 @@ class TestConnection(unittest.TestCase):
@mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_TEST_URI':
'postgres://username:[email protected]:5432/the_database',
+ 'AIRFLOW_CONN_TEST_URI':
'postgresql://username:[email protected]:5432/the_database',
},
)
def test_using_env_var(self):
@@ -517,7 +517,7 @@ class TestConnection(unittest.TestCase):
@mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_TEST_URI_NO_CREDS':
'postgres://ec2.compute.com/the_database',
+ 'AIRFLOW_CONN_TEST_URI_NO_CREDS':
'postgresql://ec2.compute.com/the_database',
},
)
def test_using_unix_socket_env_var(self):
@@ -550,7 +550,7 @@ class TestConnection(unittest.TestCase):
with mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_AIRFLOW_DB':
'postgres://username:[email protected]:5432/the_database',
+ 'AIRFLOW_CONN_AIRFLOW_DB':
'postgresql://username:[email protected]:5432/the_database',
},
):
conn = SqliteHook.get_connection(conn_id='airflow_db')
@@ -563,23 +563,23 @@ class TestConnection(unittest.TestCase):
@mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_TEST_URI':
'postgres://username:[email protected]:5432/the_database',
- 'AIRFLOW_CONN_TEST_URI_NO_CREDS':
'postgres://ec2.compute.com/the_database',
+ 'AIRFLOW_CONN_TEST_URI':
'postgresql://username:[email protected]:5432/the_database',
+ 'AIRFLOW_CONN_TEST_URI_NO_CREDS':
'postgresql://ec2.compute.com/the_database',
},
)
def test_dbapi_get_uri(self):
conn = BaseHook.get_connection(conn_id='test_uri')
hook = conn.get_hook()
- assert
'postgres://username:[email protected]:5432/the_database' ==
hook.get_uri()
+ assert
'postgresql://username:[email protected]:5432/the_database' ==
hook.get_uri()
conn2 = BaseHook.get_connection(conn_id='test_uri_no_creds')
hook2 = conn2.get_hook()
- assert 'postgres://ec2.compute.com/the_database' == hook2.get_uri()
+ assert 'postgresql://ec2.compute.com/the_database' == hook2.get_uri()
@mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_TEST_URI':
'postgres://username:[email protected]:5432/the_database',
- 'AIRFLOW_CONN_TEST_URI_NO_CREDS':
'postgres://ec2.compute.com/the_database',
+ 'AIRFLOW_CONN_TEST_URI':
'postgresql://username:[email protected]:5432/the_database',
+ 'AIRFLOW_CONN_TEST_URI_NO_CREDS':
'postgresql://ec2.compute.com/the_database',
},
)
def test_dbapi_get_sqlalchemy_engine(self):
@@ -587,13 +587,13 @@ class TestConnection(unittest.TestCase):
hook = conn.get_hook()
engine = hook.get_sqlalchemy_engine()
assert isinstance(engine, sqlalchemy.engine.Engine)
- assert
'postgres://username:[email protected]:5432/the_database' ==
str(engine.url)
+ assert
'postgresql://username:[email protected]:5432/the_database' ==
str(engine.url)
@mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_TEST_URI':
'postgres://username:[email protected]:5432/the_database',
- 'AIRFLOW_CONN_TEST_URI_NO_CREDS':
'postgres://ec2.compute.com/the_database',
+ 'AIRFLOW_CONN_TEST_URI':
'postgresql://username:[email protected]:5432/the_database',
+ 'AIRFLOW_CONN_TEST_URI_NO_CREDS':
'postgresql://ec2.compute.com/the_database',
},
)
def test_get_connections_env_var(self):
diff --git a/tests/providers/postgres/hooks/test_postgres.py
b/tests/providers/postgres/hooks/test_postgres.py
index 57fc2d3..a0d1430 100644
--- a/tests/providers/postgres/hooks/test_postgres.py
+++ b/tests/providers/postgres/hooks/test_postgres.py
@@ -65,7 +65,7 @@ class TestPostgresHookConn(unittest.TestCase):
assert mock_connect.call_count == 1
self.assertEqual(
- self.db_hook.get_uri(),
"postgres://login:password@host/schema?client_encoding=utf-8"
+ self.db_hook.get_uri(),
"postgresql://login:password@host/schema?client_encoding=utf-8"
)
@mock.patch('airflow.providers.postgres.hooks.postgres.psycopg2.connect')
@@ -168,7 +168,7 @@ class TestPostgresHookConn(unittest.TestCase):
port=1,
)
)
- assert "postgres://login:password@host:1/schema" ==
self.db_hook.get_uri()
+ assert "postgresql://login:password@host:1/schema" ==
self.db_hook.get_uri()
def test_get_uri_from_connection_with_schema_override(self):
hook = PostgresHook(schema='schema-override')
@@ -182,7 +182,7 @@ class TestPostgresHookConn(unittest.TestCase):
port=1,
)
)
- assert "postgres://login:password@host:1/schema-override" ==
hook.get_uri()
+ assert "postgresql://login:password@host:1/schema-override" ==
hook.get_uri()
class TestPostgresHook(unittest.TestCase):
