This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 312f7e873d Remove mysql-connector-python (#30487)
312f7e873d is described below
commit 312f7e873d5141ad65e64dbffa6095232c6c29b6
Author: max <[email protected]>
AuthorDate: Thu Apr 13 18:04:19 2023 +0200
Remove mysql-connector-python (#30487)
* Turn the package 'mysql-connector-python' as an optional feature
* Update airflow/providers/mysql/provider.yaml
* Update airflow/providers/mysql/CHANGELOG.rst
---------
Co-authored-by: eladkal <[email protected]>
---
airflow/providers/mysql/hooks/mysql.py | 18 ++++-
airflow/providers/mysql/provider.yaml | 6 +-
docker_tests/test_prod_image.py | 4 +-
docs/apache-airflow/howto/set-up-database.rst | 8 +-
generated/provider_dependencies.json | 1 -
scripts/in_container/verify_providers.py | 4 +-
tests/providers/mysql/hooks/test_mysql.py | 57 --------------
.../mysql/hooks/test_mysql_connector_python.py | 86 ++++++++++++++++++++++
8 files changed, 119 insertions(+), 65 deletions(-)
diff --git a/airflow/providers/mysql/hooks/mysql.py
b/airflow/providers/mysql/hooks/mysql.py
index ea2d912a62..e105d8f96a 100644
--- a/airflow/providers/mysql/hooks/mysql.py
+++ b/airflow/providers/mysql/hooks/mysql.py
@@ -19,13 +19,20 @@
from __future__ import annotations
import json
+import logging
from typing import TYPE_CHECKING, Any, Union
+from airflow.exceptions import AirflowOptionalProviderFeatureException
from airflow.models import Connection
from airflow.providers.common.sql.hooks.sql import DbApiHook
+logger = logging.getLogger(__name__)
+
if TYPE_CHECKING:
- from mysql.connector.abstracts import MySQLConnectionAbstract
+ try:
+ from mysql.connector.abstracts import MySQLConnectionAbstract
+ except ModuleNotFoundError:
+ logger.warning("The package 'mysql-connector-python' is not installed.
Import skipped")
from MySQLdb.connections import Connection as MySQLdbConnection
MySQLConnectionTypes = Union["MySQLdbConnection", "MySQLConnectionAbstract"]
@@ -181,7 +188,14 @@ class MySqlHook(DbApiHook):
return MySQLdb.connect(**conn_config)
if client_name == "mysql-connector-python":
- import mysql.connector
+ try:
+ import mysql.connector
+ except ModuleNotFoundError:
+ raise AirflowOptionalProviderFeatureException(
+ "The pip package 'mysql-connector-python' is not
installed, therefore the connection "
+ "wasn't established. Please, consider using default driver
or pip install the package "
+ "'mysql-connector-python'. Warning! It might cause
dependency conflicts."
+ )
conn_config = self._get_conn_config_mysql_connector_python(conn)
return mysql.connector.connect(**conn_config)
diff --git a/airflow/providers/mysql/provider.yaml
b/airflow/providers/mysql/provider.yaml
index ca2a951939..ca9a30733a 100644
--- a/airflow/providers/mysql/provider.yaml
+++ b/airflow/providers/mysql/provider.yaml
@@ -47,7 +47,6 @@ versions:
dependencies:
- apache-airflow>=2.3.0
- apache-airflow-providers-common-sql>=1.3.1
- - mysql-connector-python>=8.0.11
- mysqlclient>=1.3.6
integrations:
@@ -87,3 +86,8 @@ transfers:
connection-types:
- hook-class-name: airflow.providers.mysql.hooks.mysql.MySqlHook
connection-type: mysql
+
+additional-extras:
+ - name: mysql-connector-python
+ dependencies:
+ - mysql-connector-python>=8.0.11
diff --git a/docker_tests/test_prod_image.py b/docker_tests/test_prod_image.py
index 99ddb4730f..e76f04dfd8 100644
--- a/docker_tests/test_prod_image.py
+++ b/docker_tests/test_prod_image.py
@@ -20,6 +20,7 @@ import json
import os
import subprocess
import tempfile
+from importlib.util import find_spec
from pathlib import Path
import pytest
@@ -161,7 +162,6 @@ class TestPythonPackages:
"grpc": ["grpc", "google.auth", "google_auth_httplib2"],
"hashicorp": ["hvac"],
"ldap": ["ldap"],
- "mysql": ["mysql"],
"postgres": ["psycopg2"],
"pyodbc": ["pyodbc"],
"redis": ["redis"],
@@ -171,6 +171,8 @@ class TestPythonPackages:
"statsd": ["statsd"],
"virtualenv": ["virtualenv"],
}
+ if bool(find_spec("mysql")):
+ PACKAGE_IMPORTS["mysql"] = ["mysql"]
@pytest.mark.skipif(os.environ.get("TEST_SLIM_IMAGE") == "true",
reason="Skipped with slim image")
@pytest.mark.parametrize("package_name,import_names",
PACKAGE_IMPORTS.items())
diff --git a/docs/apache-airflow/howto/set-up-database.rst
b/docs/apache-airflow/howto/set-up-database.rst
index c33f9ac0d9..b9041c8e28 100644
--- a/docs/apache-airflow/howto/set-up-database.rst
+++ b/docs/apache-airflow/howto/set-up-database.rst
@@ -299,7 +299,13 @@ We recommend using the ``mysqlclient`` driver and
specifying it in your SqlAlche
mysql+mysqldb://<user>:<password>@<host>[:<port>]/<dbname>
We also support the ``mysql-connector-python`` driver, which lets you connect
through SSL
-without any cert options provided.
+without any cert options provided. If you wish to use
``mysql-connector-python`` driver, please install it with extras.
+
+.. code-block:: text
+
+ $ pip install mysql-connector-python
+
+The connection string in this case should look like:
.. code-block:: text
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index 0b19116b29..fafcbbb606 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -520,7 +520,6 @@
"deps": [
"apache-airflow-providers-common-sql>=1.3.1",
"apache-airflow>=2.3.0",
- "mysql-connector-python>=8.0.11",
"mysqlclient>=1.3.6"
],
"cross-providers-deps": [
diff --git a/scripts/in_container/verify_providers.py
b/scripts/in_container/verify_providers.py
index 582cd56d2c..e7153548b9 100755
--- a/scripts/in_container/verify_providers.py
+++ b/scripts/in_container/verify_providers.py
@@ -150,12 +150,12 @@ KNOWN_COMMON_DEPRECATED_MESSAGES: set[str] = {
"Implementing implicit namespace packages (as specified in PEP 420) is "
"preferred to `pkg_resources.declare_namespace`",
"This module is deprecated. Please use
`airflow.providers.cncf.kubernetes.operators.pod` instead.",
- "urllib3 (1.26.6) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't
match a supported version!",
- "urllib3 (1.26.9) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't
match a supported version!",
"This operator is deprecated. Please use
`GoogleDisplayVideo360CreateQueryOperator`",
"This operator is deprecated. Please use
`GoogleDisplayVideo360RunQueryOperator`",
"This operator is deprecated. Please use
`GoogleDisplayVideo360RunQuerySensor`",
"This operator is deprecated. Please use
`GoogleDisplayVideo360DownloadReportV2Operator`",
+ "urllib3 (1.26.6) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't
match a supported version!",
+ "urllib3 (1.26.9) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't
match a supported version!",
}
# The set of warning messages generated by direct importing of some deprecated
modules. We should only
diff --git a/tests/providers/mysql/hooks/test_mysql.py
b/tests/providers/mysql/hooks/test_mysql.py
index 1ce211e075..4de9201ff5 100644
--- a/tests/providers/mysql/hooks/test_mysql.py
+++ b/tests/providers/mysql/hooks/test_mysql.py
@@ -182,63 +182,6 @@ class TestMySqlHookConn:
)
-class TestMySqlHookConnMySqlConnectorPython:
- def setup_method(self):
- self.connection = Connection(
- login="login",
- password="password",
- host="host",
- schema="schema",
- extra='{"client": "mysql-connector-python"}',
- )
-
- self.db_hook = MySqlHook()
- self.db_hook.get_connection = mock.Mock()
- self.db_hook.get_connection.return_value = self.connection
-
- @mock.patch("mysql.connector.connect")
- def test_get_conn(self, mock_connect):
- self.db_hook.get_conn()
- assert mock_connect.call_count == 1
- args, kwargs = mock_connect.call_args
- assert args == ()
- assert kwargs["user"] == "login"
- assert kwargs["password"] == "password"
- assert kwargs["host"] == "host"
- assert kwargs["database"] == "schema"
-
- @mock.patch("mysql.connector.connect")
- def test_get_conn_port(self, mock_connect):
- self.connection.port = 3307
- self.db_hook.get_conn()
- assert mock_connect.call_count == 1
- args, kwargs = mock_connect.call_args
- assert args == ()
- assert kwargs["port"] == 3307
-
- @mock.patch("mysql.connector.connect")
- def test_get_conn_allow_local_infile(self, mock_connect):
- extra_dict = self.connection.extra_dejson
- self.connection.extra = json.dumps(extra_dict)
- self.db_hook.local_infile = True
- self.db_hook.get_conn()
- assert mock_connect.call_count == 1
- args, kwargs = mock_connect.call_args
- assert args == ()
- assert kwargs["allow_local_infile"] == 1
-
- @mock.patch("mysql.connector.connect")
- def test_get_ssl_mode(self, mock_connect):
- extra_dict = self.connection.extra_dejson
- extra_dict.update(ssl_disabled=True)
- self.connection.extra = json.dumps(extra_dict)
- self.db_hook.get_conn()
- assert mock_connect.call_count == 1
- args, kwargs = mock_connect.call_args
- assert args == ()
- assert kwargs["ssl_disabled"] == 1
-
-
class MockMySQLConnectorConnection:
DEFAULT_AUTOCOMMIT = "default"
diff --git a/tests/providers/mysql/hooks/test_mysql_connector_python.py
b/tests/providers/mysql/hooks/test_mysql_connector_python.py
new file mode 100644
index 0000000000..7359347000
--- /dev/null
+++ b/tests/providers/mysql/hooks/test_mysql_connector_python.py
@@ -0,0 +1,86 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import json
+from unittest import mock
+
+import pytest
+
+from airflow.models import Connection
+from airflow.providers.mysql.hooks.mysql import MySqlHook
+
+# Make sure that the optional package 'mysql-connector-python' is installed
(which is not by default)
+pytest.importorskip("mysql")
+
+
+class TestMySqlHookConnMySqlConnectorPython:
+ def setup_method(self):
+ self.connection = Connection(
+ login="login",
+ password="password",
+ host="host",
+ schema="schema",
+ extra='{"client": "mysql-connector-python"}',
+ )
+
+ self.db_hook = MySqlHook()
+ self.db_hook.get_connection = mock.Mock()
+ self.db_hook.get_connection.return_value = self.connection
+
+ @mock.patch("mysql.connector.connect")
+ def test_get_conn(self, mock_connect):
+ self.db_hook.get_conn()
+ assert mock_connect.call_count == 1
+ args, kwargs = mock_connect.call_args
+ assert args == ()
+ assert kwargs["user"] == "login"
+ assert kwargs["password"] == "password"
+ assert kwargs["host"] == "host"
+ assert kwargs["database"] == "schema"
+
+ @mock.patch("mysql.connector.connect")
+ def test_get_conn_port(self, mock_connect):
+ self.connection.port = 3307
+ self.db_hook.get_conn()
+ assert mock_connect.call_count == 1
+ args, kwargs = mock_connect.call_args
+ assert args == ()
+ assert kwargs["port"] == 3307
+
+ @mock.patch("mysql.connector.connect")
+ def test_get_conn_allow_local_infile(self, mock_connect):
+ extra_dict = self.connection.extra_dejson
+ self.connection.extra = json.dumps(extra_dict)
+ self.db_hook.local_infile = True
+ self.db_hook.get_conn()
+ assert mock_connect.call_count == 1
+ args, kwargs = mock_connect.call_args
+ assert args == ()
+ assert kwargs["allow_local_infile"] == 1
+
+ @mock.patch("mysql.connector.connect")
+ def test_get_ssl_mode(self, mock_connect):
+ extra_dict = self.connection.extra_dejson
+ extra_dict.update(ssl_disabled=True)
+ self.connection.extra = json.dumps(extra_dict)
+ self.db_hook.get_conn()
+ assert mock_connect.call_count == 1
+ args, kwargs = mock_connect.call_args
+ assert args == ()
+ assert kwargs["ssl_disabled"] == 1
