This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-7-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 38fc9cd823feafd8ec61d5d5c7eddb9e9162f755 Author: Jarek Potiuk <[email protected]> AuthorDate: Fri Aug 4 14:28:35 2023 +0200 Allows to choose SSL context for IMAP provider (#33108) * Allows to choose SSL context for IMAP provider This change add two options to choose from when SSL IMAP connection is created: * default - for balance between compatibility and security * none - in case compatibility with existing infrastructure is preferred The fallback is: * The Airflow "email", "ssl_context" * "default" Co-authored-by: Ephraim Anierobi <[email protected]> (cherry picked from commit 52ca7bfc988f4c9b608f544bc3e9524fd6564639) --- airflow/providers/imap/CHANGELOG.rst | 12 ++++ airflow/providers/imap/hooks/imap.py | 37 +++++++++--- airflow/providers/imap/provider.yaml | 23 ++++++++ .../configurations-ref.rst | 18 ++++++ docs/apache-airflow-providers-imap/index.rst | 1 + docs/apache-airflow/configurations-ref.rst | 1 + tests/providers/imap/hooks/test_imap.py | 69 +++++++++++++++++++++- 7 files changed, 150 insertions(+), 11 deletions(-) diff --git a/airflow/providers/imap/CHANGELOG.rst b/airflow/providers/imap/CHANGELOG.rst index 24cf83e5b8..dc5ae11b22 100644 --- a/airflow/providers/imap/CHANGELOG.rst +++ b/airflow/providers/imap/CHANGELOG.rst @@ -26,6 +26,18 @@ Changelog --------- +In case of IMAP SSL connection, the context now uses the "default" context + +The "default" context is Python's ``default_ssl_context`` instead of previously used "none". The +``default_ssl_context`` provides a balance between security and compatibility but in some cases, +when certificates are old, self-signed or misconfigured, it might not work. This can be configured +by setting "ssl_context" in "imap" configuration of the provider. If it is not explicitly set, +it will default to "email", "ssl_context" setting in Airflow. + +Setting it to "none" brings back the "none" setting that was used in previous versions of the provider, +but it is not recommended due to security reasons and this setting disables validation +of certificates and allows MITM attacks. + 3.2.2 ..... diff --git a/airflow/providers/imap/hooks/imap.py b/airflow/providers/imap/hooks/imap.py index 4a00e6965a..6523c61cf7 100644 --- a/airflow/providers/imap/hooks/imap.py +++ b/airflow/providers/imap/hooks/imap.py @@ -26,6 +26,7 @@ import email import imaplib import os import re +import ssl from typing import Any, Iterable from airflow.exceptions import AirflowException @@ -78,16 +79,34 @@ class ImapHook(BaseHook): return self def _build_client(self, conn: Connection) -> imaplib.IMAP4_SSL | imaplib.IMAP4: - IMAP: type[imaplib.IMAP4_SSL] | type[imaplib.IMAP4] - if conn.extra_dejson.get("use_ssl", True): - IMAP = imaplib.IMAP4_SSL - else: - IMAP = imaplib.IMAP4 - - if conn.port: - mail_client = IMAP(conn.host, conn.port) + mail_client: imaplib.IMAP4_SSL | imaplib.IMAP4 + use_ssl = conn.extra_dejson.get("use_ssl", True) + if use_ssl: + from airflow.configuration import conf + + ssl_context_string = conf.get("imap", "SSL_CONTEXT", fallback=None) + if ssl_context_string is None: + ssl_context_string = conf.get("email", "SSL_CONTEXT", fallback=None) + if ssl_context_string is None: + ssl_context_string = "default" + if ssl_context_string == "default": + ssl_context = ssl.create_default_context() + elif ssl_context_string == "none": + ssl_context = None + else: + raise RuntimeError( + f"The email.ssl_context configuration variable must " + f"be set to 'default' or 'none' and is '{ssl_context_string}'." + ) + if conn.port: + mail_client = imaplib.IMAP4_SSL(conn.host, conn.port, ssl_context=ssl_context) + else: + mail_client = imaplib.IMAP4_SSL(conn.host, ssl_context=ssl_context) else: - mail_client = IMAP(conn.host) + if conn.port: + mail_client = imaplib.IMAP4(conn.host, conn.port) + else: + mail_client = imaplib.IMAP4(conn.host) return mail_client diff --git a/airflow/providers/imap/provider.yaml b/airflow/providers/imap/provider.yaml index 997880d4bd..e7d427a813 100644 --- a/airflow/providers/imap/provider.yaml +++ b/airflow/providers/imap/provider.yaml @@ -62,3 +62,26 @@ hooks: connection-types: - hook-class-name: airflow.providers.imap.hooks.imap.ImapHook connection-type: imap + +config: + imap: + description: "Options for IMAP provider." + options: + ssl_context: + description: | + ssl context to use when using SMTP and IMAP SSL connections. By default, the context is "default" + which sets it to ``ssl.create_default_context()`` which provides the right balance between + compatibility and security, it however requires that certificates in your operating system are + updated and that SMTP/IMAP servers of yours have valid certificates that have corresponding public + keys installed on your machines. You can switch it to "none" if you want to disable checking + of the certificates, but it is not recommended as it allows MITM (man-in-the-middle) attacks + if your infrastructure is not sufficiently secured. It should only be set temporarily while you + are fixing your certificate configuration. This can be typically done by upgrading to newer + version of the operating system you run Airflow components on,by upgrading/refreshing proper + certificates in the OS or by updating certificates for your mail servers. + If you do not set this option explicitly, it will use Airflow "email.ssl_context" configuration, + but if this configuration is not present, it will use "default" value. + type: string + version_added: 3.3.0 + example: "default" + default: ~ diff --git a/docs/apache-airflow-providers-imap/configurations-ref.rst b/docs/apache-airflow-providers-imap/configurations-ref.rst new file mode 100644 index 0000000000..5885c9d91b --- /dev/null +++ b/docs/apache-airflow-providers-imap/configurations-ref.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/providers-configurations-ref.rst diff --git a/docs/apache-airflow-providers-imap/index.rst b/docs/apache-airflow-providers-imap/index.rst index ef8e28b9e0..60f86f1ece 100644 --- a/docs/apache-airflow-providers-imap/index.rst +++ b/docs/apache-airflow-providers-imap/index.rst @@ -34,6 +34,7 @@ :maxdepth: 1 :caption: References + Configuration <configurations-ref> Connection types <connections/imap> Python API <_api/airflow/providers/imap/index> diff --git a/docs/apache-airflow/configurations-ref.rst b/docs/apache-airflow/configurations-ref.rst index 42ff5b9a6b..43173fe6ac 100644 --- a/docs/apache-airflow/configurations-ref.rst +++ b/docs/apache-airflow/configurations-ref.rst @@ -39,6 +39,7 @@ in the provider's documentation. The pre-installed providers that you may want t * :doc:`Configuration Reference for Apache Hive Provider <apache-airflow-providers-apache-hive:configurations-ref>` * :doc:`Configuration Reference for CNCF Kubernetes Provider <apache-airflow-providers-cncf-kubernetes:configurations-ref>` * :doc:`Configuration Reference for SMTP Provider <apache-airflow-providers-smtp:configurations-ref>` +* :doc:`Configuration Reference for IMAP Provider <apache-airflow-providers-imap:configurations-ref>` .. note:: For more information see :doc:`/howto/set-config`. diff --git a/tests/providers/imap/hooks/test_imap.py b/tests/providers/imap/hooks/test_imap.py index c2be420be6..5e57d7196e 100644 --- a/tests/providers/imap/hooks/test_imap.py +++ b/tests/providers/imap/hooks/test_imap.py @@ -27,6 +27,7 @@ from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.imap.hooks.imap import ImapHook from airflow.utils import db +from tests.test_utils.config import conf_vars imaplib_string = "airflow.providers.imap.hooks.imap.imaplib" open_string = "airflow.providers.imap.hooks.imap.open" @@ -85,13 +86,77 @@ class TestImapHook: ) @patch(imaplib_string) - def test_connect_and_disconnect(self, mock_imaplib): + @patch("ssl.create_default_context") + def test_connect_and_disconnect(self, create_default_context, mock_imaplib): mock_conn = _create_fake_imap(mock_imaplib) with ImapHook(): pass - mock_imaplib.IMAP4_SSL.assert_called_once_with("imap_server_address", 1993) + assert create_default_context.called + mock_imaplib.IMAP4_SSL.assert_called_once_with( + "imap_server_address", 1993, ssl_context=create_default_context.return_value + ) + mock_conn.login.assert_called_once_with("imap_user", "imap_password") + assert mock_conn.logout.call_count == 1 + + @patch(imaplib_string) + @patch("ssl.create_default_context") + def test_connect_and_disconnect_imap_ssl_context_none(self, create_default_context, mock_imaplib): + mock_conn = _create_fake_imap(mock_imaplib) + + with conf_vars({("imap", "ssl_context"): "none"}): + with ImapHook(): + pass + + assert not create_default_context.called + mock_imaplib.IMAP4_SSL.assert_called_once_with("imap_server_address", 1993, ssl_context=None) + mock_conn.login.assert_called_once_with("imap_user", "imap_password") + assert mock_conn.logout.call_count == 1 + + @patch(imaplib_string) + @patch("ssl.create_default_context") + def test_connect_and_disconnect_imap_ssl_context_default(self, create_default_context, mock_imaplib): + mock_conn = _create_fake_imap(mock_imaplib) + + with conf_vars({("imap", "ssl_context"): "default"}): + with ImapHook(): + pass + + assert create_default_context.called + mock_imaplib.IMAP4_SSL.assert_called_once_with( + "imap_server_address", 1993, ssl_context=create_default_context.return_value + ) + mock_conn.login.assert_called_once_with("imap_user", "imap_password") + assert mock_conn.logout.call_count == 1 + + @patch(imaplib_string) + @patch("ssl.create_default_context") + def test_connect_and_disconnect_email_ssl_context_none(self, create_default_context, mock_imaplib): + mock_conn = _create_fake_imap(mock_imaplib) + + with conf_vars({("email", "ssl_context"): "none"}): + with ImapHook(): + pass + + assert not create_default_context.called + mock_imaplib.IMAP4_SSL.assert_called_once_with("imap_server_address", 1993, ssl_context=None) + mock_conn.login.assert_called_once_with("imap_user", "imap_password") + assert mock_conn.logout.call_count == 1 + + @patch(imaplib_string) + @patch("ssl.create_default_context") + def test_connect_and_disconnect_imap_ssl_context_override(self, create_default_context, mock_imaplib): + mock_conn = _create_fake_imap(mock_imaplib) + + with conf_vars({("email", "ssl_context"): "none", ("imap", "ssl_context"): "default"}): + with ImapHook(): + pass + + assert create_default_context.called + mock_imaplib.IMAP4_SSL.assert_called_once_with( + "imap_server_address", 1993, ssl_context=create_default_context.return_value + ) mock_conn.login.assert_called_once_with("imap_user", "imap_password") assert mock_conn.logout.call_count == 1
