This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-7-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit dbacacbd4d476da757de148a4e747924c34fd7fe Author: Jarek Potiuk <[email protected]> AuthorDate: Fri Aug 4 12:30:50 2023 +0200 Allows to choose SSL context for SMTP provider (#33075) * Allows to choose SSL context for SMTP provider This change add two options to choose from when SSL SMTP connection is created: * default - for balance between compatibility and security * none - in case compatibility with existing infrastructure is preferred The fallback is: * The Airflow "email", "ssl_context" * "default" * Update airflow/providers/smtp/CHANGELOG.rst Co-authored-by: Ephraim Anierobi <[email protected]> (cherry picked from commit e20325db38fdfdd9db423a345b13d18aab6fe578) --- airflow/providers/smtp/CHANGELOG.rst | 12 +++ airflow/providers/smtp/hooks/smtp.py | 20 +++- airflow/providers/smtp/provider.yaml | 24 +++++ .../configurations-ref.rst | 18 ++++ docs/apache-airflow-providers-smtp/index.rst | 1 + docs/apache-airflow/configurations-ref.rst | 1 + tests/providers/smtp/hooks/test_smtp.py | 103 +++++++++++++++++++-- 7 files changed, 171 insertions(+), 8 deletions(-) diff --git a/airflow/providers/smtp/CHANGELOG.rst b/airflow/providers/smtp/CHANGELOG.rst index 6e0a955e56..527af5d7a3 100644 --- a/airflow/providers/smtp/CHANGELOG.rst +++ b/airflow/providers/smtp/CHANGELOG.rst @@ -27,6 +27,18 @@ Changelog --------- +In case of SMTP SSL connection, the default context now uses "default" context + +The "default" context is Python's ``default_ssl_context`` instead of previously used "none". The +``default_ssl_context`` provides a balance between security and compatibility but in some cases, +when certificates are old, self-signed or misconfigured, it might not work. This can be configured +by setting "ssl_context" in "smtp_provider" configuration of the provider. If it is not explicitly set, +it will default to "email", "ssl_context" setting in Airflow. + +Setting it to "none" brings back the "none" setting that was used in previous versions of the provider, +but it is not recommended due to security reasons ad this setting disables validation +of certificates and allows MITM attacks. + 1.2.0 ..... diff --git a/airflow/providers/smtp/hooks/smtp.py b/airflow/providers/smtp/hooks/smtp.py index 6279ea4f94..c196db1680 100644 --- a/airflow/providers/smtp/hooks/smtp.py +++ b/airflow/providers/smtp/hooks/smtp.py @@ -26,6 +26,7 @@ import collections.abc import os import re import smtplib +import ssl from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -87,7 +88,6 @@ class SmtpHook(BaseHook): if attempt < self.smtp_retry_limit: continue raise AirflowException("Unable to connect to smtp server") - if self.smtp_starttls: self.smtp_client.starttls() if self.smtp_user and self.smtp_password: @@ -109,6 +109,24 @@ class SmtpHook(BaseHook): smtp_kwargs["port"] = self.port smtp_kwargs["timeout"] = self.timeout + if self.use_ssl: + from airflow.configuration import conf + + ssl_context_string = conf.get("smtp_provider", "SSL_CONTEXT", fallback=None) + if ssl_context_string is None: + ssl_context_string = conf.get("email", "SSL_CONTEXT", fallback=None) + if ssl_context_string is None: + ssl_context_string = "default" + if ssl_context_string == "default": + ssl_context = ssl.create_default_context() + elif ssl_context_string == "none": + ssl_context = None + else: + raise RuntimeError( + f"The email.ssl_context configuration variable must " + f"be set to 'default' or 'none' and is '{ssl_context_string}'." + ) + smtp_kwargs["context"] = ssl_context return SMTP(**smtp_kwargs) @classmethod diff --git a/airflow/providers/smtp/provider.yaml b/airflow/providers/smtp/provider.yaml index 9303246608..6132969a2c 100644 --- a/airflow/providers/smtp/provider.yaml +++ b/airflow/providers/smtp/provider.yaml @@ -54,3 +54,27 @@ connection-types: notifications: - airflow.providers.smtp.notifications.smtp.SmtpNotifier + +config: + smtp_provider: + description: "Options for SMTP provider." + options: + ssl_context: + description: | + ssl context to use when using SMTP and IMAP SSL connections. By default, the context is "default" + which sets it to ``ssl.create_default_context()`` which provides the right balance between + compatibility and security, it however requires that certificates in your operating system are + updated and that SMTP/IMAP servers of yours have valid certificates that have corresponding public + keys installed on your machines. You can switch it to "none" if you want to disable checking + of the certificates, but it is not recommended as it allows MITM (man-in-the-middle) attacks + if your infrastructure is not sufficiently secured. It should only be set temporarily while you + are fixing your certificate configuration. This can be typically done by upgrading to newer + version of the operating system you run Airflow components on,by upgrading/refreshing proper + certificates in the OS or by updating certificates for your mail servers. + + If you do not set this option explicitly, it will use Airflow "email.ssl_context" configuration, + but if this configuration is not present, it will use "default" value. + type: string + version_added: 1.3.0 + example: "default" + default: ~ diff --git a/docs/apache-airflow-providers-smtp/configurations-ref.rst b/docs/apache-airflow-providers-smtp/configurations-ref.rst new file mode 100644 index 0000000000..5885c9d91b --- /dev/null +++ b/docs/apache-airflow-providers-smtp/configurations-ref.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/providers-configurations-ref.rst diff --git a/docs/apache-airflow-providers-smtp/index.rst b/docs/apache-airflow-providers-smtp/index.rst index 0cc35fa4e1..f188d1d08a 100644 --- a/docs/apache-airflow-providers-smtp/index.rst +++ b/docs/apache-airflow-providers-smtp/index.rst @@ -34,6 +34,7 @@ :maxdepth: 1 :caption: References + Configuration <configurations-ref> Connection types <connections/smtp> SMTP Notifications <notifications/smtp_notifier_howto_guide> Python API <_api/airflow/providers/smtp/index> diff --git a/docs/apache-airflow/configurations-ref.rst b/docs/apache-airflow/configurations-ref.rst index c4882a8b90..42ff5b9a6b 100644 --- a/docs/apache-airflow/configurations-ref.rst +++ b/docs/apache-airflow/configurations-ref.rst @@ -38,6 +38,7 @@ in the provider's documentation. The pre-installed providers that you may want t * :doc:`Configuration Reference for Celery Provider <apache-airflow-providers-celery:configurations-ref>` * :doc:`Configuration Reference for Apache Hive Provider <apache-airflow-providers-apache-hive:configurations-ref>` * :doc:`Configuration Reference for CNCF Kubernetes Provider <apache-airflow-providers-cncf-kubernetes:configurations-ref>` +* :doc:`Configuration Reference for SMTP Provider <apache-airflow-providers-smtp:configurations-ref>` .. note:: For more information see :doc:`/howto/set-config`. diff --git a/tests/providers/smtp/hooks/test_smtp.py b/tests/providers/smtp/hooks/test_smtp.py index 10fe5df673..3b8032df18 100644 --- a/tests/providers/smtp/hooks/test_smtp.py +++ b/tests/providers/smtp/hooks/test_smtp.py @@ -30,6 +30,7 @@ from airflow.models import Connection from airflow.providers.smtp.hooks.smtp import SmtpHook from airflow.utils import db from airflow.utils.session import create_session +from tests.test_utils.config import conf_vars smtplib_string = "airflow.providers.smtp.hooks.smtp.smtplib" @@ -75,13 +76,16 @@ class TestSmtpHook: ) @patch(smtplib_string) - def test_connect_and_disconnect(self, mock_smtplib): + @patch("ssl.create_default_context") + def test_connect_and_disconnect(self, create_default_context, mock_smtplib): mock_conn = _create_fake_smtp(mock_smtplib) with SmtpHook(): pass - - mock_smtplib.SMTP_SSL.assert_called_once_with(host="smtp_server_address", port=465, timeout=30) + assert create_default_context.called + mock_smtplib.SMTP_SSL.assert_called_once_with( + host="smtp_server_address", port=465, timeout=30, context=create_default_context.return_value + ) mock_conn.login.assert_called_once_with("smtp_user", "smtp_password") assert mock_conn.close.call_count == 1 @@ -201,12 +205,90 @@ class TestSmtpHook: @patch("smtplib.SMTP_SSL") @patch("smtplib.SMTP") - def test_send_mime_ssl(self, mock_smtp, mock_smtp_ssl): + @patch("ssl.create_default_context") + def test_send_mime_ssl(self, create_default_context, mock_smtp, mock_smtp_ssl): mock_smtp_ssl.return_value = Mock() with SmtpHook() as smtp_hook: smtp_hook.send_email_smtp(to="to", subject="subject", html_content="content", from_email="from") assert not mock_smtp.called - mock_smtp_ssl.assert_called_once_with(host="smtp_server_address", port=465, timeout=30) + assert create_default_context.called + mock_smtp_ssl.assert_called_once_with( + host="smtp_server_address", port=465, timeout=30, context=create_default_context.return_value + ) + + @patch("smtplib.SMTP_SSL") + @patch("smtplib.SMTP") + @patch("ssl.create_default_context") + def test_send_mime_ssl_none_email_context(self, create_default_context, mock_smtp, mock_smtp_ssl): + mock_smtp_ssl.return_value = Mock() + with conf_vars({("smtp", "smtp_ssl"): "True", ("email", "ssl_context"): "none"}): + with SmtpHook() as smtp_hook: + smtp_hook.send_email_smtp( + to="to", subject="subject", html_content="content", from_email="from" + ) + assert not mock_smtp.called + assert not create_default_context.called + mock_smtp_ssl.assert_called_once_with(host="smtp_server_address", port=465, timeout=30, context=None) + + @patch("smtplib.SMTP_SSL") + @patch("smtplib.SMTP") + @patch("ssl.create_default_context") + def test_send_mime_ssl_none_smtp_provider_context(self, create_default_context, mock_smtp, mock_smtp_ssl): + mock_smtp_ssl.return_value = Mock() + with conf_vars({("smtp", "smtp_ssl"): "True", ("smtp_provider", "ssl_context"): "none"}): + with SmtpHook() as smtp_hook: + smtp_hook.send_email_smtp( + to="to", subject="subject", html_content="content", from_email="from" + ) + assert not mock_smtp.called + assert not create_default_context.called + mock_smtp_ssl.assert_called_once_with(host="smtp_server_address", port=465, timeout=30, context=None) + + @patch("smtplib.SMTP_SSL") + @patch("smtplib.SMTP") + @patch("ssl.create_default_context") + def test_send_mime_ssl_none_smtp_provider_default_email_context( + self, create_default_context, mock_smtp, mock_smtp_ssl + ): + mock_smtp_ssl.return_value = Mock() + with conf_vars( + { + ("smtp", "smtp_ssl"): "True", + ("email", "ssl_context"): "default", + ("smtp_provider", "ssl_context"): "none", + } + ): + with SmtpHook() as smtp_hook: + smtp_hook.send_email_smtp( + to="to", subject="subject", html_content="content", from_email="from" + ) + assert not mock_smtp.called + assert not create_default_context.called + mock_smtp_ssl.assert_called_once_with(host="smtp_server_address", port=465, timeout=30, context=None) + + @patch("smtplib.SMTP_SSL") + @patch("smtplib.SMTP") + @patch("ssl.create_default_context") + def test_send_mime_ssl_default_smtp_provider_none_email_context( + self, create_default_context, mock_smtp, mock_smtp_ssl + ): + mock_smtp_ssl.return_value = Mock() + with conf_vars( + { + ("smtp", "smtp_ssl"): "True", + ("email", "ssl_context"): "none", + ("smtp_provider", "ssl_context"): "default", + } + ): + with SmtpHook() as smtp_hook: + smtp_hook.send_email_smtp( + to="to", subject="subject", html_content="content", from_email="from" + ) + assert not mock_smtp.called + assert create_default_context.called + mock_smtp_ssl.assert_called_once_with( + host="smtp_server_address", port=465, timeout=30, context=create_default_context.return_value + ) @patch("smtplib.SMTP_SSL") @patch("smtplib.SMTP") @@ -269,7 +351,10 @@ class TestSmtpHook: @patch("airflow.models.connection.Connection") @patch("smtplib.SMTP_SSL") - def test_send_mime_custom_timeout_retrylimit(self, mock_smtp_ssl, connection_mock): + @patch("ssl.create_default_context") + def test_send_mime_custom_timeout_retrylimit( + self, create_default_context, mock_smtp_ssl, connection_mock + ): mock_smtp_ssl().sendmail.side_effect = smtplib.SMTPServerDisconnected() custom_retry_limit = 10 custom_timeout = 60 @@ -287,6 +372,10 @@ class TestSmtpHook: with pytest.raises(smtplib.SMTPServerDisconnected): smtp_hook.send_email_smtp(to="to", subject="subject", html_content="content") mock_smtp_ssl.assert_any_call( - host=fake_conn.host, port=fake_conn.port, timeout=fake_conn.extra_dejson["timeout"] + host=fake_conn.host, + port=fake_conn.port, + timeout=fake_conn.extra_dejson["timeout"], + context=create_default_context.return_value, ) + assert create_default_context.called assert mock_smtp_ssl().sendmail.call_count == 10
