This is an automated email from the ASF dual-hosted git repository. eladkal pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new 42f581f0eae Add OAuth 2 / XOAUTH2 support via `auth_type` & token/credential extras (#53554) 42f581f0eae is described below commit 42f581f0eaeb6f2669cbb024e452558c5251105e Author: Aaron Chen <nail...@gmail.com> AuthorDate: Tue Aug 5 00:34:54 2025 -0700 Add OAuth 2 / XOAUTH2 support via `auth_type` & token/credential extras (#53554) * support auth_type=oauth2 for smtp * add auth_type to notifications * add ut for build_xoauth2_string & send_mime_email * add test for smtp * add test for notifications * add ui widgets & property for oauth2 * add extra parameters to the docs * fix CI test * fix CI test #2 * fix CI test #3 * expose `build_xoauth2_string` & add fallback for old cores * rollback email.py and test_email.py & refactor smtp.py * refactor the doc to enrich examples and explnations --- providers/smtp/docs/connections/smtp.rst | 270 ++++++++++++++++++--- .../smtp/src/airflow/providers/smtp/hooks/smtp.py | 86 ++++++- .../airflow/providers/smtp/notifications/smtp.py | 4 +- providers/smtp/tests/unit/smtp/hooks/test_smtp.py | 58 ++++- .../tests/unit/smtp/notifications/test_smtp.py | 20 ++ 5 files changed, 396 insertions(+), 42 deletions(-) diff --git a/providers/smtp/docs/connections/smtp.rst b/providers/smtp/docs/connections/smtp.rst index 62e8548d0af..153eb46d9e1 100644 --- a/providers/smtp/docs/connections/smtp.rst +++ b/providers/smtp/docs/connections/smtp.rst @@ -15,71 +15,267 @@ specific language governing permissions and limitations under the License. - - .. _howto/connection:smtp: SMTP Connection =============== -The SMTP connection type enables integrations with the SMTP client. +The **SMTP** connection type enables integrations such as +:class:`~airflow.providers.smtp.hooks.smtp.SmtpHook`. + +.. note:: + The legacy helper in ``airflow.utils.email`` is **scheduled for deprecation** + and will be removed in a future major release. + Please migrate to :class:`~airflow.providers.smtp.hooks.smtp.SmtpHook` + or other provider-level utilities for sending emails. + +Default Connection ID +--------------------- + +The default ID is ``smtp_default`` when no ``conn_id`` is supplied. Authenticating to SMTP ---------------------- -Authenticate to the SMTP client with the login and password field. -Use standard `SMTP authentication -<https://docs.python.org/3/library/smtplib.html>`_ +Two methods are supported: -Default Connection IDs ----------------------- +* **Basic** – traditional *username + password*. +* **OAuth 2 / XOAUTH2** – bearer-token based, required by Gmail API, + Microsoft 365 / Outlook.com and other modern providers. -Hooks, operators, and sensors related to SMTP use ``smtp_default`` by default. +If you omit credentials the hook attempts an **anonymous** session, accepted +only by open-relay test servers. Configuring the Connection -------------------------- -Login - Specify the username used for the SMTP client. +**Login** + Username (for example ``u...@example.com``). + +**Password** + Password or *app-specific* password. + Ignored when ``auth_type="oauth2"``. + +**Host** + SMTP server hostname (for example ``smtp.gmail.com``). + +**Port** + Port number. Defaults to **465** when SSL is enabled, otherwise **587**. + +**Extra** *(optional – JSON)* + Additional parameters. + + **General** + + * ``from_email`` – Default **From:** address. + * ``disable_ssl`` *(bool)* – Disable SSL/TLS entirely. Default ``false``. + * ``disable_tls`` *(bool)* – Skip ``STARTTLS``. Default ``false``. + * ``timeout`` *(int)* – Socket timeout (seconds). Default ``30``. + * ``retry_limit`` *(int)* – Connection attempts before raising. Default ``5``. + * ``ssl_context`` – ``"default"`` | ``"none"`` + See :ref:`howto/connection:smtp:ssl-context`. + + **Templating** + + * ``subject_template`` – File path for custom subject. + * ``html_content_template`` – File path for custom HTML body. -Password - Specify the password used for the SMTP client. + **Authentication** -Host - Specify the SMTP host url. + * ``auth_type`` – ``"basic"`` *(default)* | ``"oauth2"`` + * ``access_token`` – OAuth 2 bearer (one-hour). + * ``client_id`` / ``client_secret`` – Credentials for token refresh. + (auto-defaults to Google or Microsoft). + * ``tenant_id`` – Azure tenant (default ``"common"``). + * ``scope`` – OAuth scope -Port - Specify the SMTP port to connect to. The default depends on the whether you use ssl or not. + * **Gmail**: ``https://mail.google.com/`` + * **Outlook (Graph)**: ``https://outlook.office.com/.default`` -Extra (optional) - Specify the extra parameters (as json dictionary) +.. _howto/connection:smtp:ssl-context: - * ``from_email``: The email address from which you want to send the email. - * ``disable_ssl``: If set to true, then a non-ssl connection is being used. Default is false. Also note that changing the ssl option also influences the default port being used. - * ``timeout``: The SMTP connection creation timeout in seconds. Default is 30. - * ``disable_tls``: By default the SMTP connection is created in TLS mode. Set to false to disable tls mode. - * ``retry_limit``: How many attempts to connect to the server before raising an exception. Default is 5. - * ``ssl_context``: Can be "default" or "none". Only valid when SSL is used. The "default" context provides a balance between security and compatibility, "none" is not recommended - as it disables validation of certificates and allow MITM attacks, and is only needed in case your certificates are wrongly configured in your system. If not specified, defaults are taken from the - "smtp_provider", "ssl_context" configuration with the fallback to "email". "ssl_context" configuration. If none of it is specified, "default" is used. - * ``subject_template``: A path to a file containing the email subject template. - * ``html_content_template``: A path to a file containing the email html content template. +SSL / TLS Notes +^^^^^^^^^^^^^^^ -When specifying the connection in environment variable you should specify -it using URI syntax. +* ``ssl_context="default"`` – reasonable trust store & secure ciphers *(recommended)* +* ``ssl_context="none"`` – **disables certificate validation**; use only for + local testing with self-signed certificates. -Note that all components of the URI should be URL-encoded. +Examples +-------- -For example: +Basic Auth — SendGrid (STARTTLS 587) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: bash - export AIRFLOW_CONN_SMTP_DEFAULT='smtp://username:passw...@smtp.sendgrid.net:587' + export AIRFLOW_CONN_SMTP_SENDGRID='smtp://apikey:sg.your_api_...@smtp.sendgrid.net:587?\ + disable_ssl=true&\ + from_email=you%40example.com' + +OAuth 2 — Gmail (access token, STARTTLS 587) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + export AIRFLOW_CONN_SMTP_GMAIL='smtp://your.name%40gmail....@smtp.gmail.com:587?\ + auth_type=oauth2&\ + access_token=ya29.<URL_ENCODED_TOKEN>&\ + from_email=your.name%40gmail.com&\ + disable_ssl=true' + + +OAuth 2 — Gmail (SSL 465) +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + export AIRFLOW_CONN_SMTP_GMAIL_SSL='smtp://your.name%40gmail....@smtp.gmail.com:465?\ + auth_type=oauth2&\ + access_token=ya29.<URL_ENCODED_TOKEN>&\ + from_email=your.name%40gmail.com&\ + disable_tls=true' + +OAuth 2 — Microsoft 365 (client credentials 587) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + export AIRFLOW_CONN_SMTP_M365='smtp://user%40contoso....@smtp.office365.com:587?\ + auth_type=oauth2&\ + client_id=YOUR_APP_ID&\ + client_secret=YOUR_SECRET&\ + tenant_id=YOUR_TENANT_ID&\ + scope=https%3A%2F%2Foutlook.office.com%2F.default&\ + disable_ssl=true' + -Another example for connecting via a non-SSL connection. +OAuth2 — Microsoft 365 (client-credential flow) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: bash - export AIRFLOW_CONN_SMTP_NOSSL='smtp://username:passw...@smtp.sendgrid.net:587?disable_ssl=true' + export AIRFLOW_CONN_SMTP_M365='smtp://u...@contoso.com@smtp.office365.com:587?\ + auth_type=oauth2&\ + client_id=YOUR_APP_ID&\ + client_secret=YOUR_SECRET&\ + tenant_id=YOUR_TENANT_ID&\ + scope=https%3A%2F%2Foutlook.office.com%2F.default' + + +Troubleshooting +~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: 25 35 40 + + * - **Error message** + - **Likely cause** + - **Fix** + * - ``SSL WRONG_VERSION_NUMBER`` + - Port 587 but the connection starts with SSL (no **STARTTLS**). + - Add ``disable_ssl=true`` **or** switch to port 465. + * - ``STARTTLS required`` + - Port 465 yet the hook still issues ``STARTTLS``. + - Add ``disable_tls=true`` **or** switch to port 587. + * - ``530 Authentication Required`` + - Access-token expired or missing the ``https://mail.google.com/`` scope. + - Generate a fresh token. + * - ``550 From address not verified`` + - Sender identity not verified at the provider **or** ``from_email`` mismatch. + - Verify the sender / domain and ensure ``from_email`` exactly matches it. + + +Programmatic creation +^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + from airflow.models.connection import Connection + + conn = Connection( + conn_id="smtp_gmail_token", + conn_type="smtp", + host="smtp.gmail.com", + login="m...@gmail.com", + extra={"auth_type": "oauth2", "access_token": "ya29.a0AfB..."}, + ) + print(conn.test_connection()) + +URI encoding +^^^^^^^^^^^^ + +When creating connections programmatically or via the CLI, ensure that + +When fields contain special characters (``/``, ``@``, ``:`` …), URL-encode them, +for example via +:py:meth:`airflow.models.connection.Connection.get_uri`. + +CLI creation (Gmail OAuth 2) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Prefer environment variables for portability, but you can also create the +connection via **CLI**: + +.. code-block:: bash + + airflow connections add smtp_gmail_oauth2 \ + --conn-type smtp \ + --conn-host smtp.gmail.com \ + --conn-port 587 \ + --conn-login '<YOUR_EMAIL>@gmail.com' \ + --conn-extra '{ + "from_email": "<YOUR_EMAIL>@gmail.com", + "auth_type": "oauth2", + "access_token": "<YOUR_OAUTH2_ACCESS_TOKEN>", + "disable_ssl": "true" + }' + +.. note:: + The ``[smtp]`` section in ``airflow.cfg`` is used by the **core** + e-mail helper slated for deprecation. + When you switch to :class:`~airflow.providers.smtp.hooks.smtp.SmtpHook` + *and* supply a ``smtp_conn_id``, the hook's connection settings take + precedence and the global ``[smtp]`` options may be ignored. + +Using ``SmtpHook`` in a DAG +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + :linenos: + + from datetime import datetime + + from airflow import DAG + from airflow.operators.python import PythonOperator + from airflow.providers.smtp.hooks.smtp import SmtpHook + + + def gmail_oauth2_test(): + with SmtpHook(smtp_conn_id="smtp_gmail_oauth2") as hook: + hook.send_email_smtp( + to="recipi...@example.com", + subject="[Airflow→Gmail] OAuth2 OK", + html_content="<h3>Gmail XOAUTH2 works 🎉</h3>", + ) + + + with DAG( + dag_id="test_gmail_oauth2", + start_date=datetime(2025, 7, 1), + schedule=None, + catchup=False, + tags=["example"], + ) as dag: + PythonOperator( + task_id="send_mail", + python_callable=gmail_oauth2_test, + ) + +---- -Note that you can set the port regardless of whether you choose to use ssl or not. The above examples show default ports for SSL and Non-SSL connections. +.. seealso:: + * :class:`airflow.providers.smtp.hooks.smtp.SmtpHook` + * Google OAuth 2.0 for Gmail – https://developers.google.com/identity/protocols/oauth2 + * Microsoft Graph OAuth 2.0 – https://learn.microsoft.com/graph/auth/ diff --git a/providers/smtp/src/airflow/providers/smtp/hooks/smtp.py b/providers/smtp/src/airflow/providers/smtp/hooks/smtp.py index 245e2772a3e..53982137dab 100644 --- a/providers/smtp/src/airflow/providers/smtp/hooks/smtp.py +++ b/providers/smtp/src/airflow/providers/smtp/hooks/smtp.py @@ -46,6 +46,11 @@ if TYPE_CHECKING: from airflow.models.connection import Connection # type: ignore[assignment] +def build_xoauth2_string(username: str, token: str) -> str: + """Local fallback for older Airflow cores (≤2.11).""" + return f"user={username}\x01auth=Bearer {token}\x01\x01" + + class SmtpHook(BaseHook): """ This hook connects to a mail server by using the smtp protocol. @@ -62,11 +67,13 @@ class SmtpHook(BaseHook): conn_type = "smtp" hook_name = "SMTP" - def __init__(self, smtp_conn_id: str = default_conn_name) -> None: + def __init__(self, smtp_conn_id: str = default_conn_name, auth_type: str = "basic") -> None: super().__init__() self.smtp_conn_id = smtp_conn_id self.smtp_connection: Connection | None = None self.smtp_client: smtplib.SMTP_SSL | smtplib.SMTP | None = None + self._auth_type = auth_type + self._access_token: str | None = None def __enter__(self) -> SmtpHook: return self.get_conn() @@ -98,7 +105,21 @@ class SmtpHook(BaseHook): else: if self.smtp_starttls: self.smtp_client.starttls() - if self.smtp_user and self.smtp_password: + + # choose auth + if self._auth_type == "oauth2": + if not self._access_token: + self._access_token = self._get_oauth2_token() + user_identity = self.smtp_user or self.from_email + if user_identity is None: + raise AirflowException( + "smtp_user or from_email must be set for OAuth2 authentication" + ) + self.smtp_client.auth( + "XOAUTH2", + lambda _=None: build_xoauth2_string(user_identity, self._access_token), + ) + elif self.smtp_user and self.smtp_password: self.smtp_client.login(self.smtp_user, self.smtp_password) break @@ -136,7 +157,7 @@ class SmtpHook(BaseHook): from flask_appbuilder.fieldwidgets import BS3TextFieldWidget from flask_babel import lazy_gettext from wtforms import BooleanField, IntegerField, StringField - from wtforms.validators import NumberRange + from wtforms.validators import NumberRange, any_of return { "from_email": StringField(lazy_gettext("From email"), widget=BS3TextFieldWidget()), @@ -160,6 +181,18 @@ class SmtpHook(BaseHook): "html_content_template": StringField( lazy_gettext("Path to the html content template"), widget=BS3TextFieldWidget() ), + "auth_type": StringField( + lazy_gettext("Auth Type"), + widget=BS3TextFieldWidget(), + description="basic or oauth2", + validators=[any_of(["basic", "oauth2"])], + default="basic", + ), + "access_token": StringField(lazy_gettext("Access Token"), widget=BS3TextFieldWidget()), + "client_id": StringField(lazy_gettext("Client ID"), widget=BS3TextFieldWidget()), + "client_secret": StringField(lazy_gettext("Client Secret"), widget=BS3TextFieldWidget()), + "tenant_id": StringField(lazy_gettext("Tenant ID"), widget=BS3TextFieldWidget()), + "scope": StringField(lazy_gettext("Scope"), widget=BS3TextFieldWidget()), } def test_connection(self) -> tuple[bool, str]: @@ -353,6 +386,40 @@ class SmtpHook(BaseHook): pattern = r"\s*[,;]\s*" return re.split(pattern, addresses) + def _get_oauth2_token(self) -> str: + """ + Return a valid OAuth 2.0 access-token. + + If access_token provided in connection extra, then use it. + Else, try MSAL client-credential flow when client_id & client_secret exist. + """ + extra = self.conn.extra_dejson + + if token := extra.get("access_token"): + return token + + client_id = extra.get("client_id") + client_secret = extra.get("client_secret") + tenant_id = extra.get("tenant_id", "common") + scope = extra.get("scope", "https://outlook.office.com/.default") + + if client_id and client_secret: + from msal import ConfidentialClientApplication + + app = ConfidentialClientApplication( + client_id=client_id, + client_credential=client_secret, + authority=f"https://login.microsoftonline.com/{tenant_id}", + ) + result = app.acquire_token_for_client(scopes=[scope]) + if "access_token" not in result: + raise AirflowException(f"Unable to obtain access token: {result.get('error_description')}") + return result["access_token"] + + raise AirflowException( + "auth_type='oauth2' but neither 'access_token' nor client credentials supplied in connection extra." + ) + @property def conn(self) -> Connection: if not self.smtp_connection: @@ -407,6 +474,19 @@ class SmtpHook(BaseHook): def ssl_context(self) -> str | None: return self.conn.extra_dejson.get("ssl_context") + @property + def auth_type(self) -> str: + return self.conn.extra_dejson.get("auth_type", self._auth_type) + + @property + def access_token(self) -> str | None: + if self._access_token: + return self._access_token + token = self.conn.extra_dejson.get("access_token") + if token: + self._access_token = token + return self._access_token + @staticmethod def _read_template(template_path: str) -> str: """ diff --git a/providers/smtp/src/airflow/providers/smtp/notifications/smtp.py b/providers/smtp/src/airflow/providers/smtp/notifications/smtp.py index e7c2496718d..49d463f8819 100644 --- a/providers/smtp/src/airflow/providers/smtp/notifications/smtp.py +++ b/providers/smtp/src/airflow/providers/smtp/notifications/smtp.py @@ -77,6 +77,7 @@ class SmtpNotifier(BaseNotifier): mime_charset: str = "utf-8", custom_headers: dict[str, Any] | None = None, smtp_conn_id: str = SmtpHook.default_conn_name, + auth_type: str = "basic", *, template: str | None = None, ): @@ -92,6 +93,7 @@ class SmtpNotifier(BaseNotifier): self.custom_headers = custom_headers self.subject = subject self.html_content = html_content + self.auth_type = auth_type if self.html_content is None and template is not None: self.html_content = self._read_template(template) @@ -102,7 +104,7 @@ class SmtpNotifier(BaseNotifier): @cached_property def hook(self) -> SmtpHook: """Smtp Events Hook.""" - return SmtpHook(smtp_conn_id=self.smtp_conn_id) + return SmtpHook(smtp_conn_id=self.smtp_conn_id, auth_type=self.auth_type) def notify(self, context): """Send a email via smtp server.""" diff --git a/providers/smtp/tests/unit/smtp/hooks/test_smtp.py b/providers/smtp/tests/unit/smtp/hooks/test_smtp.py index c3c33a0b3f3..a1d2b91ef6a 100644 --- a/providers/smtp/tests/unit/smtp/hooks/test_smtp.py +++ b/providers/smtp/tests/unit/smtp/hooks/test_smtp.py @@ -26,8 +26,9 @@ from unittest.mock import Mock, patch import pytest +from airflow.exceptions import AirflowException from airflow.models import Connection -from airflow.providers.smtp.hooks.smtp import SmtpHook +from airflow.providers.smtp.hooks.smtp import SmtpHook, build_xoauth2_string smtplib_string = "airflow.providers.smtp.hooks.smtp.smtplib" @@ -72,6 +73,17 @@ class TestSmtpHook: extra=json.dumps(dict(disable_ssl=True, from_email="from")), ) ) + create_connection_without_db( + Connection( + conn_id="smtp_oauth2", + conn_type="smtp", + host="smtp_server_address", + login="smtp_user", + password="smtp_password", + port=587, + extra=json.dumps(dict(disable_ssl=True, from_email="from", access_token="test-token")), + ) + ) @patch(smtplib_string) @patch("ssl.create_default_context") @@ -374,3 +386,47 @@ class TestSmtpHook: ) assert create_default_context.called assert mock_smtp_ssl().sendmail.call_count == 10 + + @patch(smtplib_string) + def test_oauth2_auth_called(self, mock_smtplib): + mock_conn = _create_fake_smtp(mock_smtplib, use_ssl=False) + + with SmtpHook(smtp_conn_id="smtp_oauth2", auth_type="oauth2") as smtp_hook: + smtp_hook.send_email_smtp( + to="t...@example.com", + subject="subject", + html_content="content", + from_email="from", + ) + + assert mock_conn.auth.called + args, _ = mock_conn.auth.call_args + assert args[0] == "XOAUTH2" + assert build_xoauth2_string("smtp_user", "test-token") == args[1]() + + @patch(smtplib_string) + def test_oauth2_missing_token_raises(self, mock_smtplib, create_connection_without_db): + mock_conn = _create_fake_smtp(mock_smtplib, use_ssl=False) + + create_connection_without_db( + Connection( + conn_id="smtp_oauth2_empty", + conn_type="smtp", + host="smtp_server_address", + login="smtp_user", + password="smtp_password", + port=587, + extra=json.dumps(dict(disable_ssl=True, from_email="from")), + ) + ) + + with pytest.raises(AirflowException): + with SmtpHook(smtp_conn_id="smtp_oauth2_empty", auth_type="oauth2") as h: + h.send_email_smtp( + to="t...@example.com", + subject="subject", + html_content="content", + from_email="from", + ) + + assert not mock_conn.auth.called diff --git a/providers/smtp/tests/unit/smtp/notifications/test_smtp.py b/providers/smtp/tests/unit/smtp/notifications/test_smtp.py index 73c1de253a5..22092b17a0b 100644 --- a/providers/smtp/tests/unit/smtp/notifications/test_smtp.py +++ b/providers/smtp/tests/unit/smtp/notifications/test_smtp.py @@ -177,3 +177,23 @@ class TestSmtpNotifier: mime_charset="utf-8", custom_headers=None, ) + + @mock.patch("airflow.providers.smtp.notifications.smtp.SmtpHook") + def test_notifier_oauth2_passes_auth_type(self, mock_smtphook_hook, dag_maker): + with dag_maker("test_notifier_oauth2") as dag: + EmptyOperator(task_id="task1") + + notifier = SmtpNotifier( + from_email="test_sen...@test.com", + to="test_reci...@test.com", + auth_type="oauth2", + subject="subject", + html_content="body", + ) + + notifier({"dag": dag}) + + mock_smtphook_hook.assert_called_once_with( + smtp_conn_id="smtp_default", + auth_type="oauth2", + )