This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 10df7436f3 Add `ChimeWebhookHook` (#31939)
10df7436f3 is described below
commit 10df7436f3bcdf8465b8cea536511d6126ae377c
Author: Cary <[email protected]>
AuthorDate: Mon Jun 26 22:41:14 2023 -0700
Add `ChimeWebhookHook` (#31939)
* Create ChimeWebhook Hook to send messages to chime room from Airflow.
---------
Co-authored-by: Beata Kossakowska
<[email protected]>
Co-authored-by: Beata Kossakowska <[email protected]>
Co-authored-by: eladkal <[email protected]>
Co-authored-by: Jarek Potiuk <[email protected]>
---
airflow/providers/amazon/aws/hooks/chime.py | 116 +++++++++++++++++++++
airflow/providers/amazon/provider.yaml | 11 ++
dev/breeze/tests/test_selective_checks.py | 14 +--
.../connections/chime.rst | 60 +++++++++++
docs/apache-airflow-providers-amazon/index.rst | 1 +
.../aws/Amazon-Chime-light-bg.png | Bin 0 -> 9969 bytes
generated/provider_dependencies.json | 2 +
tests/providers/amazon/aws/hooks/test_chime.py | 105 +++++++++++++++++++
8 files changed, 302 insertions(+), 7 deletions(-)
diff --git a/airflow/providers/amazon/aws/hooks/chime.py
b/airflow/providers/amazon/aws/hooks/chime.py
new file mode 100644
index 0000000000..e2ac1a35a0
--- /dev/null
+++ b/airflow/providers/amazon/aws/hooks/chime.py
@@ -0,0 +1,116 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""This module contains a web hook for Chime."""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from airflow.exceptions import AirflowException
+from airflow.providers.http.hooks.http import HttpHook
+
+
+class ChimeWebhookHook(HttpHook):
+ """Interact with Chime Web Hooks to create notifications.
+
+ .. warning:: This hook is only designed to work with web hooks and not
chat bots.
+
+ :param chime_conn_id: Chime connection ID with Endpoint as
"https://hooks.chime.aws"; and
+ the webhook token in the form of
```{webhook.id}?token{webhook.token}```
+
+ """
+
+ conn_name_attr = "chime_conn_id"
+ default_conn_name = "chime_default"
+ conn_type = "chime"
+ hook_name = "Chime Web Hook"
+
+ def __init__(
+ self,
+ chime_conn_id: str,
+ *args: Any,
+ **kwargs: Any,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ self.webhook_endpoint = self._get_webhook_endpoint(chime_conn_id)
+
+ def _get_webhook_endpoint(self, conn_id: str) -> str:
+ """
+ Given a Chime conn_id return the default webhook endpoint.
+
+ :param conn_id: The provided connection ID.
+ :return: Endpoint(str) for chime webhook.
+ """
+ conn = self.get_connection(conn_id)
+ token = conn.get_password()
+ if token is None:
+ raise AirflowException("Webhook token field is missing and is
required.")
+ url = conn.schema + "://" + conn.host
+ endpoint = url + token
+ # Check to make sure the endpoint matches what Chime expects
+ if not re.match(r"^[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+$", token):
+ raise AirflowException(
+ "Expected Chime webhook token in the form of
'{webhook.id}?token={webhook.token}'."
+ )
+
+ return endpoint
+
+ def _build_chime_payload(self, message: str) -> str:
+ """
+ Builds payload for Chime and ensures messages do not exceed max length
allowed.
+
+ :param message: The message you want to send to your Chime room.
+ (max 4096 characters)
+ """
+ payload: dict[str, Any] = {}
+ # We need to make sure that the message does not exceed the max length
for Chime
+ if len(message) > 4096:
+ raise AirflowException("Chime message must be 4096 characters or
less.")
+
+ payload["Content"] = message
+ return json.dumps(payload)
+
+ def send_message(self, message: str) -> None:
+ """Execute calling the Chime webhook endpoint.
+
+ :param message: The message you want to send to your Chime room.
+ (max 4096 characters)
+
+ """
+ chime_payload = self._build_chime_payload(message)
+ self.run(
+ endpoint=self.webhook_endpoint, data=chime_payload,
headers={"Content-type": "application/json"}
+ )
+
+ @classmethod
+ def get_ui_field_behaviour(cls) -> dict[str, Any]:
+ """Returns custom field behaviour to only get what is needed for Chime
webhooks to function."""
+ return {
+ "hidden_fields": ["login", "port", "extra"],
+ "relabeling": {
+ "host": "Chime Webhook Endpoint",
+ "password": "Webhook Token",
+ },
+ "placeholders": {
+ "schema": "https",
+ "host": "hooks.chime.aws/incomingwebhook/",
+ "password": "T00000000?token=XXXXXXXXXXXXXXXXXXXXXXXX",
+ },
+ }
diff --git a/airflow/providers/amazon/provider.yaml
b/airflow/providers/amazon/provider.yaml
index 223bc553ef..5439f9c8cb 100644
--- a/airflow/providers/amazon/provider.yaml
+++ b/airflow/providers/amazon/provider.yaml
@@ -61,6 +61,7 @@ versions:
dependencies:
- apache-airflow>=2.4.0
- apache-airflow-providers-common-sql>=1.3.1
+ - apache-airflow-providers-http
- boto3>=1.24.0
- asgiref
# watchtower 3 has been released end Jan and introduced breaking change
across the board that might
@@ -84,6 +85,10 @@ integrations:
how-to-guide:
- /docs/apache-airflow-providers-amazon/operators/athena.rst
tags: [aws]
+ - integration-name: Amazon Chime
+ external-doc-url: https://aws.amazon.com/chime/
+ logo: /integration-logos/aws/Amazon-Chime-light-bg.png
+ tags: [aws]
- integration-name: Amazon CloudFormation
external-doc-url: https://aws.amazon.com/cloudformation/
logo: /integration-logos/aws/[email protected]
@@ -407,6 +412,9 @@ hooks:
- integration-name: Amazon Athena
python-modules:
- airflow.providers.amazon.aws.hooks.athena
+ - integration-name: Amazon Chime
+ python-modules:
+ - airflow.providers.amazon.aws.hooks.chime
- integration-name: Amazon DynamoDB
python-modules:
- airflow.providers.amazon.aws.hooks.dynamodb
@@ -624,11 +632,14 @@ extra-links:
connection-types:
- hook-class-name: airflow.providers.amazon.aws.hooks.base_aws.AwsGenericHook
connection-type: aws
+ - hook-class-name: airflow.providers.amazon.aws.hooks.chime.ChimeWebhookHook
+ connection-type: chime
- hook-class-name: airflow.providers.amazon.aws.hooks.emr.EmrHook
connection-type: emr
- hook-class-name:
airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook
connection-type: redshift
+
secrets-backends:
- airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend
-
airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index dac17ce815..0081b6a2a7 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -191,7 +191,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"tests/providers/http/file.py",
),
{
- "affected-providers-list-as-string": "airbyte apache.livy "
+ "affected-providers-list-as-string": "airbyte amazon
apache.livy "
"dbt.cloud dingding discord http",
"all-python-versions": "['3.8']",
"all-python-versions-list-as-string": "3.8",
@@ -200,11 +200,11 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
- "run-amazon-tests": "false",
+ "run-amazon-tests": "true",
"docs-build": "true",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
- "parallel-test-types-list-as-string": "Always "
+ "parallel-test-types-list-as-string": "Providers[amazon]
Always "
"Providers[airbyte,apache.livy,dbt.cloud,dingding,discord,http]",
},
id="Helm tests, http and all relevant providers, kubernetes
tests and "
@@ -311,7 +311,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
("airflow/providers/amazon/__init__.py",),
{
"affected-providers-list-as-string": "amazon apache.hive
cncf.kubernetes "
- "common.sql exasol ftp google imap "
+ "common.sql exasol ftp google http imap "
"mongo mysql postgres salesforce ssh",
"all-python-versions": "['3.8']",
"all-python-versions-list-as-string": "3.8",
@@ -325,7 +325,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"upgrade-to-newer-dependencies": "false",
"run-amazon-tests": "true",
"parallel-test-types-list-as-string": "Providers[amazon]
Always "
-
"Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,imap,"
+
"Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http,imap,"
"mongo,mysql,postgres,salesforce,ssh] Providers[google]",
},
id="Providers tests run including amazon tests if amazon provider
files changed",
@@ -353,7 +353,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
("airflow/providers/amazon/file.py",),
{
"affected-providers-list-as-string": "amazon apache.hive
cncf.kubernetes "
- "common.sql exasol ftp google imap "
+ "common.sql exasol ftp google http imap "
"mongo mysql postgres salesforce ssh",
"all-python-versions": "['3.8']",
"all-python-versions-list-as-string": "3.8",
@@ -368,7 +368,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Providers[amazon]
Always "
"Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,"
- "imap,mongo,mysql,postgres,salesforce,ssh] Providers[google]",
+ "http,imap,mongo,mysql,postgres,salesforce,ssh]
Providers[google]",
},
id="Providers tests run including amazon tests if amazon provider
files changed",
),
diff --git a/docs/apache-airflow-providers-amazon/connections/chime.rst
b/docs/apache-airflow-providers-amazon/connections/chime.rst
new file mode 100644
index 0000000000..94a356f81d
--- /dev/null
+++ b/docs/apache-airflow-providers-amazon/connections/chime.rst
@@ -0,0 +1,60 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+.. _howto/connection:chime:
+
+Amazon Chime Connection
+==========================
+
+The Chime connection works with calling Chime webhooks to send messages to a
chime room.
+
+Authenticating to Amazon Chime
+---------------------------------
+When a webhook is created in a Chime room a token will be included in the url
for authentication.
+
+
+Default Connection IDs
+----------------------
+
+The default connection ID is ``chime_default``.
+
+Configuring the Connection
+--------------------------
+Chime Webhook Endpoint:
+ Specify the entire url or the base of the url for the service.
+
+
+Chime Webhook token:
+ The token for authentication including the webhook ID.
+
+Schema:
+ Whether or not the endpoint should be http or https
+
+
+Examples
+--------
+
+**Connection**
+
+* **Chime Webhook Endpoint**: hooks.chime.aws
+* **Chime Webhook Token**:
+
+.. code-block:: text
+
+ abceasd-3423-a1237-ffff-000cccccccc?token=somechimetoken
+
+* **Schema**: https
diff --git a/docs/apache-airflow-providers-amazon/index.rst
b/docs/apache-airflow-providers-amazon/index.rst
index dff699d55a..e0b148425c 100644
--- a/docs/apache-airflow-providers-amazon/index.rst
+++ b/docs/apache-airflow-providers-amazon/index.rst
@@ -134,6 +134,7 @@ Dependent package
`apache-airflow-providers-exasol
<https://airflow.apache.org/docs/apache-airflow-providers-exasol>`_
``exasol``
`apache-airflow-providers-ftp
<https://airflow.apache.org/docs/apache-airflow-providers-ftp>`_
``ftp``
`apache-airflow-providers-google
<https://airflow.apache.org/docs/apache-airflow-providers-google>`_
``google``
+`apache-airflow-providers-http
<https://airflow.apache.org/docs/apache-airflow-providers-http>`_
``http``
`apache-airflow-providers-imap
<https://airflow.apache.org/docs/apache-airflow-providers-imap>`_
``imap``
`apache-airflow-providers-mongo
<https://airflow.apache.org/docs/apache-airflow-providers-mongo>`_
``mongo``
`apache-airflow-providers-salesforce
<https://airflow.apache.org/docs/apache-airflow-providers-salesforce>`_
``salesforce``
diff --git a/docs/integration-logos/aws/Amazon-Chime-light-bg.png
b/docs/integration-logos/aws/Amazon-Chime-light-bg.png
new file mode 100644
index 0000000000..bd412e9815
Binary files /dev/null and
b/docs/integration-logos/aws/Amazon-Chime-light-bg.png differ
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index 2a1ae40e3f..2755e9f08b 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -20,6 +20,7 @@
"amazon": {
"deps": [
"apache-airflow-providers-common-sql>=1.3.1",
+ "apache-airflow-providers-http",
"apache-airflow>=2.4.0",
"asgiref",
"asgiref",
@@ -40,6 +41,7 @@
"exasol",
"ftp",
"google",
+ "http",
"imap",
"mongo",
"salesforce",
diff --git a/tests/providers/amazon/aws/hooks/test_chime.py
b/tests/providers/amazon/aws/hooks/test_chime.py
new file mode 100644
index 0000000000..d5130964f4
--- /dev/null
+++ b/tests/providers/amazon/aws/hooks/test_chime.py
@@ -0,0 +1,105 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from airflow.exceptions import AirflowException
+from airflow.models import Connection
+from airflow.providers.amazon.aws.hooks.chime import ChimeWebhookHook
+from airflow.utils import db
+
+
+class TestChimeWebhookHook:
+
+ _config = {
+ "chime_conn_id": "default-chime-webhook",
+ "webhook_endpoint":
"incomingwebhooks/abcd-1134-ZeDA?token=somechimetoken-111",
+ "message": "your message here",
+ }
+
+ expected_payload_dict = {
+ "Content": _config["message"],
+ }
+
+ expected_payload = json.dumps(expected_payload_dict)
+
+ def setup_method(self):
+ db.merge_conn(
+ Connection(
+ conn_id="default-chime-webhook",
+ conn_type="chime",
+ host="hooks.chime.aws/incomingwebhooks/",
+ password="abcd-1134-ZeDA?token=somechimetoken111",
+ schema="https",
+ )
+ )
+ db.merge_conn(
+ Connection(
+ conn_id="chime-bad-url",
+ conn_type="chime",
+ host="https://hooks.chime.aws/";,
+ password="somebadurl",
+ schema="https",
+ )
+ )
+
+ def test_get_webhook_endpoint_invalid_url(self):
+ # Given
+
+ # When/Then
+ expected_message = r"Expected Chime webhook token in the form"
+ with pytest.raises(AirflowException, match=expected_message):
+ ChimeWebhookHook(chime_conn_id="chime-bad-url")
+
+ def test_get_webhook_endpoint_conn_id(self):
+ # Given
+ conn_id = "default-chime-webhook"
+ hook = ChimeWebhookHook(chime_conn_id=conn_id)
+ expected_webhook_endpoint = (
+
"https://hooks.chime.aws/incomingwebhooks/abcd-1134-ZeDA?token=somechimetoken111";
+ )
+
+ # When
+ webhook_endpoint = hook._get_webhook_endpoint(conn_id)
+
+ # Then
+ assert webhook_endpoint == expected_webhook_endpoint
+
+ def test_build_chime_payload(self):
+ # Given
+ hook = ChimeWebhookHook(self._config["chime_conn_id"])
+ message = self._config["message"]
+ # When
+ payload = hook._build_chime_payload(message)
+ # Then
+ assert self.expected_payload == payload
+
+ def test_build_chime_payload_message_length(self):
+ # Given
+ self._config.copy()
+ # create message over the character limit
+ message = "c" * 4097
+ hook = ChimeWebhookHook(self._config["chime_conn_id"])
+
+ # When/Then
+ expected_message = "Chime message must be 4096 characters or less."
+ with pytest.raises(AirflowException, match=expected_message):
+ hook._build_chime_payload(message)
