This is an automated email from the ASF dual-hosted git repository. vavila pushed a commit to branch chore/cache-slack-channels in repository https://gitbox.apache.org/repos/asf/superset.git
commit 3ef17d08b43301443ac7034cf3411dde08d98aa4 Author: Vitor Avila <[email protected]> AuthorDate: Wed Mar 5 19:12:20 2025 -0300 chore: Cache Slack channels --- superset/reports/api.py | 6 ++- superset/utils/slack.py | 101 ++++++++++++++++++++++++++++++------------------ 2 files changed, 68 insertions(+), 39 deletions(-) diff --git a/superset/reports/api.py b/superset/reports/api.py index c9b5d76916..48f3ca365c 100644 --- a/superset/reports/api.py +++ b/superset/reports/api.py @@ -577,8 +577,12 @@ class ReportScheduleRestApi(BaseSupersetModelRestApi): search_string = [params.get("search_string", "").strip()] types = params.get("types", []) exact_match = params.get("exact_match", False) + force = params.get("force", False) channels = get_channels_with_search( - search_string=search_string, types=types, exact_match=exact_match + search_string=search_string, + types=types, + exact_match=exact_match, + force=force, ) return self.response(200, result=channels) except SupersetException as ex: diff --git a/superset/utils/slack.py b/superset/utils/slack.py index df13c6b8e5..051d57ee7a 100644 --- a/superset/utils/slack.py +++ b/superset/utils/slack.py @@ -17,7 +17,7 @@ import logging -from typing import Optional +from typing import Any, Optional from flask import current_app from slack_sdk import WebClient @@ -25,7 +25,9 @@ from slack_sdk.errors import SlackApiError from superset import feature_flag_manager from superset.exceptions import SupersetException +from superset.extensions import cache_manager from superset.reports.schemas import SlackChannelSchema +from superset.utils import cache as cache_util from superset.utils.backports import StrEnum logger = logging.getLogger(__name__) @@ -47,59 +49,82 @@ def get_slack_client() -> WebClient: return WebClient(token=token, proxy=current_app.config["SLACK_PROXY"]) +@cache_util.memoized_func( + key="slack_conversations_list", + cache=cache_manager.cache, +) +def get_channels(limit: int, extra_params: dict[str, Any]) -> list[SlackChannelSchema]: + """ + Retrieves a list of all conversations accessible by the bot + from the Slack API, and caches results (to avoid rate limits). + + The Slack API does not provide search so to apply a search use + get_channels_with_search instead. + """ + client = get_slack_client() + channel_schema = SlackChannelSchema() + channels: list[SlackChannelSchema] = [] + cursor = None + + while True: + response = client.conversations_list( + limit=limit, cursor=cursor, exclude_archived=True, **extra_params + ) + channels.extend( + channel_schema.load(channel) for channel in response.data["channels"] + ) + cursor = response.data.get("response_metadata", {}).get("next_cursor") + if not cursor: + break + + return channels + + def get_channels_with_search( search_string: list[str] | None = None, limit: int = 999, types: Optional[list[SlackChannelTypes]] = None, exact_match: bool = False, + force: bool = False, ) -> list[SlackChannelSchema]: """ The slack api is paginated but does not include search, so we need to fetch all channels and filter them ourselves This will search by slack name or id """ + extra_params = {} + extra_params["types"] = ",".join(types) if types else None try: - client = get_slack_client() - channel_schema = SlackChannelSchema() - channels: list[SlackChannelSchema] = [] - cursor = None - extra_params = {} - extra_params["types"] = ",".join(types) if types else None - - while True: - response = client.conversations_list( - limit=limit, cursor=cursor, exclude_archived=True, **extra_params - ) - channels.extend( - channel_schema.load(channel) for channel in response.data["channels"] - ) - cursor = response.data.get("response_metadata", {}).get("next_cursor") - if not cursor: - break - - # The search string can be multiple channels separated by commas - if search_string: - channels = [ - channel - for channel in channels - if any( - ( - search.lower() == channel["name"].lower() - or search.lower() == channel["id"].lower() - if exact_match - else ( - search.lower() in channel["name"].lower() - or search.lower() in channel["id"].lower() - ) - ) - for search in search_string - ) - ] - return channels + channels = get_channels( + limit=limit, + extra_params=extra_params, + force=force, + cache_timeout=86400, + ) except (SlackClientError, SlackApiError) as ex: raise SupersetException(f"Failed to list channels: {ex}") from ex + # The search string can be multiple channels separated by commas + if search_string: + channels = [ + channel + for channel in channels + if any( + ( + search.lower() == channel["name"].lower() + or search.lower() == channel["id"].lower() + if exact_match + else ( + search.lower() in channel["name"].lower() + or search.lower() in channel["id"].lower() + ) + ) + for search in search_string + ) + ] + return channels + def should_use_v2_api() -> bool: if not feature_flag_manager.is_feature_enabled("ALERT_REPORT_SLACK_V2"):
