This is an automated email from the ASF dual-hosted git repository. beto pushed a commit to branch remove_retry_dep in repository https://gitbox.apache.org/repos/asf/superset.git
commit c63a2a334bb27a66ff9ae1a08faebb0aff07cdee Author: Beto Dealmeida <[email protected]> AuthorDate: Mon Jul 19 15:50:18 2021 -0700 chore: remove retry dep in favor of backoff --- setup.cfg | 2 +- setup.py | 1 - superset/reports/notifications/slack.py | 7 +++-- superset/tasks/schedules.py | 13 ++++++---- superset/tasks/slack_util.py | 6 +++-- superset/utils/retries.py | 46 +++++++++++++++++++++++++++++++++ superset/utils/webdriver.py | 4 +-- 7 files changed, 66 insertions(+), 13 deletions(-) diff --git a/setup.cfg b/setup.cfg index afb034b..4ba32fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ combine_as_imports = true include_trailing_comma = true line_length = 88 known_first_party = superset -known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,contextlib2,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pathlib [...] +known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,contextlib2,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pathlib [...] multi_line_output = 3 order_by_type = false diff --git a/setup.py b/setup.py index f5b062b..e00229d 100644 --- a/setup.py +++ b/setup.py @@ -104,7 +104,6 @@ setup( "pyyaml>=5.4", "PyJWT>=1.7.1, <2", "redis", - "retry>=0.9.2", "selenium>=3.141.0", "simplejson>=3.15.0", "slackclient==2.5.0", # PINNED! slack changes file upload api in the future versions diff --git a/superset/reports/notifications/slack.py b/superset/reports/notifications/slack.py index 4fd9fb3..598defc 100644 --- a/superset/reports/notifications/slack.py +++ b/superset/reports/notifications/slack.py @@ -20,8 +20,8 @@ import logging from io import IOBase from typing import Optional, Union +import backoff from flask_babel import gettext as __ -from retry.api import retry from slack import WebClient from slack.errors import SlackApiError, SlackClientError @@ -29,6 +29,7 @@ from superset import app from superset.models.reports import ReportRecipientType from superset.reports.notifications.base import BaseNotification from superset.reports.notifications.exceptions import NotificationError +from superset.utils.retries import linear_backoff logger = logging.getLogger(__name__) @@ -79,7 +80,9 @@ class SlackNotification(BaseNotification): # pylint: disable=too-few-public-met return self._content.screenshot return None - @retry(SlackApiError, delay=10, backoff=2, tries=5) + @backoff.on_exception( + linear_backoff, SlackApiError, delay=10, multiplier=2, max_tries=5 + ) def send(self) -> None: file = self._get_inline_file() title = self._content.name diff --git a/superset/tasks/schedules.py b/superset/tasks/schedules.py index c6e8e96..96e8c2d 100644 --- a/superset/tasks/schedules.py +++ b/superset/tasks/schedules.py @@ -45,7 +45,6 @@ from celery.app.task import Task from dateutil.tz import tzlocal from flask import current_app, render_template, url_for from flask_babel import gettext as __ -from retry.api import retry_call from selenium.common.exceptions import WebDriverException from selenium.webdriver import chrome, firefox from selenium.webdriver.remote.webdriver import WebDriver @@ -69,6 +68,7 @@ from superset.tasks.alerts.validator import get_validator_function from superset.tasks.slack_util import deliver_slack_msg from superset.utils.celery import session_scope from superset.utils.core import get_email_address_list, send_email_smtp +from superset.utils.retries import retry_call from superset.utils.screenshots import ChartScreenshot, WebDriverProxy from superset.utils.urls import get_url_path @@ -230,7 +230,7 @@ def destroy_webdriver( # This is some very flaky code in selenium. Hence the retries # and catch-all exceptions try: - retry_call(driver.close, tries=2) + retry_call(driver.close, max_tries=2) except Exception: # pylint: disable=broad-except pass try: @@ -271,7 +271,10 @@ def deliver_dashboard( # pylint: disable=too-many-locals # This is buggy in certain selenium versions with firefox driver get_element = getattr(driver, "find_element_by_class_name") element = retry_call( - get_element, fargs=["grid-container"], tries=2, delay=EMAIL_PAGE_RENDER_WAIT + get_element, + fargs=["grid-container"], + max_tries=2, + interval=EMAIL_PAGE_RENDER_WAIT, ) try: @@ -420,8 +423,8 @@ def _get_slice_visualization( element = retry_call( driver.find_element_by_class_name, fargs=["chart-container"], - tries=2, - delay=EMAIL_PAGE_RENDER_WAIT, + max_tries=2, + interval=EMAIL_PAGE_RENDER_WAIT, ) try: diff --git a/superset/tasks/slack_util.py b/superset/tasks/slack_util.py index 5440698..4681db4 100644 --- a/superset/tasks/slack_util.py +++ b/superset/tasks/slack_util.py @@ -21,17 +21,19 @@ import logging from io import IOBase from typing import cast, Optional, Union +import backoff from flask import current_app -from retry.api import retry from slack import WebClient from slack.errors import SlackApiError from slack.web.slack_response import SlackResponse +from superset.utils.retries import linear_backoff + # Globals logger = logging.getLogger("tasks.slack_util") -@retry(SlackApiError, delay=10, backoff=2, tries=5) [email protected]_exception(linear_backoff, SlackApiError, delay=2, multiplier=2, max_tries=5) def deliver_slack_msg( slack_channel: str, subject: str, diff --git a/superset/utils/retries.py b/superset/utils/retries.py new file mode 100644 index 0000000..e4c85ae --- /dev/null +++ b/superset/utils/retries.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Any, Callable, Dict, Generator, List, Optional, Type + +import backoff + + +def linear_backoff(delay: int = 0, multiplier: int = 1) -> Generator[int, None, None]: + """ + Linear backoff generator for the ``backoff`` library. + """ + while True: + yield delay + delay *= multiplier + + +def retry_call( + func: Callable[..., Any], + strategy: Callable[..., Generator[int, None, None]] = backoff.constant, + exception: Type[Exception] = Exception, + fargs: Optional[List[Any]] = None, + fkwargs: Optional[Dict[str, Any]] = None, + *args: Any, + **kwargs: Any +) -> Any: + """ + Retry a given call. + """ + decorated = backoff.on_exception(strategy, exception, *args, **kwargs)(func) + fkwargs = fkwargs or {} + return decorated(*fargs, **fkwargs) diff --git a/superset/utils/webdriver.py b/superset/utils/webdriver.py index eee8874..732f52a 100644 --- a/superset/utils/webdriver.py +++ b/superset/utils/webdriver.py @@ -20,7 +20,6 @@ from time import sleep from typing import Any, Dict, Optional, Tuple, TYPE_CHECKING from flask import current_app -from retry.api import retry_call from selenium.common.exceptions import ( StaleElementReferenceException, TimeoutException, @@ -33,6 +32,7 @@ from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait from superset.extensions import machine_auth_provider_factory +from superset.utils.retries import retry_call WindowSize = Tuple[int, int] logger = logging.getLogger(__name__) @@ -85,7 +85,7 @@ class WebDriverProxy: # This is some very flaky code in selenium. Hence the retries # and catch-all exceptions try: - retry_call(driver.close, tries=tries) + retry_call(driver.close, max_tries=tries) except Exception: # pylint: disable=broad-except pass try:
