This is an automated email from the ASF dual-hosted git repository. taragolis pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new cc901cdaab Replace to broad exceptions into the dev (#38489) cc901cdaab is described below commit cc901cdaab287742e966c23d9bbe9b198aef2f46 Author: Andrey Anshin <andrey.ans...@taragol.is> AuthorDate: Fri Apr 5 00:27:56 2024 +0400 Replace to broad exceptions into the dev (#38489) * Replace to broad exceptions into the Dev * Add changes according to the review Co-authored-by: Jarek Potiuk <ja...@potiuk.com> Co-authored-by: Tzu-ping Chung <uranu...@gmail.com> * RuntimeError -> SystemExit if detect it run in __main__ Co-authored-by: Tzu-ping Chung <uranu...@gmail.com> --------- Co-authored-by: Jarek Potiuk <ja...@potiuk.com> Co-authored-by: Tzu-ping Chung <uranu...@gmail.com> --- .../commands/release_management_commands.py | 7 ++++--- dev/breeze/src/airflow_breeze/global_constants.py | 2 +- dev/breeze/src/airflow_breeze/utils/docs_publisher.py | 8 ++++++-- dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py | 3 ++- dev/breeze/src/airflow_breeze/utils/packages.py | 15 ++++++++------- docs/conf.py | 2 +- docs/exts/docs_build/docs_builder.py | 16 ---------------- docs/exts/operators_and_hooks_ref.py | 4 ++-- docs/exts/provider_yaml_utils.py | 5 +++-- hatch_build.py | 3 ++- scripts/ci/pre_commit/generate_pypi_readme.py | 2 +- scripts/ci/pre_commit/insert_extras.py | 4 ++-- scripts/ci/pre_commit/json_schema.py | 4 ++-- scripts/ci/pre_commit/replace_bad_characters.py | 2 +- scripts/ci/pre_commit/update_chart_dependencies.py | 4 ++-- scripts/ci/pre_commit/update_versions.py | 2 +- scripts/in_container/run_provider_yaml_files_check.py | 7 ++++--- scripts/in_container/update_quarantined_test_status.py | 4 ++-- scripts/tools/check_if_limited_dependencies.py | 2 +- scripts/tools/list-integrations.py | 2 +- 20 files changed, 46 insertions(+), 52 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index e5ad5b0422..0fae67e485 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -1143,7 +1143,7 @@ def _get_all_providers_in_dist( for file in DIST_DIR.glob(f"{filename_prefix}*.tar.gz"): matched = filename_pattern.match(file.name) if not matched: - raise Exception(f"Cannot parse provider package name from {file.name}") + raise SystemExit(f"Cannot parse provider package name from {file.name}") provider_package_id = matched.group(1).replace("_", ".") yield provider_package_id @@ -1168,7 +1168,7 @@ def get_all_providers_in_dist(package_format: str, install_selected_providers: s ) ) else: - raise Exception(f"Unknown package format {package_format}") + raise SystemExit(f"Unknown package format {package_format}") if install_selected_providers: filter_list = install_selected_providers.split(",") return [provider for provider in all_found_providers if provider in filter_list] @@ -1293,11 +1293,12 @@ def install_provider_packages( if dependency not in chunk: chunk.append(dependency) if len(list_of_all_providers) != total_num_providers: - raise Exception( + msg = ( f"Total providers {total_num_providers} is different " f"than {len(list_of_all_providers)} (just to be sure" f" no rounding errors crippled in)" ) + raise RuntimeError(msg) parallelism = min(parallelism, len(provider_chunks)) with ci_group(f"Installing providers in {parallelism} chunks"): all_params = [f"Chunk {n}" for n in range(parallelism)] diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index ebd9a29d8e..2c8d9523bf 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -364,7 +364,7 @@ def get_airflow_version(): airflow_version = line.split()[2][1:-1] break if airflow_version == "unknown": - raise Exception("Unable to determine Airflow version") + raise RuntimeError("Unable to determine Airflow version") return airflow_version diff --git a/dev/breeze/src/airflow_breeze/utils/docs_publisher.py b/dev/breeze/src/airflow_breeze/utils/docs_publisher.py index 5150eead2b..1d7a88e245 100644 --- a/dev/breeze/src/airflow_breeze/utils/docs_publisher.py +++ b/dev/breeze/src/airflow_breeze/utils/docs_publisher.py @@ -58,7 +58,11 @@ class DocsPublisher: @property def _current_version(self): if not self.is_versioned: - raise Exception("This documentation package is not versioned") + msg = ( + "This documentation package is not versioned. " + "Make sure to add version in `provider.yaml` for the package." + ) + raise RuntimeError(msg) if self.package_name == "apache-airflow": return get_airflow_version() if self.package_name.startswith("apache-airflow-providers-"): @@ -66,7 +70,7 @@ class DocsPublisher: return provider["versions"][0] if self.package_name == "helm-chart": return chart_version() - return Exception(f"Unsupported package: {self.package_name}") + raise SystemExit(f"Unsupported package: {self.package_name}") @property def _publish_dir(self) -> str: diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py index 3812277d4b..ef94500900 100644 --- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py @@ -92,7 +92,8 @@ def get_architecture_string_for_urls() -> str: return "amd64" if architecture == Architecture.ARM: return "arm64" - raise Exception(f"The architecture {architecture} is not supported when downloading kubernetes tools!") + msg = f"The architecture {architecture} is not supported when downloading kubernetes tools!" + raise SystemExit(msg) def _download_with_retries(num_tries, path, tool, url): diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 94ec01b419..68e91b2489 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -132,8 +132,9 @@ def refresh_provider_metadata_from_yaml_file(provider_yaml_path: Path): try: jsonschema.validate(provider, schema=schema) - except jsonschema.ValidationError: - raise Exception(f"Unable to parse: {provider_yaml_path}.") + except jsonschema.ValidationError as ex: + msg = f"Unable to parse: {provider_yaml_path}. Original error {type(ex).__name__}: {ex}" + raise RuntimeError(msg) except ImportError: # we only validate the schema if jsonschema is available. This is needed for autocomplete # to not fail with import error if jsonschema is not installed @@ -176,12 +177,12 @@ def validate_provider_info_with_runtime_schema(provider_info: dict[str, Any]) -> try: jsonschema.validate(provider_info, schema=schema) except jsonschema.ValidationError as ex: - get_console().print("[red]Provider info not validated against runtime schema[/]") - raise Exception( - "Error when validating schema. The schema must be compatible with " - "airflow/provider_info.schema.json.", - ex, + get_console().print( + "[red]Error when validating schema. The schema must be compatible with " + "[bold]'airflow/provider_info.schema.json'[/bold].\n" + f"Original exception [bold]{type(ex).__name__}: {ex}[/]" ) + raise SystemExit(1) def get_provider_info_dict(provider_id: str) -> dict[str, Any]: diff --git a/docs/conf.py b/docs/conf.py index 90c9b68e6f..91b4f23996 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -72,7 +72,7 @@ elif PACKAGE_NAME.startswith("apache-airflow-providers-"): if provider_yaml["package-name"] == PACKAGE_NAME ) except StopIteration: - raise Exception(f"Could not find provider.yaml file for package: {PACKAGE_NAME}") + raise RuntimeError(f"Could not find provider.yaml file for package: {PACKAGE_NAME}") PACKAGE_DIR = pathlib.Path(CURRENT_PROVIDER["package-dir"]) PACKAGE_VERSION = CURRENT_PROVIDER["versions"][0] SYSTEM_TESTS_DIR = CURRENT_PROVIDER["system-tests-dir"] diff --git a/docs/exts/docs_build/docs_builder.py b/docs/exts/docs_build/docs_builder.py index 6bd2304832..afc796a505 100644 --- a/docs/exts/docs_build/docs_builder.py +++ b/docs/exts/docs_build/docs_builder.py @@ -33,7 +33,6 @@ from docs.exts.docs_build.code_utils import ( PROCESS_TIMEOUT, ) from docs.exts.docs_build.errors import DocBuildError, parse_sphinx_warnings -from docs.exts.docs_build.helm_chart_utils import chart_version from docs.exts.docs_build.spelling_checks import SpellingError, parse_spelling_warnings console = Console(force_terminal=True, color_system="standard", width=CONSOLE_WIDTH) @@ -88,21 +87,6 @@ class AirflowDocsBuilder: """Warnings from build job.""" return os.path.join(self._build_dir, f"warning-build-{self.package_name}.log") - @property - def _current_version(self): - if not self.is_versioned: - raise Exception("This documentation package is not versioned") - if self.package_name == "apache-airflow": - from airflow.version import version as airflow_version - - return airflow_version - if self.package_name.startswith("apache-airflow-providers-"): - provider = next(p for p in ALL_PROVIDER_YAMLS if p["package-name"] == self.package_name) - return provider["versions"][0] - if self.package_name == "helm-chart": - return chart_version() - return Exception(f"Unsupported package: {self.package_name}") - @property def _src_dir(self) -> str: return f"{DOCS_DIR}/{self.package_name}" diff --git a/docs/exts/operators_and_hooks_ref.py b/docs/exts/operators_and_hooks_ref.py index 3674b509ef..43f954ebb0 100644 --- a/docs/exts/operators_and_hooks_ref.py +++ b/docs/exts/operators_and_hooks_ref.py @@ -69,10 +69,10 @@ def _render_template(template_name, **kwargs): def _docs_path(filepath: str): if not filepath.startswith("/docs/"): - raise Exception(f"The path must starts with '/docs/'. Current value: {filepath}") + raise RuntimeError(f"The path must starts with '/docs/'. Current value: {filepath}") if not filepath.endswith(".rst"): - raise Exception(f"The path must ends with '.rst'. Current value: {filepath}") + raise RuntimeError(f"The path must ends with '.rst'. Current value: {filepath}") if filepath.startswith("/docs/apache-airflow-providers-"): _, _, provider, rest = filepath.split("/", maxsplit=3) diff --git a/docs/exts/provider_yaml_utils.py b/docs/exts/provider_yaml_utils.py index d417ba259d..1eaf93a698 100644 --- a/docs/exts/provider_yaml_utils.py +++ b/docs/exts/provider_yaml_utils.py @@ -67,8 +67,9 @@ def load_package_data(include_suspended: bool = False) -> list[dict[str, Any]]: provider = yaml.safe_load(yaml_file) try: jsonschema.validate(provider, schema=schema) - except jsonschema.ValidationError: - raise Exception(f"Unable to parse: {provider_yaml_path}.") + except jsonschema.ValidationError as ex: + msg = f"Unable to parse: {provider_yaml_path}. Original error {type(ex).__name__}: {ex}" + raise RuntimeError(msg) if provider["state"] == "suspended" and not include_suspended: continue provider_yaml_dir = os.path.dirname(provider_yaml_path) diff --git a/hatch_build.py b/hatch_build.py index 439e486ec5..0ace661347 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -592,11 +592,12 @@ for provider_spec in PRE_INSTALLED_PROVIDERS: if PROVIDER_DEPENDENCIES[provider_id]["state"] not in ["ready", "suspended", "removed"]: for dependency in PROVIDER_DEPENDENCIES[provider_id]["deps"]: if dependency.startswith("apache-airflow-providers"): - raise Exception( + msg = ( f"The provider {provider_id} is pre-installed and it has as dependency " f"to another provider {dependency}. This is not allowed. Pre-installed" f"providers should only have 'apache-airflow' and regular dependencies." ) + raise SystemExit(msg) if not dependency.startswith("apache-airflow"): PREINSTALLED_NOT_READY_DEPS.append(dependency) diff --git a/scripts/ci/pre_commit/generate_pypi_readme.py b/scripts/ci/pre_commit/generate_pypi_readme.py index 0223b0a46b..e263e798ff 100755 --- a/scripts/ci/pre_commit/generate_pypi_readme.py +++ b/scripts/ci/pre_commit/generate_pypi_readme.py @@ -54,7 +54,7 @@ def extract_section(content, section_name): if section_match: return section_match.group(1) else: - raise Exception(f"Cannot find section {section_name} in README.md") + raise RuntimeError(f"Cannot find section {section_name} in README.md") if __name__ == "__main__": diff --git a/scripts/ci/pre_commit/insert_extras.py b/scripts/ci/pre_commit/insert_extras.py index e32ad199b4..16faeaad2c 100755 --- a/scripts/ci/pre_commit/insert_extras.py +++ b/scripts/ci/pre_commit/insert_extras.py @@ -42,7 +42,7 @@ def get_header_and_footer(extra_type: str, file_format: str) -> tuple[str, str]: elif file_format == "txt": return f"# START {extra_type.upper()} HERE", f"# END {extra_type.upper()} HERE" else: - raise Exception(f"Bad format {format} passed. Only rst and txt are supported") + raise ValueError(f"Bad format {format} passed. Only rst and txt are supported") def get_wrapped_list(extras_set: list[str]) -> list[str]: @@ -81,7 +81,7 @@ def process_documentation_files() -> bool: extra_type_dict = get_extra_types_dict() for file, file_format, add_comment in FILES_TO_UPDATE: if not file.exists(): - raise Exception(f"File {file} does not exist") + raise FileNotFoundError(f"File {file} does not exist") for extra_type_description, extra_list in extra_type_dict.items(): header, footer = get_header_and_footer(extra_type_description, file_format) if insert_documentation( diff --git a/scripts/ci/pre_commit/json_schema.py b/scripts/ci/pre_commit/json_schema.py index 886ff13fe8..75b9999772 100755 --- a/scripts/ci/pre_commit/json_schema.py +++ b/scripts/ci/pre_commit/json_schema.py @@ -30,7 +30,7 @@ from jsonschema.exceptions import ValidationError from jsonschema.validators import extend, validator_for if __name__ != "__main__": - raise Exception( + raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." "To run this script, run the ./build_docs.py command" ) @@ -150,7 +150,7 @@ def _load_spec(spec_file: str | None, spec_url: str | None): if spec_url: spec_file = fetch_and_cache(url=spec_url, output_filename=re.sub(r"[^a-zA-Z0-9]", "-", spec_url)) if not spec_file: - raise Exception(f"The {spec_file} was None and {spec_url} did not lead to any file loading.") + raise ValueError(f"The {spec_file} was None and {spec_url} did not lead to any file loading.") with open(spec_file) as schema_file: schema = json.loads(schema_file.read()) return schema diff --git a/scripts/ci/pre_commit/replace_bad_characters.py b/scripts/ci/pre_commit/replace_bad_characters.py index 67377f9974..7144eb6e9a 100755 --- a/scripts/ci/pre_commit/replace_bad_characters.py +++ b/scripts/ci/pre_commit/replace_bad_characters.py @@ -25,7 +25,7 @@ from typing import NamedTuple from rich.console import Console if __name__ != "__main__": - raise Exception( + raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." f"To run this script, run the {__file__} command" ) diff --git a/scripts/ci/pre_commit/update_chart_dependencies.py b/scripts/ci/pre_commit/update_chart_dependencies.py index 939c227897..21673f503e 100755 --- a/scripts/ci/pre_commit/update_chart_dependencies.py +++ b/scripts/ci/pre_commit/update_chart_dependencies.py @@ -36,7 +36,7 @@ def get_latest_prometheus_statsd_exporter_version() -> str: for version in quay_data["tags"]: if version["name"].startswith("v"): return version["name"] - raise Exception("ERROR! No version found") + raise RuntimeError("ERROR! No version found") if __name__ == "__main__": @@ -64,7 +64,7 @@ if __name__ == "__main__": next_line = f" tag: {latest_prometheus_statsd_exporter_version}" replace = False else: - raise Exception( + raise ValueError( f"ERROR! The next line after repository: should be tag: - " f"index {index} in {VALUES_YAML_FILE}" ) diff --git a/scripts/ci/pre_commit/update_versions.py b/scripts/ci/pre_commit/update_versions.py index a2e3361eac..60947789bc 100755 --- a/scripts/ci/pre_commit/update_versions.py +++ b/scripts/ci/pre_commit/update_versions.py @@ -31,7 +31,7 @@ def update_version(pattern: re.Pattern, v: str, file_path: Path): with file_path.open("r+") as f: file_content = f.read() if not pattern.search(file_content): - raise Exception(f"Pattern {pattern!r} doesn't found in {file_path!r} file") + raise RuntimeError(f"Pattern {pattern!r} doesn't found in {file_path!r} file") new_content = pattern.sub(rf"\g<1>{v}\g<2>", file_content) if file_content == new_content: return diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index b1608c25bc..29ef9812aa 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -62,7 +62,7 @@ except ImportError: from yaml import SafeLoader # type: ignore if __name__ != "__main__": - raise Exception( + raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." ) @@ -112,8 +112,9 @@ def _load_package_data(package_paths: Iterable[str]): rel_path = pathlib.Path(provider_yaml_path).relative_to(ROOT_DIR).as_posix() try: jsonschema.validate(provider, schema=schema) - except jsonschema.ValidationError: - raise Exception(f"Unable to parse: {rel_path}.") + except jsonschema.ValidationError as ex: + msg = f"Unable to parse: {provider_yaml_path}. Original error {type(ex).__name__}: {ex}" + raise RuntimeError(msg) if not provider["state"] == "suspended": result[rel_path] = provider else: diff --git a/scripts/in_container/update_quarantined_test_status.py b/scripts/in_container/update_quarantined_test_status.py index a3f045843c..72cd04c33e 100755 --- a/scripts/in_container/update_quarantined_test_status.py +++ b/scripts/in_container/update_quarantined_test_status.py @@ -202,14 +202,14 @@ if __name__ == "__main__": print(f"Token: {token}") github_repository = os.environ.get("GITHUB_REPOSITORY") if not github_repository: - raise Exception("GitHub Repository must be defined!") + raise RuntimeError("GitHub Repository must be defined!") user, repo = github_repository.split("/") print(f"User: {user}, Repo: {repo}") issue_id = int(os.environ.get("ISSUE_ID", 0)) num_runs = int(os.environ.get("NUM_RUNS", 10)) if issue_id == 0: - raise Exception("You need to define ISSUE_ID as environment variable") + raise RuntimeError("You need to define ISSUE_ID as environment variable") gh = login(token=token) diff --git a/scripts/tools/check_if_limited_dependencies.py b/scripts/tools/check_if_limited_dependencies.py index 16f879d80e..855c1b044a 100755 --- a/scripts/tools/check_if_limited_dependencies.py +++ b/scripts/tools/check_if_limited_dependencies.py @@ -27,7 +27,7 @@ from rich.console import Console program = f"./{__file__}" if not __file__.startswith("./") else __file__ if __name__ != "__main__": - raise Exception( + raise SystemExit( "This file is intended to be used as an executable program. You cannot use it as a module." f"To execute this script, run the '{program}' command" ) diff --git a/scripts/tools/list-integrations.py b/scripts/tools/list-integrations.py index d563869c54..8054d59159 100755 --- a/scripts/tools/list-integrations.py +++ b/scripts/tools/list-integrations.py @@ -34,7 +34,7 @@ from airflow.sensors.base import BaseSensorOperator program = f"./{__file__}" if not __file__.startswith("./") else __file__ if __name__ != "__main__": - raise Exception( + raise SystemExit( "This file is intended to be used as an executable program. You cannot use it as a module." f"To execute this script, run the '{program}' command" )