This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9079093291 Consolidate importing of os.path.* (#34060)
9079093291 is described below
commit 907909329195c6655d1e2989b05609466ef50563
Author: Miroslav Šedivý <[email protected]>
AuthorDate: Thu Sep 7 01:00:10 2023 +0000
Consolidate importing of os.path.* (#34060)
Co-authored-by: Tzu-ping Chung <[email protected]>
---
airflow/providers/amazon/aws/hooks/base_aws.py | 3 +--
airflow/providers/amazon/aws/operators/glue.py | 2 +-
airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py | 4 ++--
airflow/providers/ftp/hooks/ftp.py | 2 +-
.../google/cloud/example_dags/example_cloud_sql_query.py | 9 +++------
airflow/providers/google/cloud/hooks/cloud_sql.py | 1 -
airflow/providers/google/cloud/hooks/gcs.py | 3 +--
airflow/providers/google/cloud/utils/credentials_provider.py | 2 +-
airflow/www/extensions/init_views.py | 11 ++++++-----
dev/provider_packages/prepare_provider_packages.py | 5 ++---
docs/exts/exampleinclude.py | 6 +++---
scripts/ci/pre_commit/pre_commit_check_order_setup.py | 9 ++++-----
.../pre_commit/pre_commit_check_setup_extra_packages_ref.py | 11 +++++------
scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py | 9 ++++-----
scripts/in_container/update_quarantined_test_status.py | 4 ++--
setup.py | 5 +++--
.../providers/google/cloud/operators/test_dataprep_system.py | 4 ++--
.../providers/amazon/aws/example_google_api_sheets_to_s3.py | 8 ++++----
tests/system/providers/amazon/aws/utils/__init__.py | 4 ++--
tests/utils/test_file.py | 1 -
20 files changed, 47 insertions(+), 56 deletions(-)
diff --git a/airflow/providers/amazon/aws/hooks/base_aws.py
b/airflow/providers/amazon/aws/hooks/base_aws.py
index 5e2ce926ef..02214f406d 100644
--- a/airflow/providers/amazon/aws/hooks/base_aws.py
+++ b/airflow/providers/amazon/aws/hooks/base_aws.py
@@ -32,7 +32,6 @@ import os
import warnings
from copy import deepcopy
from functools import cached_property, wraps
-from os import PathLike
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Generic, TypeVar, Union
@@ -819,7 +818,7 @@ class AwsGenericHook(BaseHook, Generic[BaseAwsConnection]):
return False, str(f"{type(e).__name__!r} error occurred while
testing connection: {e}")
@cached_property
- def waiter_path(self) -> PathLike[str] | None:
+ def waiter_path(self) -> os.PathLike[str] | None:
filename = self.client_type if self.client_type else self.resource_type
path =
Path(__file__).parents[1].joinpath(f"waiters/{filename}.json").resolve()
return path if path.exists() else None
diff --git a/airflow/providers/amazon/aws/operators/glue.py
b/airflow/providers/amazon/aws/operators/glue.py
index d47d1c25de..3f67a639c2 100644
--- a/airflow/providers/amazon/aws/operators/glue.py
+++ b/airflow/providers/amazon/aws/operators/glue.py
@@ -17,7 +17,7 @@
# under the License.
from __future__ import annotations
-import os.path
+import os
import urllib.parse
from functools import cached_property
from typing import TYPE_CHECKING, Sequence
diff --git a/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py
b/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py
index a41ccc2dae..b83ff48906 100644
--- a/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py
+++ b/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py
@@ -19,11 +19,11 @@
from __future__ import annotations
import json
+import os
from copy import copy
from datetime import datetime
from decimal import Decimal
from functools import cached_property
-from os.path import getsize
from tempfile import NamedTemporaryFile
from typing import IO, TYPE_CHECKING, Any, Callable, Sequence
from uuid import uuid4
@@ -197,7 +197,7 @@ class DynamoDBToS3Operator(AwsToAwsBaseOperator):
scan_kwargs["ExclusiveStartKey"] = last_evaluated_key
# Upload the file to S3 if reach file size limit
- if getsize(temp_file.name) >= self.file_size:
+ if os.path.getsize(temp_file.name) >= self.file_size:
_upload_file_to_s3(temp_file, self.s3_bucket_name,
self.s3_key_prefix, self.dest_aws_conn_id)
temp_file.close()
diff --git a/airflow/providers/ftp/hooks/ftp.py
b/airflow/providers/ftp/hooks/ftp.py
index 26080aaa69..ea94b32c47 100644
--- a/airflow/providers/ftp/hooks/ftp.py
+++ b/airflow/providers/ftp/hooks/ftp.py
@@ -19,7 +19,7 @@ from __future__ import annotations
import datetime
import ftplib
-import os.path
+import os
from typing import Any, Callable
from airflow.hooks.base import BaseHook
diff --git
a/airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
b/airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
index c3ec7b8d1a..b94c8a0c40 100644
--- a/airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
+++ b/airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
@@ -40,7 +40,7 @@ from __future__ import annotations
import os
import subprocess
from datetime import datetime
-from os.path import expanduser
+from pathlib import Path
from urllib.parse import quote_plus
from airflow import models
@@ -87,17 +87,14 @@ SQL = [
# [START howto_operator_cloudsql_query_connections]
-HOME_DIR = expanduser("~")
+HOME_DIR = Path.home()
def get_absolute_path(path):
"""
Returns absolute path.
"""
- if path.startswith("/"):
- return path
- else:
- return os.path.join(HOME_DIR, path)
+ return os.fspath(HOME_DIR / path)
postgres_kwargs = {
diff --git a/airflow/providers/google/cloud/hooks/cloud_sql.py
b/airflow/providers/google/cloud/hooks/cloud_sql.py
index 0b8bece36f..47738fa931 100644
--- a/airflow/providers/google/cloud/hooks/cloud_sql.py
+++ b/airflow/providers/google/cloud/hooks/cloud_sql.py
@@ -21,7 +21,6 @@ from __future__ import annotations
import errno
import json
import os
-import os.path
import platform
import random
import re
diff --git a/airflow/providers/google/cloud/hooks/gcs.py
b/airflow/providers/google/cloud/hooks/gcs.py
index 05279583da..acc4ad688a 100644
--- a/airflow/providers/google/cloud/hooks/gcs.py
+++ b/airflow/providers/google/cloud/hooks/gcs.py
@@ -28,7 +28,6 @@ import warnings
from contextlib import contextmanager
from functools import partial
from io import BytesIO
-from os import path
from tempfile import NamedTemporaryFile
from typing import IO, TYPE_CHECKING, Any, Callable, Generator, Sequence,
TypeVar, cast, overload
from urllib.parse import urlsplit
@@ -1290,7 +1289,7 @@ class GCSHook(GoogleBaseHook):
self, blob: storage.Blob, destination_object: str | None,
source_object_prefix_len: int
) -> str:
return (
- path.join(destination_object, blob.name[source_object_prefix_len:])
+ os.path.join(destination_object,
blob.name[source_object_prefix_len:])
if destination_object
else blob.name[source_object_prefix_len:]
)
diff --git a/airflow/providers/google/cloud/utils/credentials_provider.py
b/airflow/providers/google/cloud/utils/credentials_provider.py
index b6d7bcc385..18c00b0844 100644
--- a/airflow/providers/google/cloud/utils/credentials_provider.py
+++ b/airflow/providers/google/cloud/utils/credentials_provider.py
@@ -21,7 +21,7 @@ from __future__ import annotations
import json
import logging
-import os.path
+import os
import tempfile
from contextlib import ExitStack, contextmanager
from typing import Collection, Generator, Sequence
diff --git a/airflow/www/extensions/init_views.py
b/airflow/www/extensions/init_views.py
index 16813031e3..bb3c046081 100644
--- a/airflow/www/extensions/init_views.py
+++ b/airflow/www/extensions/init_views.py
@@ -17,9 +17,10 @@
from __future__ import annotations
import logging
+import os
import warnings
from functools import cached_property
-from os import path
+from pathlib import Path
from typing import TYPE_CHECKING
from connexion import FlaskApi, ProblemException, Resolver
@@ -39,7 +40,7 @@ if TYPE_CHECKING:
log = logging.getLogger(__name__)
# airflow/www/extensions/init_views.py => airflow/
-ROOT_APP_DIR = path.abspath(path.join(path.dirname(__file__), path.pardir,
path.pardir))
+ROOT_APP_DIR = Path(__file__).parents[2].resolve()
def init_flash_views(app):
@@ -253,7 +254,7 @@ def init_api_connexion(app: Flask) -> None:
else:
return views.method_not_allowed(ex)
- with open(path.join(ROOT_APP_DIR, "api_connexion", "openapi", "v1.yaml"))
as f:
+ with ROOT_APP_DIR.joinpath("api_connexion", "openapi", "v1.yaml").open()
as f:
specification = safe_load(f)
api_bp = FlaskApi(
specification=specification,
@@ -261,7 +262,7 @@ def init_api_connexion(app: Flask) -> None:
base_path=base_path,
options={
"swagger_ui": conf.getboolean("webserver", "enable_swagger_ui",
fallback=True),
- "swagger_path": path.join(ROOT_APP_DIR, "www", "static", "dist",
"swagger-ui"),
+ "swagger_path": os.fspath(ROOT_APP_DIR.joinpath("www", "static",
"dist", "swagger-ui")),
},
strict_validation=True,
validate_responses=True,
@@ -279,7 +280,7 @@ def init_api_internal(app: Flask, standalone_api: bool =
False) -> None:
if not standalone_api and not conf.getboolean("webserver",
"run_internal_api", fallback=False):
return
- with open(path.join(ROOT_APP_DIR, "api_internal", "openapi",
"internal_api_v1.yaml")) as f:
+ with ROOT_APP_DIR.joinpath("api_internal", "openapi",
"internal_api_v1.yaml").open() as f:
specification = safe_load(f)
api_bp = FlaskApi(
specification=specification,
diff --git a/dev/provider_packages/prepare_provider_packages.py
b/dev/provider_packages/prepare_provider_packages.py
index fb38e8f2dc..4dd54076eb 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -37,7 +37,6 @@ from copy import deepcopy
from datetime import datetime, timedelta
from enum import Enum
from functools import lru_cache
-from os.path import dirname, relpath
from pathlib import Path
from random import choice
from shutil import copyfile
@@ -261,7 +260,7 @@ def get_target_folder() -> str:
:return: the folder path
"""
- return os.path.abspath(os.path.join(dirname(__file__), os.pardir,
os.pardir, "provider_packages"))
+ return os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir,
os.pardir, "provider_packages"))
def get_target_providers_folder() -> str:
@@ -1154,7 +1153,7 @@ def get_provider_jinja_context(
"PIP_REQUIREMENTS_TABLE": pip_requirements_table,
"PIP_REQUIREMENTS_TABLE_RST": pip_requirements_table_rst,
"PROVIDER_INFO": provider_info,
- "CHANGELOG_RELATIVE_PATH": relpath(
+ "CHANGELOG_RELATIVE_PATH": os.path.relpath(
provider_details.source_provider_package_path,
provider_details.documentation_provider_package_path,
),
diff --git a/docs/exts/exampleinclude.py b/docs/exts/exampleinclude.py
index 7a4498d4e7..eade660515 100644
--- a/docs/exts/exampleinclude.py
+++ b/docs/exts/exampleinclude.py
@@ -20,8 +20,8 @@
from __future__ import annotations
"""Nice formatted include for examples"""
+import os
import traceback
-from os import path
from docutils import nodes
@@ -227,8 +227,8 @@ def doctree_read(app, doctree):
for objnode in doctree.traverse(ExampleHeader):
filepath = objnode.get("filename")
- relative_path = path.relpath(
- filepath, path.commonprefix([app.config.exampleinclude_sourceroot,
filepath])
+ relative_path = os.path.relpath(
+ filepath,
os.path.commonprefix([app.config.exampleinclude_sourceroot, filepath])
)
modname = relative_path.replace("/", ".")[:-3]
show_button = register_source(app, env, modname)
diff --git a/scripts/ci/pre_commit/pre_commit_check_order_setup.py
b/scripts/ci/pre_commit/pre_commit_check_order_setup.py
index 05b8136e4b..38bc2048d7 100755
--- a/scripts/ci/pre_commit/pre_commit_check_order_setup.py
+++ b/scripts/ci/pre_commit/pre_commit_check_order_setup.py
@@ -26,15 +26,14 @@ import os
import re
import sys
import textwrap
-from os.path import abspath, dirname
+from pathlib import Path
from rich import print
errors: list[str] = []
-MY_DIR_PATH = os.path.dirname(__file__)
-SOURCE_DIR_PATH = os.path.abspath(os.path.join(MY_DIR_PATH, os.pardir,
os.pardir, os.pardir))
-sys.path.insert(0, SOURCE_DIR_PATH)
+SOURCE_DIR_PATH = Path(__file__).parents[3].resolve()
+sys.path.insert(0, os.fspath(SOURCE_DIR_PATH))
class ConsoleDiff(difflib.Differ):
@@ -124,7 +123,7 @@ def check_install_and_setup_requires() -> None:
from setuptools.config import read_configuration
- path = abspath(os.path.join(dirname(__file__), os.pardir, os.pardir,
os.pardir, "setup.cfg"))
+ path = os.fspath(SOURCE_DIR_PATH / "setup.cfg")
config = read_configuration(path)
pattern_dependent_version = re.compile("[~|><=;].*")
diff --git a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
index 04b707f30a..df8e95e5e3 100755
--- a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
+++ b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
@@ -24,18 +24,18 @@ from __future__ import annotations
import os
import re
import sys
-from os.path import dirname
+from pathlib import Path
from rich import print
from rich.console import Console
from rich.table import Table
-AIRFLOW_SOURCES_DIR = os.path.join(dirname(__file__), os.pardir, os.pardir,
os.pardir)
+AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve()
SETUP_PY_FILE = "setup.py"
DOCS_FILE = os.path.join("docs", "apache-airflow", "extra-packages-ref.rst")
PY_IDENTIFIER = r"[a-zA-Z_][a-zA-Z0-9_\.]*"
-sys.path.insert(0, AIRFLOW_SOURCES_DIR)
+sys.path.insert(0, os.fspath(AIRFLOW_SOURCES_DIR))
os.environ["_SKIP_PYTHON_VERSION_CHECK"] = "true"
@@ -49,9 +49,8 @@ from setup import ( # noqa # isort:skip
def get_file_content(*path_elements: str) -> str:
- file_path = os.path.join(AIRFLOW_SOURCES_DIR, *path_elements)
- with open(file_path) as file_to_read:
- return file_to_read.read()
+ file_path = AIRFLOW_SOURCES_DIR.joinpath(*path_elements)
+ return file_path.read_text()
def get_extras_from_setup() -> set[str]:
diff --git a/scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py
b/scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py
index 31f6142b5a..ece43801e1 100755
--- a/scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py
+++ b/scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py
@@ -17,7 +17,6 @@
# under the License.
from __future__ import annotations
-from os import listdir
from pathlib import Path
AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve()
@@ -45,16 +44,16 @@ if __name__ == "__main__":
SCRIPTS_DOCKER_DIR = AIRFLOW_SOURCES_DIR / "scripts" / "docker"
for file in [DOCKERFILE_FILE, DOCKERFILE_CI_FILE]:
- for script in listdir(SCRIPTS_DOCKER_DIR):
- script_content = (SCRIPTS_DOCKER_DIR /
script).read_text().splitlines(keepends=True)
+ for script in SCRIPTS_DOCKER_DIR.iterdir():
+ script_content = script.read_text().splitlines(keepends=True)
no_comments_script_content = [
line for line in script_content if not line.startswith("#") or
line.startswith("#!")
]
- no_comments_script_content.insert(0, f'COPY <<"EOF" /{script}\n')
+ no_comments_script_content.insert(0, f'COPY <<"EOF"
/{script.name}\n')
insert_content(
file_path=file,
content=no_comments_script_content,
header="# The content below is automatically copied from
scripts/docker/",
footer="EOF",
- file_name=script,
+ file_name=script.name,
)
diff --git a/scripts/in_container/update_quarantined_test_status.py
b/scripts/in_container/update_quarantined_test_status.py
index f08dbbc81f..526eb2d961 100755
--- a/scripts/in_container/update_quarantined_test_status.py
+++ b/scripts/in_container/update_quarantined_test_status.py
@@ -21,7 +21,7 @@ import os
import re
import sys
from datetime import datetime
-from os.path import dirname, join, realpath
+from pathlib import Path
from typing import NamedTuple
from urllib.parse import urlsplit
@@ -234,7 +234,7 @@ if __name__ == "__main__":
print()
print(table)
print()
- with open(join(dirname(realpath(__file__)), "quarantine_issue_header.md"))
as f:
+ with
Path(__file__).resolve().with_name("quarantine_issue_header.md").open() as f:
header = jinja2.Template(f.read(), autoescape=True,
undefined=StrictUndefined).render(
DATE_UTC_NOW=datetime.utcnow()
)
diff --git a/setup.py b/setup.py
index cb2e115f46..3bd81e8391 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,6 @@ import subprocess
import sys
import unittest
from copy import deepcopy
-from os.path import relpath
from pathlib import Path
from textwrap import wrap
from typing import Iterable
@@ -864,7 +863,9 @@ class AirflowDistribution(Distribution):
]
provider_yaml_files =
glob.glob("airflow/providers/**/provider.yaml", recursive=True)
for provider_yaml_file in provider_yaml_files:
- provider_relative_path = relpath(provider_yaml_file,
str(AIRFLOW_SOURCES_ROOT / "airflow"))
+ provider_relative_path = os.path.relpath(
+ provider_yaml_file, str(AIRFLOW_SOURCES_ROOT / "airflow")
+ )
self.package_data["airflow"].append(provider_relative_path)
else:
self.install_requires.extend(
diff --git a/tests/providers/google/cloud/operators/test_dataprep_system.py
b/tests/providers/google/cloud/operators/test_dataprep_system.py
index f2838efb8e..fcba01fe59 100644
--- a/tests/providers/google/cloud/operators/test_dataprep_system.py
+++ b/tests/providers/google/cloud/operators/test_dataprep_system.py
@@ -18,7 +18,7 @@
from __future__ import annotations
import json
-from os import environ
+import os
import pytest
@@ -27,7 +27,7 @@ from airflow.utils.session import create_session
from tests.test_utils.db import clear_db_connections
from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER,
GoogleSystemTest
-TOKEN = environ.get("DATAPREP_TOKEN")
+TOKEN = os.environ.get("DATAPREP_TOKEN")
EXTRA = {"token": TOKEN}
diff --git
a/tests/system/providers/amazon/aws/example_google_api_sheets_to_s3.py
b/tests/system/providers/amazon/aws/example_google_api_sheets_to_s3.py
index 926ab55f8f..b8c5c5f695 100644
--- a/tests/system/providers/amazon/aws/example_google_api_sheets_to_s3.py
+++ b/tests/system/providers/amazon/aws/example_google_api_sheets_to_s3.py
@@ -20,8 +20,8 @@ You need to set all env variables to request the data.
"""
from __future__ import annotations
+import os
from datetime import datetime
-from os import getenv
from airflow import DAG
from airflow.models.baseoperator import chain
@@ -34,9 +34,9 @@ sys_test_context_task = SystemTestContextBuilder().build()
DAG_ID = "example_google_api_sheets_to_s3"
-GOOGLE_SHEET_ID = getenv("GOOGLE_SHEET_ID", "test-google-sheet-id")
-GOOGLE_SHEET_RANGE = getenv("GOOGLE_SHEET_RANGE", "test-google-sheet-range")
-S3_DESTINATION_KEY = getenv("S3_DESTINATION_KEY", "s3://test-bucket/key.json")
+GOOGLE_SHEET_ID = os.getenv("GOOGLE_SHEET_ID", "test-google-sheet-id")
+GOOGLE_SHEET_RANGE = os.getenv("GOOGLE_SHEET_RANGE", "test-google-sheet-range")
+S3_DESTINATION_KEY = os.getenv("S3_DESTINATION_KEY",
"s3://test-bucket/key.json")
with DAG(
dag_id=DAG_ID,
diff --git a/tests/system/providers/amazon/aws/utils/__init__.py
b/tests/system/providers/amazon/aws/utils/__init__.py
index fa942aae34..7c78f1bd0b 100644
--- a/tests/system/providers/amazon/aws/utils/__init__.py
+++ b/tests/system/providers/amazon/aws/utils/__init__.py
@@ -20,7 +20,7 @@ import inspect
import json
import logging
import os
-from os.path import basename, splitext
+from pathlib import Path
from time import sleep
from typing import TYPE_CHECKING
from uuid import uuid4
@@ -70,7 +70,7 @@ def _get_test_name() -> str:
test_filename: str = next(
frame.filename for frame in inspect.stack() if TEST_FILE_IDENTIFIER in
frame.filename
)
- return splitext(basename(test_filename))[0]
+ return Path(test_filename).stem
def _validate_env_id(env_id: str) -> str:
diff --git a/tests/utils/test_file.py b/tests/utils/test_file.py
index daa23fa64e..70a67d0bb5 100644
--- a/tests/utils/test_file.py
+++ b/tests/utils/test_file.py
@@ -18,7 +18,6 @@
from __future__ import annotations
import os
-import os.path
import zipfile
from pathlib import Path
from unittest import mock
