This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch main
in repository
https://gitbox.apache.org/repos/asf/dolphinscheduler-sdk-python.git
The following commit(s) were added to refs/heads/main by this push:
new 6e5c6e9 feat: introduce ruff for linter (#125)
6e5c6e9 is described below
commit 6e5c6e9fbb0aad406309fea7d37790543540be09
Author: Jay Chung <[email protected]>
AuthorDate: Mon Dec 25 16:53:19 2023 +0800
feat: introduce ruff for linter (#125)
---
.pre-commit-config.yaml | 25 ++---------
.ruff.toml | 50 +++++++++++++++++++++
setup.cfg | 52 ++++------------------
src/pydolphinscheduler/configuration.py | 4 +-
src/pydolphinscheduler/core/mixin.py | 2 +-
src/pydolphinscheduler/core/resource_plugin.py | 2 +-
src/pydolphinscheduler/core/task.py | 6 +--
src/pydolphinscheduler/core/yaml_workflow.py | 4 +-
src/pydolphinscheduler/models/meta.py | 2 +-
.../resources_plugin/base/bucket.py | 2 +-
.../resources_plugin/base/git.py | 2 +-
src/pydolphinscheduler/resources_plugin/gitlab.py | 2 +-
src/pydolphinscheduler/resources_plugin/local.py | 6 +--
src/pydolphinscheduler/utils/versions.py | 2 +-
tests/core/test_yaml_workflow.py | 2 +-
tests/testing/file.py | 2 +-
tests/utils/test_default_config_yaml.py | 2 +-
tests/utils/test_yaml_parser.py | 2 +-
18 files changed, 82 insertions(+), 87 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c624332..501fe7d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -24,29 +24,12 @@ default_language_version:
python: python3
repos:
# Python API Hooks
- - repo: https://github.com/pycqa/isort
- rev: 5.10.1
- hooks:
- - id: isort
- name: isort (python)
- repo: https://github.com/psf/black
rev: 22.3.0
hooks:
- id: black
- - repo: https://github.com/pycqa/flake8
- rev: 4.0.1
- hooks:
- - id: flake8
- additional_dependencies: [
- 'flake8-docstrings>=1.6',
- 'flake8-black>=0.2',
- ]
- - repo: https://github.com/pycqa/autoflake
- rev: v1.4
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.1.0
hooks:
- - id: autoflake
- args: [
- --remove-all-unused-imports,
- --ignore-init-module-imports,
- --in-place
- ]
+ - id: ruff
+ args: [ --fix, --exit-non-zero-on-fix ]
diff --git a/.ruff.toml b/.ruff.toml
new file mode 100644
index 0000000..4e42e71
--- /dev/null
+++ b/.ruff.toml
@@ -0,0 +1,50 @@
+src = ["src"]
+
+# max-line-length = 110
+line-length = 110
+
+extend-select = [
+ "I", # isort
+ "D", # pydocstyle
+ "UP", # pyupgrade
+]
+
+ignore = [
+ # D107: Missing docstring in __init__
+ "D107",
+ # D105: Missing docstring in magic method
+ "D105",
+ # D418: Function/ Method decorated with @overload shouldn’t contain a
docstring
+ "D418",
+ # D400: First line should end with a period
+ "D400",
+]
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+ "__pycache__",
+ ".egg-info",
+ ".eggs",
+ ".git",
+ ".pytest_cache",
+ ".tox",
+ "build",
+ "dist",
+ "examples",
+ "venv",
+ "docs/source/conf.py",
+ "htmlcov"
+]
+
+[extend-per-file-ignores]
+"*/pydolphinscheduler/side/__init__.py" = ["F401"]
+"*/pydolphinscheduler/tasks/__init__.py" = ["F401"]
+
+[isort]
+# Mark sqlfluff, test and it's plugins as known first party
+known-first-party = [
+ "pydolphinscheduler",
+]
+
+[pydocstyle]
+convention = "google"
\ No newline at end of file
diff --git a/setup.cfg b/setup.cfg
index cca6044..4de7379 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -104,11 +104,8 @@ test =
pytest-cov>=3.0
docker>=5.0.3
style =
- flake8>=4.0
- flake8-docstrings>=1.6
- flake8-black>=0.2
- isort>=5.10
- autoflake>=1.4
+ black>=22.8
+ ruff>=0.1
doc =
sphinx>=4.3
sphinx_rtd_theme>=1.0
@@ -151,6 +148,10 @@ omit =
*/pydolphinscheduler/java_gateway.py
[coverage:report]
+# Mark no cover for typing.TYPE_CHECKING, see details
https://github.com/nedbat/coveragepy/issues/831
+exclude_lines =
+ pragma: no cover
+ if TYPE_CHECKING:
# Don’t report files that are 100% covered
skip_covered = True
show_missing = True
@@ -158,40 +159,6 @@ precision = 2
# Report will fail when coverage under 90.00%
fail_under = 90
-# ---------------------------------------------
-# Code Style Settings
-# ---------------------------------------------
-[flake8]
-max-line-length = 110
-exclude =
- .git,
- __pycache__,
- .pytest_cache,
- *.egg-info,
- docs/source/conf.py
- old,
- build,
- dist,
- htmlcov,
- .tox,
-ignore =
- # It's clear and not need to add docstring
- # D107: Don't require docstrings on __init__
- D107,
- # D105: Missing docstring in magic method
- D105,
- # Conflict to Black
- # W503: Line breaks before binary operators
- W503,
- # D400: First line should end with a period
- D400
-per-file-ignores =
- */pydolphinscheduler/side/__init__.py:F401
- */pydolphinscheduler/tasks/__init__.py:F401
-
-[isort]
-profile=black
-
# ---------------------------------------------
# TOX Settings
# ---------------------------------------------
@@ -216,17 +183,14 @@ allowlist_externals =
[testenv:auto-lint]
extras = style
commands =
- python -m isort .
python -m black .
- python -m autoflake --in-place --remove-all-unused-imports
--ignore-init-module-imports --recursive .
+ python -m ruff check --fix .
[testenv:lint]
extras = style
commands =
- python -m isort --check .
python -m black --check .
- python -m flake8
- python -m autoflake --remove-all-unused-imports
--ignore-init-module-imports --check --recursive .
+ python -m ruff check .
[testenv:code-test]
extras = test
diff --git a/src/pydolphinscheduler/configuration.py
b/src/pydolphinscheduler/configuration.py
index 91ecba9..00438db 100644
--- a/src/pydolphinscheduler/configuration.py
+++ b/src/pydolphinscheduler/configuration.py
@@ -44,7 +44,7 @@ def get_configs() -> YamlParser:
default path.
"""
path = str(config_path()) if config_path().exists() else
BUILD_IN_CONFIG_PATH
- with open(path, mode="r") as f:
+ with open(path) as f:
return YamlParser(f.read())
@@ -131,7 +131,7 @@ def token_alert(auth_token: str) -> None:
"Auth token is None, highly recommend add a token in production, "
"especially you deploy in public network."
)
- with open(BUILD_IN_CONFIG_PATH, mode="r") as f:
+ with open(BUILD_IN_CONFIG_PATH) as f:
config = YamlParser(f.read())
if config.get("java_gateway.auth_token") == auth_token:
logger.warning(
diff --git a/src/pydolphinscheduler/core/mixin.py
b/src/pydolphinscheduler/core/mixin.py
index 1cf35b3..efff7de 100644
--- a/src/pydolphinscheduler/core/mixin.py
+++ b/src/pydolphinscheduler/core/mixin.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-"""WorkerResource Mixin"""
+"""WorkerResource Mixin."""
class WorkerResourceMixin:
diff --git a/src/pydolphinscheduler/core/resource_plugin.py
b/src/pydolphinscheduler/core/resource_plugin.py
index 8b500d1..6e02370 100644
--- a/src/pydolphinscheduler/core/resource_plugin.py
+++ b/src/pydolphinscheduler/core/resource_plugin.py
@@ -23,7 +23,7 @@ from pydolphinscheduler.exceptions import PyResPluginException
# [start resource_plugin_definition]
-class ResourcePlugin(object, metaclass=ABCMeta):
+class ResourcePlugin(metaclass=ABCMeta):
"""ResourcePlugin object, declare resource plugin for task and workflow to
dolphinscheduler.
:param prefix: A string representing the prefix of ResourcePlugin.
diff --git a/src/pydolphinscheduler/core/task.py
b/src/pydolphinscheduler/core/task.py
index 5393322..5f9fccb 100644
--- a/src/pydolphinscheduler/core/task.py
+++ b/src/pydolphinscheduler/core/task.py
@@ -396,8 +396,7 @@ class Task(Base):
def _set_deps(
self, tasks: Union["Task", Sequence["Task"]], upstream: bool = True
) -> None:
- """
- Set parameter tasks dependent to current task.
+ """Set parameter tasks dependent to current task.
it is a wrapper for :func:`set_upstream` and :func:`set_downstream`.
"""
@@ -438,8 +437,7 @@ class Task(Base):
# TODO code should better generate in bulk mode when :ref: workflow run
submit or start
def gen_code_and_version(self) -> Tuple:
- """
- Generate task code and version from java gateway.
+ """Generate task code and version from java gateway.
If task name do not exists in workflow before, if will generate new
code and version id
equal to 0 by java gateway, otherwise if will return the exists code
and version.
diff --git a/src/pydolphinscheduler/core/yaml_workflow.py
b/src/pydolphinscheduler/core/yaml_workflow.py
index e994d6d..df8d65e 100644
--- a/src/pydolphinscheduler/core/yaml_workflow.py
+++ b/src/pydolphinscheduler/core/yaml_workflow.py
@@ -52,7 +52,7 @@ class ParseTool:
path = re.findall(r"\$FILE\{\"(.*?)\"\}", string_param)[0]
base_folder = kwargs.get("base_folder", ".")
path = ParseTool.get_possible_path(path, base_folder)
- with open(path, "r") as read_file:
+ with open(path) as read_file:
string_param = "".join(read_file)
return string_param
@@ -147,7 +147,7 @@ class YamlWorkflow(YamlParser):
]
def __init__(self, yaml_file: str):
- with open(yaml_file, "r") as f:
+ with open(yaml_file) as f:
content = f.read()
self._base_folder = Path(yaml_file).parent
diff --git a/src/pydolphinscheduler/models/meta.py
b/src/pydolphinscheduler/models/meta.py
index 1d7f2d8..6aea866 100644
--- a/src/pydolphinscheduler/models/meta.py
+++ b/src/pydolphinscheduler/models/meta.py
@@ -56,7 +56,7 @@ class ModelMeta(type):
for attr_name, attr_value in attrs.items():
if isinstance(attr_value, classmethod) and not
attr_name.startswith("__"):
attrs[attr_name] = mcs.j2p(attr_value, name, attrs, param)
- return super(ModelMeta, mcs).__new__(mcs, name, bases, attrs)
+ return super().__new__(mcs, name, bases, attrs)
@classmethod
def j2p(mcs, cm: classmethod, name: str, attrs: Dict, params=None):
diff --git a/src/pydolphinscheduler/resources_plugin/base/bucket.py
b/src/pydolphinscheduler/resources_plugin/base/bucket.py
index bae4366..a9d48a3 100644
--- a/src/pydolphinscheduler/resources_plugin/base/bucket.py
+++ b/src/pydolphinscheduler/resources_plugin/base/bucket.py
@@ -75,7 +75,7 @@ class S3FileInfo(BucketFileInfo):
super().__init__(bucket=bucket, file_path=file_path, *args, **kwargs)
-class Bucket(object, metaclass=ABCMeta):
+class Bucket(metaclass=ABCMeta):
"""An abstract class of online code repository based on git
implementation."""
_bucket_file_info: Optional = None
diff --git a/src/pydolphinscheduler/resources_plugin/base/git.py
b/src/pydolphinscheduler/resources_plugin/base/git.py
index 4fc2a17..53b5509 100644
--- a/src/pydolphinscheduler/resources_plugin/base/git.py
+++ b/src/pydolphinscheduler/resources_plugin/base/git.py
@@ -104,7 +104,7 @@ class GitLabFileInfo(GitFileInfo):
self.host = host
-class Git(object, metaclass=ABCMeta):
+class Git(metaclass=ABCMeta):
"""An abstract class of online code repository based on git
implementation."""
_git_file_info: Optional = None
diff --git a/src/pydolphinscheduler/resources_plugin/gitlab.py
b/src/pydolphinscheduler/resources_plugin/gitlab.py
index f035eca..fb01117 100644
--- a/src/pydolphinscheduler/resources_plugin/gitlab.py
+++ b/src/pydolphinscheduler/resources_plugin/gitlab.py
@@ -101,7 +101,7 @@ class GitLab(ResourcePlugin, Git):
self.get_git_file_info(path)
gl = self.authentication()
project = gl.projects.get(
- "%s/%s" % (self._git_file_info.user, self._git_file_info.repo_name)
+ f"{self._git_file_info.user}/{self._git_file_info.repo_name}"
)
return (
project.files.get(
diff --git a/src/pydolphinscheduler/resources_plugin/local.py
b/src/pydolphinscheduler/resources_plugin/local.py
index c1fc56d..bcdaac2 100644
--- a/src/pydolphinscheduler/resources_plugin/local.py
+++ b/src/pydolphinscheduler/resources_plugin/local.py
@@ -44,12 +44,12 @@ class Local(ResourcePlugin):
"""
path = Path(self.prefix).joinpath(suf)
if not path.exists():
- raise PyResPluginException("{} is not found".format(str(path)))
+ raise PyResPluginException(f"{str(path)} is not found")
if not os.access(str(path), os.R_OK):
raise PyResPluginException(
- "You don't have permission to access {}".format(self.prefix +
suf)
+ f"You don't have permission to access {self.prefix + suf}"
)
- with open(path, "r") as f:
+ with open(path) as f:
content = f.read()
return content
diff --git a/src/pydolphinscheduler/utils/versions.py
b/src/pydolphinscheduler/utils/versions.py
index 4c1ea6f..faaf75d 100644
--- a/src/pydolphinscheduler/utils/versions.py
+++ b/src/pydolphinscheduler/utils/versions.py
@@ -40,4 +40,4 @@ def version_match(name: str, version: str) -> bool:
return req.specifier.contains(version)
except InvalidVersion:
return False
- raise ValueError("%s is not in %s" % (name, Version.FILE_NAME))
+ raise ValueError(f"{name} is not in {Version.FILE_NAME}")
diff --git a/tests/core/test_yaml_workflow.py b/tests/core/test_yaml_workflow.py
index 80e32c1..42999d0 100644
--- a/tests/core/test_yaml_workflow.py
+++ b/tests/core/test_yaml_workflow.py
@@ -91,7 +91,7 @@ def test_parse_possible_yaml_file():
file_name = "Shell.yaml"
path = folder.joinpath(file_name)
- with open(path, "r") as f:
+ with open(path) as f:
expect = "".join(f)
string_param = '$FILE{"%s"}' % file_name
diff --git a/tests/testing/file.py b/tests/testing/file.py
index 82e0837..9816b94 100644
--- a/tests/testing/file.py
+++ b/tests/testing/file.py
@@ -23,7 +23,7 @@ from typing import Union
def get_file_content(path: Union[str, Path]) -> str:
"""Get file content in given path."""
- with open(path, mode="r") as f:
+ with open(path) as f:
return f.read()
diff --git a/tests/utils/test_default_config_yaml.py
b/tests/utils/test_default_config_yaml.py
index b4d5e07..03cfe58 100644
--- a/tests/utils/test_default_config_yaml.py
+++ b/tests/utils/test_default_config_yaml.py
@@ -34,6 +34,6 @@ def nested_key_check(comment_map: CommentedMap) -> None:
def test_key_without_dot_delimiter():
"""Test wrapper of whether default configuration file exists specific
character."""
yaml = YAML()
- with open(path_default_config_yaml, "r") as f:
+ with open(path_default_config_yaml) as f:
comment_map = yaml.load(f.read())
nested_key_check(comment_map)
diff --git a/tests/utils/test_yaml_parser.py b/tests/utils/test_yaml_parser.py
index 40becc7..6f084ba 100644
--- a/tests/utils/test_yaml_parser.py
+++ b/tests/utils/test_yaml_parser.py
@@ -80,7 +80,7 @@ name:
"""
]
-with open(path_default_config_yaml, "r") as f:
+with open(path_default_config_yaml) as f:
param.append(f.read())
