This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 54933b3 [ci][python] Add coverage check in CI (#6861)
54933b3 is described below
commit 54933b33e32424f92b2b0df1969a63c42582e078
Author: Jiajie Zhong <[email protected]>
AuthorDate: Wed Nov 17 09:46:40 2021 +0800
[ci][python] Add coverage check in CI (#6861)
* [ci] Add coverage check in CI
* Coverage add dependent
* Install pydolphinscheduler before run coverage
* Up test coverage to 87% and down threshold to 85%
* Fix code style
* Add doc about coverage
---
.github/workflows/py-ci.yml | 17 +++++
.gitignore | 9 +++
.../{requirements_dev.txt => .coveragerc} | 22 ++++--
.../pydolphinscheduler/README.md | 14 ++++
.../pydolphinscheduler/requirements_dev.txt | 2 +
.../src/pydolphinscheduler/constants.py | 1 +
.../src/pydolphinscheduler/utils/string.py | 9 ++-
.../tests/core/test_process_definition.py | 17 +++++
.../pydolphinscheduler/tests/core/test_task.py | 74 +++++++++++++++++++
.../pydolphinscheduler/tests/utils/test_date.py | 9 ++-
.../pydolphinscheduler/tests/utils/test_string.py | 86 ++++++++++++++++++++++
11 files changed, 249 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/py-ci.yml b/.github/workflows/py-ci.yml
index 5b8e42a..fb8324e 100644
--- a/.github/workflows/py-ci.yml
+++ b/.github/workflows/py-ci.yml
@@ -78,3 +78,20 @@ jobs:
- name: Run tests
run: |
pytest
+ coverage:
+ name: Tests coverage
+ needs:
+ - pytest
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up Python 3.7
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.7
+ - name: Install Development Dependences
+ run: |
+ pip install -r requirements_dev.txt
+ pip install -e .
+ - name: Run Tests && Check coverage
+ run: coverage run && coverage report
diff --git a/.gitignore b/.gitignore
index 9b44df3..0ab7a2f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,7 +49,16 @@ docker/build/apache-dolphinscheduler*
dolphinscheduler-common/sql
dolphinscheduler-common/test
+# ------------------
# pydolphinscheduler
+# ------------------
+# Cache
__pycache__/
+
+# Build
build/
*egg-info/
+
+# Test coverage
+.coverage
+htmlcov/
diff --git a/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt
b/dolphinscheduler-python/pydolphinscheduler/.coveragerc
similarity index 66%
copy from dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt
copy to dolphinscheduler-python/pydolphinscheduler/.coveragerc
index 49b4005..524cb73 100644
--- a/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt
+++ b/dolphinscheduler-python/pydolphinscheduler/.coveragerc
@@ -15,10 +15,18 @@
# specific language governing permissions and limitations
# under the License.
-# testting
-pytest~=6.2.5
-freezegun
-# code linting and formatting
-flake8
-flake8-docstrings
-flake8-black
+[run]
+command_line = -m pytest
+omit =
+ # Ignore all test cases in tests/
+ tests/*
+ # TODO. Temporary ignore java_gateway file, because we could not find good
way to test it.
+ src/pydolphinscheduler/java_gateway.py
+
+[report]
+# Don’t report files that are 100% covered
+skip_covered = True
+show_missing = True
+precision = 2
+# Report will fail when coverage under 90.00%
+fail_under = 85
diff --git a/dolphinscheduler-python/pydolphinscheduler/README.md
b/dolphinscheduler-python/pydolphinscheduler/README.md
index 0cc36d7..a0cb748 100644
--- a/dolphinscheduler-python/pydolphinscheduler/README.md
+++ b/dolphinscheduler-python/pydolphinscheduler/README.md
@@ -132,6 +132,19 @@ To test locally, you could directly run pytest after set
`PYTHONPATH`
PYTHONPATH=src/ pytest
```
+We try to keep pydolphinscheduler usable through unit test coverage. 90% test
coverage is our target, but for
+now, we require test coverage up to 85%, and each pull request leas than 85%
would fail our CI step
+`Tests coverage`. We use [coverage][coverage] to check our test coverage, and
you could check it locally by
+run command.
+
+```shell
+coverage run && coverage report
+```
+
+It would not only run unit test but also show each file coverage which cover
rate less than 100%, and `TOTAL`
+line show you total coverage of you code. If your CI failed with coverage you
could go and find some reason by
+this command output.
+
<!-- content -->
[pypi]: https://pypi.org/
[dev-setup]:
https://dolphinscheduler.apache.org/en-us/development/development-environment-setup.html
@@ -144,6 +157,7 @@ PYTHONPATH=src/ pytest
[black]: https://black.readthedocs.io/en/stable/index.html
[flake8]: https://flake8.pycqa.org/en/latest/index.html
[black-editor]:
https://black.readthedocs.io/en/stable/integrations/editors.html#pycharm-intellij-idea
+[coverage]: https://coverage.readthedocs.io/en/stable/
<!-- badge -->
[ga-py-test]:
https://github.com/apache/dolphinscheduler/actions/workflows/py-ci.yml/badge.svg?branch=dev
[ga]: https://github.com/apache/dolphinscheduler/actions
diff --git a/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt
b/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt
index 49b4005..fa40e3c 100644
--- a/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt
+++ b/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt
@@ -18,6 +18,8 @@
# testting
pytest~=6.2.5
freezegun
+# Test coverage
+coverage
# code linting and formatting
flake8
flake8-docstrings
diff --git
a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py
b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py
index d0d94c6..315a98c 100644
---
a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py
+++
b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py
@@ -94,6 +94,7 @@ class Delimiter(str):
BAR = "-"
DASH = "/"
COLON = ":"
+ UNDERSCORE = "_"
class Time(str):
diff --git
a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/string.py
b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/string.py
index 3fb6a24..e7e781c 100644
---
a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/string.py
+++
b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/string.py
@@ -17,20 +17,23 @@
"""String util function collections."""
+from pydolphinscheduler.constants import Delimiter
+
def attr2camel(attr: str, include_private=True):
"""Covert class attribute name to camel case."""
if include_private:
- attr = attr.lstrip("_")
+ attr = attr.lstrip(Delimiter.UNDERSCORE)
return snake2camel(attr)
def snake2camel(snake: str):
"""Covert snake case to camel case."""
- components = snake.split("_")
+ components = snake.split(Delimiter.UNDERSCORE)
return components[0] + "".join(x.title() for x in components[1:])
def class_name2camel(class_name: str):
"""Covert class name string to camel case."""
- return class_name[0].lower() + class_name[1:]
+ class_name = class_name.lstrip(Delimiter.UNDERSCORE)
+ return class_name[0].lower() + snake2camel(class_name[1:])
diff --git
a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py
b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py
index 0a028e8..930909a 100644
---
a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py
+++
b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py
@@ -18,6 +18,8 @@
"""Test process definition."""
from datetime import datetime
+from typing import Any
+
from pydolphinscheduler.utils.date import conv_to_schedule
import pytest
@@ -135,6 +137,21 @@ def test__parse_datetime(val, expect):
), f"Function _parse_datetime with unexpect value by {val}."
[email protected](
+ "val",
+ [
+ 20210101,
+ (2021, 1, 1),
+ {"year": "2021", "month": "1", "day": 1},
+ ],
+)
+def test__parse_datetime_not_support_type(val: Any):
+ """Test process definition function _parse_datetime not support type
error."""
+ with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd:
+ with pytest.raises(ValueError):
+ pd._parse_datetime(val)
+
+
def test_process_definition_to_dict_without_task():
"""Test process definition function to_dict without task."""
expect = {
diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py
b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py
index ef5d363..4610337 100644
--- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py
+++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py
@@ -18,8 +18,10 @@
"""Test Task class function."""
from unittest.mock import patch
+import pytest
from pydolphinscheduler.core.task import TaskParams, TaskRelation, Task
+from tests.testing.task import Task as testTask
def test_task_params_to_dict():
@@ -93,3 +95,75 @@ def test_task_to_dict():
):
task = Task(name=name, task_type=task_type,
task_params=TaskParams(raw_script))
assert task.to_dict() == expect
+
+
[email protected]("shift", ["<<", ">>"])
+def test_two_tasks_shift(shift: str):
+ """Test bit operator between tasks.
+
+ Here we test both `>>` and `<<` bit operator.
+ """
+ raw_script = "script"
+ upstream = testTask(
+ name="upstream", task_type=shift, task_params=TaskParams(raw_script)
+ )
+ downstream = testTask(
+ name="downstream", task_type=shift, task_params=TaskParams(raw_script)
+ )
+ if shift == "<<":
+ downstream << upstream
+ elif shift == ">>":
+ upstream >> downstream
+ else:
+ assert False, f"Unexpect bit operator type {shift}."
+ assert (
+ 1 == len(upstream._downstream_task_codes)
+ and downstream.code in upstream._downstream_task_codes
+ ), "Task downstream task attributes error, downstream codes size or
specific code failed."
+ assert (
+ 1 == len(downstream._upstream_task_codes)
+ and upstream.code in downstream._upstream_task_codes
+ ), "Task upstream task attributes error, upstream codes size or upstream
code failed."
+
+
[email protected](
+ "dep_expr, flag",
+ [
+ ("task << tasks", "upstream"),
+ ("tasks << task", "downstream"),
+ ("task >> tasks", "downstream"),
+ ("tasks >> task", "upstream"),
+ ],
+)
+def test_tasks_list_shift(dep_expr: str, flag: str):
+ """Test bit operator between task and sequence of tasks.
+
+ Here we test both `>>` and `<<` bit operator.
+ """
+ reverse_dict = {
+ "upstream": "downstream",
+ "downstream": "upstream",
+ }
+ task_type = "dep_task_and_tasks"
+ raw_script = "script"
+ task = testTask(
+ name="upstream", task_type=task_type,
task_params=TaskParams(raw_script)
+ )
+ tasks = [
+ testTask(
+ name="downstream1", task_type=task_type,
task_params=TaskParams(raw_script)
+ ),
+ testTask(
+ name="downstream2", task_type=task_type,
task_params=TaskParams(raw_script)
+ ),
+ ]
+
+ # Use build-in function eval to simply test case and reduce duplicate code
+ eval(dep_expr)
+ direction_attr = f"_{flag}_task_codes"
+ reverse_direction_attr = f"_{reverse_dict[flag]}_task_codes"
+ assert 2 == len(getattr(task, direction_attr))
+ assert [t.code in getattr(task, direction_attr) for t in tasks]
+
+ assert all([1 == len(getattr(t, reverse_direction_attr)) for t in tasks])
+ assert all([task.code in getattr(t, reverse_direction_attr) for t in
tasks])
diff --git
a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py
b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py
index 53ba478..648f2c4 100644
--- a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py
+++ b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py
@@ -63,7 +63,14 @@ def test_conv_from_str_success(src: str, expect: datetime)
-> None:
@pytest.mark.parametrize(
- "src", ["2021-01-01 010101", "2021:01:01", "202111", "20210101010101"]
+ "src",
+ [
+ "2021-01-01 010101",
+ "2021:01:01",
+ "202111",
+ "20210101010101",
+ "2021:01:01 01:01:01",
+ ],
)
def test_conv_from_str_not_impl(src: str) -> None:
"""Test function conv_from_str fail case."""
diff --git
a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py
b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py
new file mode 100644
index 0000000..6942d7e
--- /dev/null
+++ b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Test utils.string module."""
+
+from pydolphinscheduler.utils.string import attr2camel, snake2camel,
class_name2camel
+import pytest
+
+
[email protected](
+ "snake, expect",
+ [
+ ("snake_case", "snakeCase"),
+ ("snake_123case", "snake123Case"),
+ ("snake_c_a_s_e", "snakeCASE"),
+ ("snake__case", "snakeCase"),
+ ("snake_case_case", "snakeCaseCase"),
+ ("_snake_case", "SnakeCase"),
+ ("__snake_case", "SnakeCase"),
+ ("Snake_case", "SnakeCase"),
+ ],
+)
+def test_snake2camel(snake: str, expect: str):
+ """Test function snake2camel, this is a base function for utils.string."""
+ assert expect == snake2camel(
+ snake
+ ), f"Test case {snake} do no return expect result {expect}."
+
+
[email protected](
+ "attr, expects",
+ [
+ # source attribute, (true expect, false expect),
+ ("snake_case", ("snakeCase", "snakeCase")),
+ ("snake_123case", ("snake123Case", "snake123Case")),
+ ("snake_c_a_s_e", ("snakeCASE", "snakeCASE")),
+ ("snake__case", ("snakeCase", "snakeCase")),
+ ("snake_case_case", ("snakeCaseCase", "snakeCaseCase")),
+ ("_snake_case", ("snakeCase", "SnakeCase")),
+ ("__snake_case", ("snakeCase", "SnakeCase")),
+ ("Snake_case", ("SnakeCase", "SnakeCase")),
+ ],
+)
+def test_attr2camel(attr: str, expects: tuple):
+ """Test function attr2camel."""
+ for idx, expect in enumerate(expects):
+ include_private = idx % 2 == 0
+ assert expect == attr2camel(
+ attr, include_private
+ ), f"Test case {attr} do no return expect result {expect} when
include_private is {include_private}."
+
+
[email protected](
+ "class_name, expect",
+ [
+ ("snake_case", "snakeCase"),
+ ("snake_123case", "snake123Case"),
+ ("snake_c_a_s_e", "snakeCASE"),
+ ("snake__case", "snakeCase"),
+ ("snake_case_case", "snakeCaseCase"),
+ ("_snake_case", "snakeCase"),
+ ("_Snake_case", "snakeCase"),
+ ("__snake_case", "snakeCase"),
+ ("__Snake_case", "snakeCase"),
+ ("Snake_case", "snakeCase"),
+ ],
+)
+def test_class_name2camel(class_name: str, expect: str):
+ """Test function class_name2camel."""
+ assert expect == class_name2camel(
+ class_name
+ ), f"Test case {class_name} do no return expect result {expect}."
