zhongjiajie commented on code in PR #53:
URL:
https://github.com/apache/dolphinscheduler-sdk-python/pull/53#discussion_r1052004625
##########
src/pydolphinscheduler/core/task.py:
##########
@@ -415,3 +420,23 @@ def environment_code(self) -> str:
if self._environment_name is None:
return None
return gateway.query_environment_info(self._environment_name)
+
+ @property
+ def local_params(self):
+ """Convert local params."""
+ local_params = copy.deepcopy(self._local_params)
+ local_params.extend(
+ ParameterHelper.convert_params(self._input_params, Direction.IN)
+ )
+ local_params.extend(
+ ParameterHelper.convert_params(self._output_params, Direction.OUT)
+ )
+ return local_params
+
+ def add_in(self, name, value=None):
+ """Add input parameters."""
Review Comment:
could we also add a docstring for both `name` and `value`?
##########
docs/source/tutorial.rst:
##########
@@ -317,3 +317,13 @@ Furthermore, this feature supports recursion all the way
down.
.. _`DolphinScheduler project page`:
https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/project.html
.. _`Python context manager`:
https://docs.python.org/3/library/stdtypes.html#context-manager-types
+
+Define Local Parameters
Review Comment:
tutorial should keep as simple as possible, could your please add those
content to https://dolphinscheduler.apache.org/python/main/concept.html page
please
##########
src/pydolphinscheduler/core/task.py:
##########
@@ -144,6 +145,8 @@ def __init__(
condition_result: Optional[Dict] = None,
resource_plugin: Optional[ResourcePlugin] = None,
is_cache: Optional[bool] = False,
+ input_params: Optional[Dict] = None,
+ output_params: Optional[Dict] = None,
Review Comment:
BTW, please add doctoring for this two new parameter
##########
src/pydolphinscheduler/examples/local_parameter_example.py:
##########
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# [start workflow_declare]
+r"""
+A tutorial example set local parameter in pydolphinscheduler.
+
+Method 1:
+ task = Shell(..., input_params={"input":"a"}, output_params={"output":
"b"})
+
+Method 2:
+ task = Shell(...)
+ task.add_in("input", "a")
+ task.add_out("output", "b")
+"""
+
+from pydolphinscheduler.core.parameter import ParameterType
+from pydolphinscheduler.core.workflow import Workflow
+from pydolphinscheduler.tasks.shell import Shell
+
+with Workflow(name="local_parameter_example", release_state="offline") as
workflow:
+
+ # Add in task __init__ func
+ task_1 = Shell(
+ name="task_1",
+ command="echo hello pydolphinscheduler",
+ input_params={
+ "value_VARCHAR": "abc",
+ "value_INTEGER": 123,
+ "value_FLOAT": 0.1,
+ "value_BOOLEAN": True,
+ },
+ output_params={
+ "value_EMPTY": None,
+ },
+ )
+
+ # Add in task instance
+ task_2 = Shell(name="task_2", command="echo hello pydolphinscheduler")
+
+ task_2.add_in("value_VARCHAR", "abc")
+ task_2.add_in("value_INTEGER", 123)
+ task_2.add_in("value_FLOAT", 0.1)
+ task_2.add_in("value_BOOLEAN", True)
+ task_2.add_out("value_EMPTY")
+
+ # Task 1 is the same as task 2
+
+ # Other parameter types corresponding to DolphinScheduler
Review Comment:
```suggestion
# Others parameter types which cannot be converted automatically, must
declare type explicitly
```
##########
tests/core/test_task.py:
##########
@@ -509,3 +510,66 @@ def test_python_resource_list(
resource_list=resources,
)
assert task.resource_list == expect
+
+
+@patch(
+ "pydolphinscheduler.core.task.Task.gen_code_and_version",
+ return_value=(123, 1),
+)
+def test_local_parameter(m_code_version):
+ """Test task local_params."""
+ base_local_params = [
+ {"prop": "base", "direct": "IN", "type": "VARCHAR", "value": "2022"},
+ ]
+
+ task = Task(name="test", task_type="task_type",
local_params=base_local_params)
Review Comment:
why should we still keep `local_params`?
##########
src/pydolphinscheduler/core/task.py:
##########
@@ -415,3 +420,23 @@ def environment_code(self) -> str:
if self._environment_name is None:
return None
return gateway.query_environment_info(self._environment_name)
+
+ @property
+ def local_params(self):
+ """Convert local params."""
+ local_params = copy.deepcopy(self._local_params)
+ local_params.extend(
+ ParameterHelper.convert_params(self._input_params, Direction.IN)
+ )
+ local_params.extend(
+ ParameterHelper.convert_params(self._output_params, Direction.OUT)
+ )
+ return local_params
+
+ def add_in(self, name, value=None):
+ """Add input parameters."""
Review Comment:
same as func `add_out`
##########
src/pydolphinscheduler/examples/local_parameter_example.py:
##########
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# [start workflow_declare]
+r"""
+A tutorial example set local parameter in pydolphinscheduler.
+
+Method 1:
+ task = Shell(..., input_params={"input":"a"}, output_params={"output":
"b"})
+
+Method 2:
+ task = Shell(...)
+ task.add_in("input", "a")
+ task.add_out("output", "b")
+"""
+
+from pydolphinscheduler.core.parameter import ParameterType
+from pydolphinscheduler.core.workflow import Workflow
+from pydolphinscheduler.tasks.shell import Shell
+
+with Workflow(name="local_parameter_example", release_state="offline") as
workflow:
+
+ # Add in task __init__ func
+ task_1 = Shell(
+ name="task_1",
+ command="echo hello pydolphinscheduler",
+ input_params={
+ "value_VARCHAR": "abc",
+ "value_INTEGER": 123,
+ "value_FLOAT": 0.1,
+ "value_BOOLEAN": True,
+ },
+ output_params={
+ "value_EMPTY": None,
+ },
+ )
+
+ # Add in task instance
Review Comment:
```suggestion
# Add parameter via task instance's method
```
##########
src/pydolphinscheduler/examples/local_parameter_example.py:
##########
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# [start workflow_declare]
+r"""
+A tutorial example set local parameter in pydolphinscheduler.
+
+Method 1:
+ task = Shell(..., input_params={"input":"a"}, output_params={"output":
"b"})
+
+Method 2:
+ task = Shell(...)
+ task.add_in("input", "a")
+ task.add_out("output", "b")
+"""
+
+from pydolphinscheduler.core.parameter import ParameterType
+from pydolphinscheduler.core.workflow import Workflow
+from pydolphinscheduler.tasks.shell import Shell
+
+with Workflow(name="local_parameter_example", release_state="offline") as
workflow:
+
+ # Add in task __init__ func
Review Comment:
```suggestion
# Add parameter via task arguments
```
##########
tests/core/test_task.py:
##########
@@ -509,3 +510,66 @@ def test_python_resource_list(
resource_list=resources,
)
assert task.resource_list == expect
+
+
+@patch(
+ "pydolphinscheduler.core.task.Task.gen_code_and_version",
+ return_value=(123, 1),
+)
+def test_local_parameter(m_code_version):
+ """Test task local_params."""
+ base_local_params = [
+ {"prop": "base", "direct": "IN", "type": "VARCHAR", "value": "2022"},
+ ]
+
+ task = Task(name="test", task_type="task_type",
local_params=base_local_params)
Review Comment:
BTW, good test cases
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
