This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler-website.git
The following commit(s) were added to refs/heads/asf-site by this push:
new af98a4b36e Automated deployment:
4ffeacc0f8a624ed5950928a45559d83eeb0d57c
af98a4b36e is described below
commit af98a4b36e14cf787e0e3e7cae99d3087869cf97
Author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Dec 23 05:07:24 2022 +0000
Automated deployment: 4ffeacc0f8a624ed5950928a45559d83eeb0d57c
---
python/2.0.5/.buildinfo | 2 +-
python/2.0.6/.buildinfo | 2 +-
python/2.0.7/.buildinfo | 2 +-
python/3.0.0/.buildinfo | 2 +-
python/3.0.1/.buildinfo | 2 +-
python/3.1.0/.buildinfo | 2 +-
python/4.0.0/.buildinfo | 2 +-
python/main/.buildinfo | 2 +-
.../pydolphinscheduler/tasks/func_wrap.html | 36 +++--
.../_modules/pydolphinscheduler/tasks/python.html | 4 -
.../_modules/pydolphinscheduler/tasks/sql.html | 35 +++-
python/main/_sources/tasks/func_wrap.rst.txt | 55 ++++++-
python/main/_sources/tasks/sql.rst.txt | 52 ++++++
python/main/api.html | 34 +++-
python/main/genindex.html | 10 +-
python/main/index.html | 2 +-
python/main/objects.inv | Bin 11399 -> 11438 bytes
python/main/searchindex.js | 2 +-
python/main/tasks/condition.html | 2 +-
python/main/tasks/datax.html | 2 +-
python/main/tasks/dependent.html | 2 +-
python/main/tasks/dvc.html | 2 +-
python/main/tasks/flink.html | 2 +-
python/main/tasks/func_wrap.html | 111 ++++++++++---
python/main/tasks/http.html | 2 +-
python/main/tasks/index.html | 8 +-
python/main/tasks/kubernetes.html | 2 +-
python/main/tasks/map_reduce.html | 2 +-
python/main/tasks/mlflow.html | 2 +-
python/main/tasks/openmldb.html | 2 +-
python/main/tasks/procedure.html | 2 +-
python/main/tasks/python.html | 2 +-
python/main/tasks/pytorch.html | 2 +-
python/main/tasks/sagemaker.html | 2 +-
python/main/tasks/shell.html | 6 +-
python/main/tasks/spark.html | 2 +-
python/main/tasks/sql.html | 176 ++++++++++++++++++---
python/main/tasks/sub_workflow.html | 2 +-
python/main/tasks/switch.html | 2 +-
python/main/tutorial.html | 86 ++++++----
40 files changed, 527 insertions(+), 140 deletions(-)
diff --git a/python/2.0.5/.buildinfo b/python/2.0.5/.buildinfo
index 442e3cea19..bd4c6b3cb5 100644
--- a/python/2.0.5/.buildinfo
+++ b/python/2.0.5/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 32ec24eff90b01d9338e59b514da8e00
+config: fdf10a249d083e29da518d728734de0a
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/2.0.6/.buildinfo b/python/2.0.6/.buildinfo
index 175327eb60..7d65462ecf 100644
--- a/python/2.0.6/.buildinfo
+++ b/python/2.0.6/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 1a18944ab0c73bf591d336ad9635d833
+config: b76d04189be1d1a6ec5192ee0d6b8843
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/2.0.7/.buildinfo b/python/2.0.7/.buildinfo
index 1fb5c2fda4..06a50b0304 100644
--- a/python/2.0.7/.buildinfo
+++ b/python/2.0.7/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 829be1977f5983ee1bef856d3d2eb429
+config: e537ec41dff37ac14bc1bf0826704e31
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/3.0.0/.buildinfo b/python/3.0.0/.buildinfo
index af843f714d..a99ece3425 100644
--- a/python/3.0.0/.buildinfo
+++ b/python/3.0.0/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: d3f36467bc0b45c7178e5907e583821c
+config: 9c1d0737355a9ae4ec599193b2ed28b5
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/3.0.1/.buildinfo b/python/3.0.1/.buildinfo
index 1aac863638..fe858c74f0 100644
--- a/python/3.0.1/.buildinfo
+++ b/python/3.0.1/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 5392b0a5e236c64ede50d81648a1379e
+config: 4a92bc7412dcbd246443c1eacf05e1fa
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/3.1.0/.buildinfo b/python/3.1.0/.buildinfo
index 864cdb74d5..3f6684605a 100644
--- a/python/3.1.0/.buildinfo
+++ b/python/3.1.0/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: abe9fcc1e94f42f16c0c35427ac97651
+config: 0fd8a0cc1efbb7a75574fe118d8ac945
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/4.0.0/.buildinfo b/python/4.0.0/.buildinfo
index 6e5e888a82..d8f34e6975 100644
--- a/python/4.0.0/.buildinfo
+++ b/python/4.0.0/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 037822c4ba01c11a170712b5615503ae
+config: 70e67b82c2f8af455e8388498fec3c77
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/main/.buildinfo b/python/main/.buildinfo
index 4351e3878a..fb6cb40a49 100644
--- a/python/main/.buildinfo
+++ b/python/main/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: b29cfacf0fb3ae82ee8f90bd1dc74877
+config: 916430044b130d399c13c052c4e68e14
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/main/_modules/pydolphinscheduler/tasks/func_wrap.html
b/python/main/_modules/pydolphinscheduler/tasks/func_wrap.html
index 6a2a27c33f..146cd008e0 100644
--- a/python/main/_modules/pydolphinscheduler/tasks/func_wrap.html
+++ b/python/main/_modules/pydolphinscheduler/tasks/func_wrap.html
@@ -100,43 +100,47 @@
<span class="kn">import</span> <span class="nn">functools</span>
<span class="kn">import</span> <span class="nn">inspect</span>
-<span class="kn">import</span> <span class="nn">itertools</span>
<span class="kn">import</span> <span class="nn">types</span>
+<span class="kn">from</span> <span class="nn">pathlib</span> <span
class="kn">import</span> <span class="n">Path</span>
+
+<span class="kn">from</span> <span class="nn">stmdency.extractor</span> <span
class="kn">import</span> <span class="n">Extractor</span>
<span class="kn">from</span> <span
class="nn">pydolphinscheduler.exceptions</span> <span class="kn">import</span>
<span class="n">PyDSParamException</span>
<span class="kn">from</span> <span
class="nn">pydolphinscheduler.tasks.python</span> <span
class="kn">import</span> <span class="n">Python</span>
-<div class="viewcode-block" id="_get_func_str"><a class="viewcode-back"
href="../../../tasks/func_wrap.html#pydolphinscheduler.tasks.func_wrap._get_func_str">[docs]</a><span
class="k">def</span> <span class="nf">_get_func_str</span><span
class="p">(</span><span class="n">func</span><span class="p">:</span> <span
class="n">types</span><span class="o">.</span><span
class="n">FunctionType</span><span class="p">)</span> <span
class="o">-></span> <span class="nb">str</span><span class="p"> [...]
- <span class="sd">"""Get Python function string without
indent from decorator.</span>
+<div class="viewcode-block" id="_exists_other_decorator"><a
class="viewcode-back"
href="../../../tasks/func_wrap.html#pydolphinscheduler.tasks.func_wrap._exists_other_decorator">[docs]</a><span
class="k">def</span> <span class="nf">_exists_other_decorator</span><span
class="p">(</span><span class="n">func</span><span class="p">:</span> <span
class="n">types</span><span class="o">.</span><span
class="n">FunctionType</span><span class="p">)</span> <span
class="o">-></span> <span class=" [...]
+ <span class="sd">"""Check if the function has other
decorators except @task.</span>
<span class="sd"> :param func: The function which wraps by decorator
``@task``.</span>
<span class="sd"> """</span>
<span class="n">lines</span> <span class="o">=</span> <span
class="n">inspect</span><span class="o">.</span><span
class="n">getsourcelines</span><span class="p">(</span><span
class="n">func</span><span class="p">)[</span><span class="mi">0</span><span
class="p">]</span>
- <span class="n">src_strip</span> <span class="o">=</span> <span
class="s2">""</span>
- <span class="n">lead_space_num</span> <span class="o">=</span> <span
class="kc">None</span>
<span class="k">for</span> <span class="n">line</span> <span
class="ow">in</span> <span class="n">lines</span><span class="p">:</span>
- <span class="k">if</span> <span class="n">lead_space_num</span> <span
class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
- <span class="n">lead_space_num</span> <span class="o">=</span>
<span class="nb">sum</span><span class="p">(</span><span class="mi">1</span>
<span class="k">for</span> <span class="n">_</span> <span class="ow">in</span>
<span class="n">itertools</span><span class="o">.</span><span
class="n">takewhile</span><span class="p">(</span><span
class="nb">str</span><span class="o">.</span><span
class="n">isspace</span><span class="p">,</span> <span
class="n">line</span><span class="p"> [...]
- <span class="k">if</span> <span class="n">line</span><span
class="o">.</span><span class="n">strip</span><span class="p">()</span> <span
class="o">==</span> <span class="s2">"@task"</span><span
class="p">:</span>
- <span class="k">continue</span>
- <span class="k">elif</span> <span class="n">line</span><span
class="o">.</span><span class="n">strip</span><span class="p">()</span><span
class="o">.</span><span class="n">startswith</span><span
class="p">(</span><span class="s2">"@"</span><span class="p">):</span>
+ <span class="n">strip_line</span> <span class="o">=</span> <span
class="n">line</span><span class="o">.</span><span class="n">strip</span><span
class="p">()</span>
+ <span class="k">if</span> <span class="n">strip_line</span><span
class="o">.</span><span class="n">startswith</span><span
class="p">(</span><span class="s2">"@"</span><span class="p">)</span>
<span class="ow">and</span> <span class="ow">not</span> <span
class="n">strip_line</span> <span class="o">==</span> <span
class="s2">"@task"</span><span class="p">:</span>
<span class="k">raise</span> <span
class="n">PyDSParamException</span><span class="p">(</span>
<span class="s2">"Do no support other decorators for
function ``task`` decorator."</span>
- <span class="p">)</span>
- <span class="n">src_strip</span> <span class="o">+=</span> <span
class="n">line</span><span class="p">[</span><span
class="n">lead_space_num</span><span class="p">:]</span>
- <span class="k">return</span> <span class="n">src_strip</span></div>
+ <span class="p">)</span></div>
<div class="viewcode-block" id="task"><a class="viewcode-back"
href="../../../tasks/func_wrap.html#pydolphinscheduler.tasks.func_wrap.task">[docs]</a><span
class="k">def</span> <span class="nf">task</span><span class="p">(</span><span
class="n">func</span><span class="p">:</span> <span class="n">types</span><span
class="o">.</span><span class="n">FunctionType</span><span class="p">):</span>
- <span class="sd">"""Decorate which covert Python function
into pydolphinscheduler task."""</span>
+ <span class="sd">"""Decorate which covert Python functions
into pydolphinscheduler task.</span>
+
+<span class="sd"> :param func: The function which wraps by decorator
``@task``.</span>
+<span class="sd"> """</span>
<span class="nd">@functools</span><span class="o">.</span><span
class="n">wraps</span><span class="p">(</span><span class="n">func</span><span
class="p">)</span>
<span class="k">def</span> <span class="nf">wrapper</span><span
class="p">(</span><span class="o">*</span><span class="n">args</span><span
class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span
class="p">):</span>
- <span class="n">func_str</span> <span class="o">=</span> <span
class="n">_get_func_str</span><span class="p">(</span><span
class="n">func</span><span class="p">)</span>
+ <span class="n">_exists_other_decorator</span><span
class="p">(</span><span class="n">func</span><span class="p">)</span>
+ <span class="n">loc</span> <span class="o">=</span> <span
class="n">func</span><span class="o">.</span><span
class="vm">__code__</span><span class="o">.</span><span
class="n">co_filename</span>
+ <span class="n">extractor</span> <span class="o">=</span> <span
class="n">Extractor</span><span class="p">(</span><span
class="n">Path</span><span class="p">(</span><span class="n">loc</span><span
class="p">)</span><span class="o">.</span><span class="n">open</span><span
class="p">(</span><span class="s2">"r"</span><span
class="p">)</span><span class="o">.</span><span class="n">read</span><span
class="p">())</span>
+ <span class="n">stm</span> <span class="o">=</span> <span
class="n">extractor</span><span class="o">.</span><span
class="n">get_code</span><span class="p">(</span><span
class="n">func</span><span class="o">.</span><span
class="vm">__name__</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Python</span><span
class="p">(</span>
- <span class="n">name</span><span class="o">=</span><span
class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span
class="p">(</span><span class="s2">"name"</span><span
class="p">,</span> <span class="n">func</span><span class="o">.</span><span
class="vm">__name__</span><span class="p">),</span> <span
class="n">definition</span><span class="o">=</span><span
class="n">func_str</span><span class="p">,</span> <span class="o">*</span><span
class="n" [...]
+ <span class="n">name</span><span class="o">=</span><span
class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span
class="p">(</span><span class="s2">"name"</span><span
class="p">,</span> <span class="n">func</span><span class="o">.</span><span
class="vm">__name__</span><span class="p">),</span>
+ <span class="n">definition</span><span class="o">=</span><span
class="sa">f</span><span class="s2">"</span><span class="si">{</span><span
class="n">stm</span><span class="si">}{</span><span class="n">func</span><span
class="o">.</span><span class="vm">__name__</span><span
class="si">}</span><span class="s2">()"</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span
class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span
class="p">,</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">wrapper</span></div>
diff --git a/python/main/_modules/pydolphinscheduler/tasks/python.html
b/python/main/_modules/pydolphinscheduler/tasks/python.html
index 71a6a142dc..9195f4908b 100644
--- a/python/main/_modules/pydolphinscheduler/tasks/python.html
+++ b/python/main/_modules/pydolphinscheduler/tasks/python.html
@@ -161,10 +161,6 @@
<span class="n">pattern</span> <span class="o">=</span> <span
class="n">re</span><span class="o">.</span><span class="n">compile</span><span
class="p">(</span><span class="s2">"^def (</span><span
class="se">\\</span><span class="s2">w+)</span><span class="se">\\</span><span
class="s2">("</span><span class="p">)</span>
<span class="n">find</span> <span class="o">=</span> <span
class="n">pattern</span><span class="o">.</span><span
class="n">findall</span><span class="p">(</span><span
class="n">definition</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span
class="n">find</span><span class="p">:</span>
- <span class="n">log</span><span class="o">.</span><span
class="n">warning</span><span class="p">(</span>
- <span class="s2">"Python definition is simple script
instead of function, with value </span><span class="si">%s</span><span
class="s2">"</span><span class="p">,</span>
- <span class="n">definition</span><span class="p">,</span>
- <span class="p">)</span>
<span class="k">return</span> <span class="n">definition</span>
<span class="c1"># Keep function str and function callable always
have one blank line</span>
<span class="n">func_str</span> <span class="o">=</span> <span
class="p">(</span>
diff --git a/python/main/_modules/pydolphinscheduler/tasks/sql.html
b/python/main/_modules/pydolphinscheduler/tasks/sql.html
index a2e6297741..3572cdc5e8 100644
--- a/python/main/_modules/pydolphinscheduler/tasks/sql.html
+++ b/python/main/_modules/pydolphinscheduler/tasks/sql.html
@@ -100,7 +100,7 @@
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">re</span>
-<span class="kn">from</span> <span class="nn">typing</span> <span
class="kn">import</span> <span class="n">Dict</span><span class="p">,</span>
<span class="n">Optional</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span
class="kn">import</span> <span class="n">Dict</span><span class="p">,</span>
<span class="n">List</span><span class="p">,</span> <span
class="n">Optional</span><span class="p">,</span> <span
class="n">Sequence</span><span class="p">,</span> <span class="n">Union</span>
<span class="kn">from</span> <span
class="nn">pydolphinscheduler.constants</span> <span class="kn">import</span>
<span class="n">TaskType</span>
<span class="kn">from</span> <span
class="nn">pydolphinscheduler.core.database</span> <span
class="kn">import</span> <span class="n">Database</span>
@@ -130,6 +130,18 @@
<span class="sd"> - SQLServer</span>
<span class="sd"> You provider datasource_name contain connection
information, it decisions which</span>
<span class="sd"> database type and database instance would run this
sql.</span>
+
+<span class="sd"> :param name: SQL task name</span>
+<span class="sd"> :param datasource_name: datasource name in
dolphinscheduler, the name must exists and must be ``online``</span>
+<span class="sd"> datasource instead of ``test``.</span>
+<span class="sd"> :param sql: SQL statement, the sql script you want to
run. Support resource plugin in this parameter.</span>
+<span class="sd"> :param sql_type: SQL type, whether sql statement is
select query or not. If not provided, it will be auto</span>
+<span class="sd"> detected according to sql statement using
:func:`pydolphinscheduler.tasks.sql.Sql.sql_type`, and you</span>
+<span class="sd"> can also set it manually. by ``SqlType.SELECT`` for
query statement or ``SqlType.NOT_SELECT`` for not</span>
+<span class="sd"> query statement.</span>
+<span class="sd"> :param pre_statements: SQL statements to be executed
before the main SQL statement.</span>
+<span class="sd"> :param post_statements: SQL statements to be executed
after the main SQL statement.</span>
+<span class="sd"> :param display_rows: The number of record rows number to
be displayed in the SQL task log, default is 10.</span>
<span class="sd"> """</span>
<span class="n">_task_custom_attr</span> <span class="o">=</span> <span
class="p">{</span>
@@ -149,8 +161,8 @@
<span class="n">datasource_name</span><span class="p">:</span> <span
class="nb">str</span><span class="p">,</span>
<span class="n">sql</span><span class="p">:</span> <span
class="nb">str</span><span class="p">,</span>
<span class="n">sql_type</span><span class="p">:</span> <span
class="n">Optional</span><span class="p">[</span><span
class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span
class="kc">None</span><span class="p">,</span>
- <span class="n">pre_statements</span><span class="p">:</span> <span
class="n">Optional</span><span class="p">[</span><span
class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span
class="kc">None</span><span class="p">,</span>
- <span class="n">post_statements</span><span class="p">:</span> <span
class="n">Optional</span><span class="p">[</span><span
class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span
class="kc">None</span><span class="p">,</span>
+ <span class="n">pre_statements</span><span class="p">:</span> <span
class="n">Optional</span><span class="p">[</span><span
class="n">Union</span><span class="p">[</span><span class="nb">str</span><span
class="p">,</span> <span class="n">Sequence</span><span class="p">[</span><span
class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span
class="kc">None</span><span class="p">,</span>
+ <span class="n">post_statements</span><span class="p">:</span> <span
class="n">Optional</span><span class="p">[</span><span
class="n">Union</span><span class="p">[</span><span class="nb">str</span><span
class="p">,</span> <span class="n">Sequence</span><span class="p">[</span><span
class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span
class="kc">None</span><span class="p">,</span>
<span class="n">display_rows</span><span class="p">:</span> <span
class="n">Optional</span><span class="p">[</span><span
class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span
class="mi">10</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span
class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span>
@@ -159,10 +171,23 @@
<span class="nb">super</span><span class="p">()</span><span
class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span
class="n">name</span><span class="p">,</span> <span
class="n">TaskType</span><span class="o">.</span><span
class="n">SQL</span><span class="p">,</span> <span class="o">*</span><span
class="n">args</span><span class="p">,</span> <span class="o">**</span><span
class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span
class="n">param_sql_type</span> <span class="o">=</span> <span
class="n">sql_type</span>
<span class="bp">self</span><span class="o">.</span><span
class="n">datasource_name</span> <span class="o">=</span> <span
class="n">datasource_name</span>
- <span class="bp">self</span><span class="o">.</span><span
class="n">pre_statements</span> <span class="o">=</span> <span
class="n">pre_statements</span> <span class="ow">or</span> <span
class="p">[]</span>
- <span class="bp">self</span><span class="o">.</span><span
class="n">post_statements</span> <span class="o">=</span> <span
class="n">post_statements</span> <span class="ow">or</span> <span
class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span
class="n">pre_statements</span> <span class="o">=</span> <span
class="bp">self</span><span class="o">.</span><span
class="n">get_stm_list</span><span class="p">(</span><span
class="n">pre_statements</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span
class="n">post_statements</span> <span class="o">=</span> <span
class="bp">self</span><span class="o">.</span><span
class="n">get_stm_list</span><span class="p">(</span><span
class="n">post_statements</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span
class="n">display_rows</span> <span class="o">=</span> <span
class="n">display_rows</span>
+<div class="viewcode-block" id="Sql.get_stm_list"><a class="viewcode-back"
href="../../../tasks/sql.html#pydolphinscheduler.tasks.Sql.get_stm_list">[docs]</a>
<span class="nd">@staticmethod</span>
+ <span class="k">def</span> <span class="nf">get_stm_list</span><span
class="p">(</span><span class="n">stm</span><span class="p">:</span> <span
class="n">Union</span><span class="p">[</span><span class="nb">str</span><span
class="p">,</span> <span class="n">Sequence</span><span class="p">[</span><span
class="nb">str</span><span class="p">]])</span> <span class="o">-></span>
<span class="n">List</span><span class="p">[</span><span
class="nb">str</span><span class="p">]:</span>
+ <span class="sd">"""Convert statement to str of
list.</span>
+
+<span class="sd"> :param stm: statements string</span>
+<span class="sd"> :return: statements list</span>
+<span class="sd"> """</span>
+ <span class="k">if</span> <span class="ow">not</span> <span
class="n">stm</span><span class="p">:</span>
+ <span class="k">return</span> <span class="p">[]</span>
+ <span class="k">elif</span> <span class="nb">isinstance</span><span
class="p">(</span><span class="n">stm</span><span class="p">,</span> <span
class="nb">str</span><span class="p">):</span>
+ <span class="k">return</span> <span class="p">[</span><span
class="n">stm</span><span class="p">]</span>
+ <span class="k">return</span> <span class="nb">list</span><span
class="p">(</span><span class="n">stm</span><span class="p">)</span></div>
+
<span class="nd">@property</span>
<span class="k">def</span> <span class="nf">sql_type</span><span
class="p">(</span><span class="bp">self</span><span class="p">)</span> <span
class="o">-></span> <span class="nb">str</span><span class="p">:</span>
<span class="sd">"""Judgement sql type, it will return
the SQL type for type `SELECT` or `NOT_SELECT`.</span>
diff --git a/python/main/_sources/tasks/func_wrap.rst.txt
b/python/main/_sources/tasks/func_wrap.rst.txt
index a4a2972933..eaa74b97e8 100644
--- a/python/main/_sources/tasks/func_wrap.rst.txt
+++ b/python/main/_sources/tasks/func_wrap.rst.txt
@@ -15,10 +15,59 @@
specific language governing permissions and limitations
under the License.
-Python Function Wrapper
-=======================
+Python Function Decorate
+========================
-A decorator covert Python function into pydolphinscheduler's task.
+A decorator covert Python function into pydolphinscheduler's task. Python
function decorator use decorate
+:code:`@task` from :code:`from pydolphinscheduler.tasks.func_wrap import task`
to convert Python function into
+a single Python task of dolphinscheduler.
+
+Because we have to covert the whole Python definition into multiple Python
task in dolphinscheduler, and all of
+the seperated Python task will be executed in the different Python process, so
we need to separate not only the
+python function code, but also the all variables and the imported modules
related to decorated function.
+
+For example, we decorated function ``depend_import`` in definition
+
+.. code-block:: python
+
+ import time
+
+ @task
+ def depend_import():
+ time.sleep(2)
+
+and we can see functon ``depend_import`` depend on other modules, it use
:code:`time.sleep(2)` from module :code:`time`
+to sleep 2 seconds. So when we want to separate this function into
dolphinscheduler task, need to include the imported
+:code:`time` module.
+
+which means we not only post code
+
+.. code-block:: python
+
+ def depend_import():
+ time.sleep(2)
+
+ depend_import()
+
+to dolphinscheduler Python task, we post the dependencies of this function as
well, so you will see this in
+dolphinscheduler Python task to make it work. And if you use the global
variables or other function in the
+decorated function, it will also including them as well.
+
+.. code-block:: python
+
+ import time
+
+ def depend_import():
+ time.sleep(2)
+
+ depend_import()
+
+
+.. note::
+
+ We use third party library `stmdency <https://pypi.org/project/stmdency>`_
to get the dependencies statement
+ of current function, so if you find some unexpected behavior you can report
bug to `apache-dolphinscheduler`
+ or `stmdency`.
Example
-------
diff --git a/python/main/_sources/tasks/sql.rst.txt
b/python/main/_sources/tasks/sql.rst.txt
index 52df042b74..7110046e00 100644
--- a/python/main/_sources/tasks/sql.rst.txt
+++ b/python/main/_sources/tasks/sql.rst.txt
@@ -18,6 +18,58 @@
SQL
===
+An example for SQL task, including how to use it and the detail of it
parameters.
+
+This task type can execute multiple type of database SQL, which includes
+
+- MySQL
+- PostgreSQL
+- Oracle
+- SQL Server
+- DB2
+- Hive
+- Presto
+- Trino
+- ClickHouse
+
+Example
+-------
+
+.. literalinclude::
../../../src/pydolphinscheduler/examples/task_sql_example.py
+ :start-after: [start workflow_declare]
+ :end-before: [end workflow_declare]
+
+You can see that SQL task support three ways to declare SQL, which are
+
+- Bare SQL: Put bare SQL statement in the ``sql`` parameter, such as ``select
* from table1``.
+
+ .. literalinclude::
../../../src/pydolphinscheduler/examples/task_sql_example.py
+ :start-after: [start bare_sql_desc]
+ :end-before: [end bare_sql_desc]
+
+- SQL Files: .
+
+ .. literalinclude::
../../../src/pydolphinscheduler/examples/task_sql_example.py
+ :start-after: [start sql_file_desc]
+ :end-before: [end sql_file_desc]
+
+If you want to do some preparation before executing SQL, or do some clean up
after executing SQL, you can use
+``pre_statements`` and ``post_statements`` parameters to do that. Both
``pre_statements`` and ``post_statements``
+support one or multiple statements, you can assign type sequence of SQL
statements to them if you want to execute
+multiple statements. But if you only need to execute one statement, you can
assign a string to them.
+
+ .. literalinclude::
../../../src/pydolphinscheduler/examples/task_sql_example.py
+ :start-after: [start sql_with_pre_post_desc]
+ :end-before: [end sql_with_pre_post_desc]
+
+.. note::
+
+ Parameter ``pre_statements`` and ``post_statements`` only support not query
statements, such as ``create table``,
+ ``drop table``, ``update table`` currently. And also it only support bare
SQL instead of SQL files now.
+
+Dive Into
+---------
+
.. automodule:: pydolphinscheduler.tasks.sql
diff --git a/python/main/api.html b/python/main/api.html
index 1cad1b7da5..dcf3276a95 100644
--- a/python/main/api.html
+++ b/python/main/api.html
@@ -1107,6 +1107,7 @@
<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.Sql.get_define"><code class="docutils literal
notranslate"><span class="pre">Sql.get_define()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.Sql.get_define_custom"><code class="docutils
literal notranslate"><span
class="pre">Sql.get_define_custom()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.Sql.get_plugin"><code class="docutils literal
notranslate"><span class="pre">Sql.get_plugin()</span></code></a></li>
+<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.Sql.get_stm_list"><code class="docutils literal
notranslate"><span class="pre">Sql.get_stm_list()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.Sql.set_downstream"><code class="docutils
literal notranslate"><span
class="pre">Sql.set_downstream()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.Sql.set_upstream"><code class="docutils literal
notranslate"><span class="pre">Sql.set_upstream()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.Sql.DEFAULT_CONDITION_RESULT"><code
class="docutils literal notranslate"><span
class="pre">Sql.DEFAULT_CONDITION_RESULT</span></code></a></li>
@@ -8097,7 +8098,7 @@ directly set as python property, so we Override
Task.task_params here.</p>
<dl class="py class">
<dt class="sig sig-object py" id="pydolphinscheduler.tasks.Sql">
-<em class="property"><span class="pre">class</span><span class="w">
</span></em><span class="sig-prename descclassname"><span
class="pre">pydolphinscheduler.tasks.</span></span><span class="sig-name
descname"><span class="pre">Sql</span></span><span
class="sig-paren">(</span><em class="sig-param"><span class="n"><span
class="pre">name</span></span><span class="p"><span
class="pre">:</span></span><span class="w"> </span><span class="n"><span
class="pre">str</span></span></em>, <em class=" [...]
+<em class="property"><span class="pre">class</span><span class="w">
</span></em><span class="sig-prename descclassname"><span
class="pre">pydolphinscheduler.tasks.</span></span><span class="sig-name
descname"><span class="pre">Sql</span></span><span
class="sig-paren">(</span><em class="sig-param"><span class="n"><span
class="pre">name</span></span><span class="p"><span
class="pre">:</span></span><span class="w"> </span><span class="n"><span
class="pre">str</span></span></em>, <em class=" [...]
<dd><p>Bases: <a class="reference internal"
href="#pydolphinscheduler.core.Task"
title="pydolphinscheduler.core.task.Task"><code class="xref py py-class
docutils literal notranslate"><span class="pre">Task</span></code></a></p>
<p>Task SQL object, declare behavior for SQL task to dolphinscheduler.</p>
<p>It should run sql job in multiply sql lik engine, such as:
@@ -8111,6 +8112,23 @@ directly set as python property, so we Override
Task.task_params here.</p>
- SQLServer
You provider datasource_name contain connection information, it decisions which
database type and database instance would run this sql.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>name</strong> – SQL task name</p></li>
+<li><p><strong>datasource_name</strong> – datasource name in dolphinscheduler,
the name must exists and must be <code class="docutils literal
notranslate"><span class="pre">online</span></code>
+datasource instead of <code class="docutils literal notranslate"><span
class="pre">test</span></code>.</p></li>
+<li><p><strong>sql</strong> – SQL statement, the sql script you want to run.
Support resource plugin in this parameter.</p></li>
+<li><p><strong>sql_type</strong> – SQL type, whether sql statement is select
query or not. If not provided, it will be auto
+detected according to sql statement using <a class="reference internal"
href="tasks/sql.html#pydolphinscheduler.tasks.sql.Sql.sql_type"
title="pydolphinscheduler.tasks.sql.Sql.sql_type"><code class="xref py py-func
docutils literal notranslate"><span
class="pre">pydolphinscheduler.tasks.sql.Sql.sql_type()</span></code></a>, and
you
+can also set it manually. by <code class="docutils literal notranslate"><span
class="pre">SqlType.SELECT</span></code> for query statement or <code
class="docutils literal notranslate"><span
class="pre">SqlType.NOT_SELECT</span></code> for not
+query statement.</p></li>
+<li><p><strong>pre_statements</strong> – SQL statements to be executed before
the main SQL statement.</p></li>
+<li><p><strong>post_statements</strong> – SQL statements to be executed after
the main SQL statement.</p></li>
+<li><p><strong>display_rows</strong> – The number of record rows number to be
displayed in the SQL task log, default is 10.</p></li>
+</ul>
+</dd>
+</dl>
<dl class="py method">
<dt class="sig sig-object py" id="pydolphinscheduler.tasks.Sql._get_attr">
<span class="sig-name descname"><span class="pre">_get_attr</span></span><span
class="sig-paren">(</span><span class="sig-paren">)</span> <span
class="sig-return"><span class="sig-return-icon">→</span> <span
class="sig-return-typehint"><span class="pre">Set</span><span class="p"><span
class="pre">[</span></span><span class="pre">str</span><span class="p"><span
class="pre">]</span></span></span></span><a class="headerlink"
href="#pydolphinscheduler.tasks.Sql._get_attr" title="Perma [...]
@@ -8204,6 +8222,20 @@ object tries to communicate with Java gateway server.</p>
workflow.resource_plugin.</p>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="pydolphinscheduler.tasks.Sql.get_stm_list">
+<em class="property"><span class="pre">static</span><span class="w">
</span></em><span class="sig-name descname"><span
class="pre">get_stm_list</span></span><span class="sig-paren">(</span><em
class="sig-param"><span class="n"><span class="pre">stm</span></span><span
class="p"><span class="pre">:</span></span><span class="w"> </span><span
class="n"><span class="pre">Union</span><span class="p"><span
class="pre">[</span></span><span class="pre">str</span><span class="p"><span
class="pre"> [...]
+<dd><p>Convert statement to str of list.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>stm</strong> – statements string</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>statements list</p>
+</dd>
+</dl>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="pydolphinscheduler.tasks.Sql.set_downstream">
<span class="sig-name descname"><span
class="pre">set_downstream</span></span><span class="sig-paren">(</span><em
class="sig-param"><span class="n"><span class="pre">tasks</span></span><span
class="p"><span class="pre">:</span></span><span class="w"> </span><span
class="n"><span class="pre">Union</span><span class="p"><span
class="pre">[</span></span><a class="reference internal"
href="#pydolphinscheduler.core.Task"
title="pydolphinscheduler.core.task.Task"><span class="pre">Task</span>< [...]
diff --git a/python/main/genindex.html b/python/main/genindex.html
index b03a685910..f7e1aa5ac3 100644
--- a/python/main/genindex.html
+++ b/python/main/genindex.html
@@ -503,6 +503,8 @@
</li>
</ul></li>
<li><a
href="api.html#pydolphinscheduler.core.Workflow._ensure_side_model_exists">_ensure_side_model_exists()
(pydolphinscheduler.core.Workflow method)</a>
+</li>
+ <li><a
href="tasks/func_wrap.html#pydolphinscheduler.tasks.func_wrap._exists_other_decorator">_exists_other_decorator()
(in module pydolphinscheduler.tasks.func_wrap)</a>
</li>
<li><a
href="api.html#pydolphinscheduler.core.Engine._get_attr">_get_attr()
(pydolphinscheduler.core.Engine method)</a>
@@ -560,8 +562,6 @@
<li><a
href="api.html#pydolphinscheduler.tasks.Switch._get_attr">(pydolphinscheduler.tasks.Switch
method)</a>
</li>
</ul></li>
- <li><a
href="tasks/func_wrap.html#pydolphinscheduler.tasks.func_wrap._get_func_str">_get_func_str()
(in module pydolphinscheduler.tasks.func_wrap)</a>
-</li>
<li><a
href="resources_plugin/github.html#pydolphinscheduler.resources_plugin.github.GitHub._git_file_info">_git_file_info
(pydolphinscheduler.resources_plugin.github.GitHub attribute)</a>
</li>
<li><a
href="api.html#pydolphinscheduler.core.Workflow._handle_root_relation">_handle_root_relation()
(pydolphinscheduler.core.Workflow method)</a>
@@ -2320,6 +2320,12 @@
<li><a
href="api.html#pydolphinscheduler.tasks.MR.get_resource_info">(pydolphinscheduler.tasks.MR
method)</a>
</li>
<li><a
href="api.html#pydolphinscheduler.tasks.Spark.get_resource_info">(pydolphinscheduler.tasks.Spark
method)</a>
+</li>
+ </ul></li>
+ <li><a
href="api.html#pydolphinscheduler.tasks.Sql.get_stm_list">get_stm_list()
(pydolphinscheduler.tasks.Sql static method)</a>
+
+ <ul>
+ <li><a
href="tasks/sql.html#pydolphinscheduler.tasks.sql.Sql.get_stm_list">(pydolphinscheduler.tasks.sql.Sql
static method)</a>
</li>
</ul></li>
<li><a
href="api.html#pydolphinscheduler.core.Workflow.get_task">get_task()
(pydolphinscheduler.core.Workflow method)</a>
diff --git a/python/main/index.html b/python/main/index.html
index 8958ec3ace..9d6ed2a97c 100644
--- a/python/main/index.html
+++ b/python/main/index.html
@@ -199,7 +199,7 @@ then go and see <a class="reference internal"
href="tutorial.html"><span class="
</ul>
</li>
<li class="toctree-l1"><a class="reference internal"
href="tasks/index.html">Tasks</a><ul>
-<li class="toctree-l2"><a class="reference internal"
href="tasks/func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="tasks/func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="tasks/shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="tasks/sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="tasks/python.html">Python</a></li>
diff --git a/python/main/objects.inv b/python/main/objects.inv
index 98745f4acb..f7aaa66ffe 100644
Binary files a/python/main/objects.inv and b/python/main/objects.inv differ
diff --git a/python/main/searchindex.js b/python/main/searchindex.js
index 76f7ac7a06..f46daa8056 100644
--- a/python/main/searchindex.js
+++ b/python/main/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"docnames": ["api", "changelog", "cli", "concept", "config",
"howto/index", "howto/multi-resources", "howto/remote-submit", "index",
"resources_plugin/develop", "resources_plugin/github",
"resources_plugin/gitlab", "resources_plugin/index", "resources_plugin/local",
"resources_plugin/oss", "resources_plugin/resource-plugin",
"resources_plugin/s3", "start", "tasks/condition", "tasks/datax",
"tasks/dependent", "tasks/dvc", "tasks/flink", "tasks/func_wrap", "tasks/http",
"t [...]
\ No newline at end of file
+Search.setIndex({"docnames": ["api", "changelog", "cli", "concept", "config",
"howto/index", "howto/multi-resources", "howto/remote-submit", "index",
"resources_plugin/develop", "resources_plugin/github",
"resources_plugin/gitlab", "resources_plugin/index", "resources_plugin/local",
"resources_plugin/oss", "resources_plugin/resource-plugin",
"resources_plugin/s3", "start", "tasks/condition", "tasks/datax",
"tasks/dependent", "tasks/dvc", "tasks/flink", "tasks/func_wrap", "tasks/http",
"t [...]
\ No newline at end of file
diff --git a/python/main/tasks/condition.html b/python/main/tasks/condition.html
index cd54375073..b36f775ae3 100644
--- a/python/main/tasks/condition.html
+++ b/python/main/tasks/condition.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/datax.html b/python/main/tasks/datax.html
index 47ee3b6d56..ee9a3b3877 100644
--- a/python/main/tasks/datax.html
+++ b/python/main/tasks/datax.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/dependent.html b/python/main/tasks/dependent.html
index a181a81be3..470a7be918 100644
--- a/python/main/tasks/dependent.html
+++ b/python/main/tasks/dependent.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/dvc.html b/python/main/tasks/dvc.html
index 25d8394ceb..cf7725b957 100644
--- a/python/main/tasks/dvc.html
+++ b/python/main/tasks/dvc.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/flink.html b/python/main/tasks/flink.html
index 353e0061e5..be0c70aba9 100644
--- a/python/main/tasks/flink.html
+++ b/python/main/tasks/flink.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/func_wrap.html b/python/main/tasks/func_wrap.html
index 28f2d1a685..b20d91a5cc 100644
--- a/python/main/tasks/func_wrap.html
+++ b/python/main/tasks/func_wrap.html
@@ -4,7 +4,7 @@
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1:
http://docutils.sourceforge.net/"; />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <title>Python Function Wrapper — pydolphinscheduler 0.0.1
documentation</title>
+ <title>Python Function Decorate — pydolphinscheduler 0.0.1
documentation</title>
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../_static/tabs.css" type="text/css" />
@@ -49,10 +49,10 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2 current"><a class="current reference internal"
href="#">Python Function Wrapper</a><ul>
+<li class="toctree-l2 current"><a class="current reference internal"
href="#">Python Function Decorate</a><ul>
<li class="toctree-l3"><a class="reference internal"
href="#example">Example</a></li>
<li class="toctree-l3"><a class="reference internal"
href="#module-pydolphinscheduler.tasks.func_wrap">Dive Into</a><ul>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.func_wrap._get_func_str"><code class="docutils
literal notranslate"><span class="pre">_get_func_str()</span></code></a></li>
+<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.func_wrap._exists_other_decorator"><code
class="docutils literal notranslate"><span
class="pre">_exists_other_decorator()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.func_wrap.task"><code class="docutils literal
notranslate"><span class="pre">task()</span></code></a></li>
</ul>
</li>
@@ -102,7 +102,7 @@
<ul class="wy-breadcrumbs">
<li><a href="../index.html" class="icon icon-home"></a></li>
<li class="breadcrumb-item"><a href="index.html">Tasks</a></li>
- <li class="breadcrumb-item active">Python Function Wrapper</li>
+ <li class="breadcrumb-item active">Python Function Decorate</li>
<li class="wy-breadcrumbs-aside">
<a href="../_sources/tasks/func_wrap.rst.txt" rel="nofollow"> View
page source</a>
</li>
@@ -112,9 +112,49 @@
<div role="main" class="document" itemscope="itemscope"
itemtype="http://schema.org/Article";>
<div itemprop="articleBody">
- <section id="python-function-wrapper">
-<h1>Python Function Wrapper<a class="headerlink"
href="#python-function-wrapper" title="Permalink to this heading"></a></h1>
-<p>A decorator covert Python function into pydolphinscheduler’s task.</p>
+ <section id="python-function-decorate">
+<h1>Python Function Decorate<a class="headerlink"
href="#python-function-decorate" title="Permalink to this heading"></a></h1>
+<p>A decorator covert Python function into pydolphinscheduler’s task. Python
function decorator use decorate
+<code class="code docutils literal notranslate"><span
class="pre">@task</span></code> from <code class="code docutils literal
notranslate"><span class="pre">from</span> <span
class="pre">pydolphinscheduler.tasks.func_wrap</span> <span
class="pre">import</span> <span class="pre">task</span></code> to convert
Python function into
+a single Python task of dolphinscheduler.</p>
+<p>Because we have to covert the whole Python definition into multiple Python
task in dolphinscheduler, and all of
+the seperated Python task will be executed in the different Python process, so
we need to separate not only the
+python function code, but also the all variables and the imported modules
related to decorated function.</p>
+<p>For example, we decorated function <code class="docutils literal
notranslate"><span class="pre">depend_import</span></code> in definition</p>
+<div class="highlight-python notranslate"><div
class="highlight"><pre><span></span><span class="kn">import</span> <span
class="nn">time</span>
+
+<span class="nd">@task</span>
+<span class="k">def</span> <span class="nf">depend_import</span><span
class="p">():</span>
+ <span class="n">time</span><span class="o">.</span><span
class="n">sleep</span><span class="p">(</span><span class="mi">2</span><span
class="p">)</span>
+</pre></div>
+</div>
+<p>and we can see functon <code class="docutils literal notranslate"><span
class="pre">depend_import</span></code> depend on other modules, it use <code
class="code docutils literal notranslate"><span
class="pre">time.sleep(2)</span></code> from module <code class="code docutils
literal notranslate"><span class="pre">time</span></code>
+to sleep 2 seconds. So when we want to separate this function into
dolphinscheduler task, need to include the imported
+<code class="code docutils literal notranslate"><span
class="pre">time</span></code> module.</p>
+<p>which means we not only post code</p>
+<div class="highlight-python notranslate"><div
class="highlight"><pre><span></span><span class="k">def</span> <span
class="nf">depend_import</span><span class="p">():</span>
+ <span class="n">time</span><span class="o">.</span><span
class="n">sleep</span><span class="p">(</span><span class="mi">2</span><span
class="p">)</span>
+
+<span class="n">depend_import</span><span class="p">()</span>
+</pre></div>
+</div>
+<p>to dolphinscheduler Python task, we post the dependencies of this function
as well, so you will see this in
+dolphinscheduler Python task to make it work. And if you use the global
variables or other function in the
+decorated function, it will also including them as well.</p>
+<div class="highlight-python notranslate"><div
class="highlight"><pre><span></span><span class="kn">import</span> <span
class="nn">time</span>
+
+<span class="k">def</span> <span class="nf">depend_import</span><span
class="p">():</span>
+ <span class="n">time</span><span class="o">.</span><span
class="n">sleep</span><span class="p">(</span><span class="mi">2</span><span
class="p">)</span>
+
+<span class="n">depend_import</span><span class="p">()</span>
+</pre></div>
+</div>
+<div class="admonition note">
+<p class="admonition-title">Note</p>
+<p>We use third party library <a class="reference external"
href="https://pypi.org/project/stmdency";>stmdency</a> to get the dependencies
statement
+of current function, so if you find some unexpected behavior you can report
bug to <cite>apache-dolphinscheduler</cite>
+or <cite>stmdency</cite>.</p>
+</div>
<section id="example">
<h2>Example<a class="headerlink" href="#example" title="Permalink to this
heading"></a></h2>
<div class="highlight-default notranslate"><div
class="highlight"><pre><span></span><span class="c1"># [start
package_import]</span>
@@ -126,30 +166,44 @@
<span class="c1"># [end package_import]</span>
+<span class="n">scope_global</span> <span class="o">=</span> <span
class="s2">"global-var"</span>
+
<span class="c1"># [start task_declare]</span>
<span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_parent</span><span
class="p">():</span>
+<span class="k">def</span> <span class="nf">print_something</span><span
class="p">():</span>
<span class="sd">"""First task in this
workflow."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo hello pydolphinscheduler"</span><span
class="p">)</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="s2">"hello python function wrap task"</span><span
class="p">)</span>
+
+
+<span class="nd">@task</span>
+<span class="k">def</span> <span class="nf">depend_import</span><span
class="p">():</span>
+ <span class="sd">"""Depend on import
module."""</span>
+ <span class="n">time</span><span class="o">.</span><span
class="n">sleep</span><span class="p">(</span><span class="mi">2</span><span
class="p">)</span>
<span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_child_one</span><span
class="p">():</span>
- <span class="sd">"""Child task will be run parallel after
task ``task_parent`` finished."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo 'child one'"</span><span class="p">)</span>
+<span class="k">def</span> <span class="nf">depend_global_var</span><span
class="p">():</span>
+ <span class="sd">"""Depend on global
var."""</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="sa">f</span><span class="s2">"Use global variable </span><span
class="si">{</span><span class="n">scope_global</span><span
class="si">}</span><span class="s2">"</span><span class="p">)</span>
<span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_child_two</span><span
class="p">():</span>
- <span class="sd">"""Child task will be run parallel after
task ``task_parent`` finished."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo 'child two'"</span><span class="p">)</span>
+<span class="k">def</span> <span class="nf">depend_local_var</span><span
class="p">():</span>
+ <span class="sd">"""Depend on local
variable."""</span>
+ <span class="n">scope_global</span> <span class="o">=</span> <span
class="s2">"local"</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="sa">f</span><span class="s2">"Use local variable overwrite global
</span><span class="si">{</span><span class="n">scope_global</span><span
class="si">}</span><span class="s2">"</span><span class="p">)</span>
+
+
+<span class="k">def</span> <span class="nf">foo</span><span
class="p">():</span>
+ <span class="sd">"""Call in other
task."""</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="s2">"this is a global function"</span><span class="p">)</span>
<span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_union</span><span
class="p">():</span>
- <span class="sd">"""Last task in this
workflow."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo union"</span><span class="p">)</span>
+<span class="k">def</span> <span class="nf">depend_func</span><span
class="p">():</span>
+ <span class="sd">"""Depend on global
function."""</span>
+ <span class="n">foo</span><span class="p">()</span>
<span class="c1"># [end task_declare]</span>
@@ -164,14 +218,14 @@
<span class="c1"># [end workflow_declare]</span>
<span class="c1"># [start task_relation_declare]</span>
- <span class="n">task_group</span> <span class="o">=</span> <span
class="p">[</span><span class="n">task_child_one</span><span
class="p">(),</span> <span class="n">task_child_two</span><span
class="p">()]</span>
- <span class="n">task_parent</span><span class="p">()</span><span
class="o">.</span><span class="n">set_downstream</span><span
class="p">(</span><span class="n">task_group</span><span class="p">)</span>
+ <span class="n">task_group</span> <span class="o">=</span> <span
class="p">[</span><span class="n">depend_import</span><span
class="p">(),</span> <span class="n">depend_global_var</span><span
class="p">()]</span>
+ <span class="n">print_something</span><span class="p">()</span><span
class="o">.</span><span class="n">set_downstream</span><span
class="p">(</span><span class="n">task_group</span><span class="p">)</span>
- <span class="n">task_union</span><span class="p">()</span> <span
class="o"><<</span> <span class="n">task_group</span>
+ <span class="n">task_group</span> <span class="o">>></span> <span
class="n">depend_local_var</span><span class="p">()</span> <span
class="o">>></span> <span class="n">depend_func</span><span
class="p">()</span>
<span class="c1"># [end task_relation_declare]</span>
<span class="c1"># [start submit_or_run]</span>
- <span class="n">workflow</span><span class="o">.</span><span
class="n">run</span><span class="p">()</span>
+ <span class="n">workflow</span><span class="o">.</span><span
class="n">submit</span><span class="p">()</span>
<span class="c1"># [end submit_or_run]</span>
</pre></div>
</div>
@@ -180,9 +234,9 @@
<span id="dive-into"></span><h2>Dive Into<a class="headerlink"
href="#module-pydolphinscheduler.tasks.func_wrap" title="Permalink to this
heading"></a></h2>
<p>Task function wrapper allows using decorator to create a task.</p>
<dl class="py function">
-<dt class="sig sig-object py"
id="pydolphinscheduler.tasks.func_wrap._get_func_str">
-<span class="sig-prename descclassname"><span
class="pre">pydolphinscheduler.tasks.func_wrap.</span></span><span
class="sig-name descname"><span class="pre">_get_func_str</span></span><span
class="sig-paren">(</span><em class="sig-param"><span class="n"><span
class="pre">func</span></span><span class="p"><span
class="pre">:</span></span><span class="w"> </span><span class="n"><span
class="pre">function</span></span></em><span class="sig-paren">)</span> <span
class="sig-return"><span clas [...]
-<dd><p>Get Python function string without indent from decorator.</p>
+<dt class="sig sig-object py"
id="pydolphinscheduler.tasks.func_wrap._exists_other_decorator">
+<span class="sig-prename descclassname"><span
class="pre">pydolphinscheduler.tasks.func_wrap.</span></span><span
class="sig-name descname"><span
class="pre">_exists_other_decorator</span></span><span
class="sig-paren">(</span><em class="sig-param"><span class="n"><span
class="pre">func</span></span><span class="p"><span
class="pre">:</span></span><span class="w"> </span><span class="n"><span
class="pre">function</span></span></em><span class="sig-paren">)</span> <span
class="sig-return"> [...]
+<dd><p>Check if the function has other decorators except @task.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>func</strong> – The function which wraps by
decorator <code class="docutils literal notranslate"><span
class="pre">@task</span></code>.</p>
@@ -193,7 +247,12 @@
<dl class="py function">
<dt class="sig sig-object py" id="pydolphinscheduler.tasks.func_wrap.task">
<span class="sig-prename descclassname"><span
class="pre">pydolphinscheduler.tasks.func_wrap.</span></span><span
class="sig-name descname"><span class="pre">task</span></span><span
class="sig-paren">(</span><em class="sig-param"><span class="n"><span
class="pre">func</span></span><span class="p"><span
class="pre">:</span></span><span class="w"> </span><span class="n"><span
class="pre">function</span></span></em><span class="sig-paren">)</span><a
class="reference internal" href="../_modul [...]
-<dd><p>Decorate which covert Python function into pydolphinscheduler task.</p>
+<dd><p>Decorate which covert Python functions into pydolphinscheduler task.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>func</strong> – The function which wraps by
decorator <code class="docutils literal notranslate"><span
class="pre">@task</span></code>.</p>
+</dd>
+</dl>
</dd></dl>
</section>
diff --git a/python/main/tasks/http.html b/python/main/tasks/http.html
index 8fd8b7d8ca..b5c9eed1fb 100644
--- a/python/main/tasks/http.html
+++ b/python/main/tasks/http.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/index.html b/python/main/tasks/index.html
index d17e5fe941..39bb111214 100644
--- a/python/main/tasks/index.html
+++ b/python/main/tasks/index.html
@@ -25,7 +25,7 @@
<script src="../_static/js/theme.js"></script>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
- <link rel="next" title="Python Function Wrapper" href="func_wrap.html" />
+ <link rel="next" title="Python Function Decorate" href="func_wrap.html" />
<link rel="prev" title="Concepts" href="../concept.html" />
</head>
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="current reference internal"
href="#">Tasks</a><ul>
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
@@ -108,7 +108,7 @@
<p>In this section</p>
<div class="toctree-wrapper compound">
<ul>
-<li class="toctree-l1"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l1"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l1"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l1"><a class="reference internal"
href="python.html">Python</a></li>
@@ -137,7 +137,7 @@
</div>
<footer><div class="rst-footer-buttons" role="navigation"
aria-label="Footer">
<a href="../concept.html" class="btn btn-neutral float-left"
title="Concepts" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"
aria-hidden="true"></span> Previous</a>
- <a href="func_wrap.html" class="btn btn-neutral float-right"
title="Python Function Wrapper" accesskey="n" rel="next">Next <span class="fa
fa-arrow-circle-right" aria-hidden="true"></span></a>
+ <a href="func_wrap.html" class="btn btn-neutral float-right"
title="Python Function Decorate" accesskey="n" rel="next">Next <span class="fa
fa-arrow-circle-right" aria-hidden="true"></span></a>
</div>
<hr/>
diff --git a/python/main/tasks/kubernetes.html
b/python/main/tasks/kubernetes.html
index 3a5dca90f3..10ba7d28de 100644
--- a/python/main/tasks/kubernetes.html
+++ b/python/main/tasks/kubernetes.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/map_reduce.html
b/python/main/tasks/map_reduce.html
index c4653cc681..b02936437a 100644
--- a/python/main/tasks/map_reduce.html
+++ b/python/main/tasks/map_reduce.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/mlflow.html b/python/main/tasks/mlflow.html
index 5a9b642cec..9361e9851c 100644
--- a/python/main/tasks/mlflow.html
+++ b/python/main/tasks/mlflow.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/openmldb.html b/python/main/tasks/openmldb.html
index f8c44e1e5b..906d5ff485 100644
--- a/python/main/tasks/openmldb.html
+++ b/python/main/tasks/openmldb.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/procedure.html b/python/main/tasks/procedure.html
index 9602abcb64..019478a77d 100644
--- a/python/main/tasks/procedure.html
+++ b/python/main/tasks/procedure.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/python.html b/python/main/tasks/python.html
index 591ccf3cbc..0679bd9428 100644
--- a/python/main/tasks/python.html
+++ b/python/main/tasks/python.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2 current"><a class="current reference internal"
href="#">Python</a><ul>
diff --git a/python/main/tasks/pytorch.html b/python/main/tasks/pytorch.html
index 310083b28c..8d0cc74cc7 100644
--- a/python/main/tasks/pytorch.html
+++ b/python/main/tasks/pytorch.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/sagemaker.html b/python/main/tasks/sagemaker.html
index 9788bad47e..457c7a6b70 100644
--- a/python/main/tasks/sagemaker.html
+++ b/python/main/tasks/sagemaker.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/shell.html b/python/main/tasks/shell.html
index 3980a94f47..bbf18fc4c6 100644
--- a/python/main/tasks/shell.html
+++ b/python/main/tasks/shell.html
@@ -26,7 +26,7 @@
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="SQL" href="sql.html" />
- <link rel="prev" title="Python Function Wrapper" href="func_wrap.html" />
+ <link rel="prev" title="Python Function Decorate" href="func_wrap.html" />
</head>
<body class="wy-body-for-nav">
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2 current"><a class="current reference internal"
href="#">Shell</a><ul>
<li class="toctree-l3"><a class="reference internal"
href="#example">Example</a></li>
<li class="toctree-l3"><a class="reference internal"
href="#module-pydolphinscheduler.tasks.shell">Dive Into</a><ul>
@@ -240,7 +240,7 @@
</div>
</div>
<footer><div class="rst-footer-buttons" role="navigation"
aria-label="Footer">
- <a href="func_wrap.html" class="btn btn-neutral float-left"
title="Python Function Wrapper" accesskey="p" rel="prev"><span class="fa
fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+ <a href="func_wrap.html" class="btn btn-neutral float-left"
title="Python Function Decorate" accesskey="p" rel="prev"><span class="fa
fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
<a href="sql.html" class="btn btn-neutral float-right" title="SQL"
accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"
aria-hidden="true"></span></a>
</div>
diff --git a/python/main/tasks/spark.html b/python/main/tasks/spark.html
index ba897e5cf1..88edc6996b 100644
--- a/python/main/tasks/spark.html
+++ b/python/main/tasks/spark.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/sql.html b/python/main/tasks/sql.html
index 07df04de3f..b618d3777c 100644
--- a/python/main/tasks/sql.html
+++ b/python/main/tasks/sql.html
@@ -49,24 +49,13 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2 current"><a class="current reference internal"
href="#">SQL</a><ul>
-<li class="toctree-l3"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql"><code class="docutils literal
notranslate"><span class="pre">Sql</span></code></a><ul>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql._downstream_task_codes"><code
class="docutils literal notranslate"><span
class="pre">Sql._downstream_task_codes</span></code></a></li>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql._task_custom_attr"><code
class="docutils literal notranslate"><span
class="pre">Sql._task_custom_attr</span></code></a></li>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql._task_relation"><code class="docutils
literal notranslate"><span class="pre">Sql._task_relation</span></code></a></li>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql._timeout"><code class="docutils literal
notranslate"><span class="pre">Sql._timeout</span></code></a></li>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql._upstream_task_codes"><code
class="docutils literal notranslate"><span
class="pre">Sql._upstream_task_codes</span></code></a></li>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql.ext"><code class="docutils literal
notranslate"><span class="pre">Sql.ext</span></code></a></li>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql.ext_attr"><code class="docutils literal
notranslate"><span class="pre">Sql.ext_attr</span></code></a></li>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql.sql_type"><code class="docutils literal
notranslate"><span class="pre">Sql.sql_type</span></code></a></li>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql.task_params"><code class="docutils
literal notranslate"><span class="pre">Sql.task_params</span></code></a></li>
-</ul>
-</li>
-<li class="toctree-l3"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.SqlType"><code class="docutils literal
notranslate"><span class="pre">SqlType</span></code></a><ul>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.SqlType.NOT_SELECT"><code class="docutils
literal notranslate"><span class="pre">SqlType.NOT_SELECT</span></code></a></li>
-<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.SqlType.SELECT"><code class="docutils
literal notranslate"><span class="pre">SqlType.SELECT</span></code></a></li>
+<li class="toctree-l3"><a class="reference internal"
href="#example">Example</a></li>
+<li class="toctree-l3"><a class="reference internal"
href="#module-pydolphinscheduler.tasks.sql">Dive Into</a><ul>
+<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql"><code class="docutils literal
notranslate"><span class="pre">Sql</span></code></a></li>
+<li class="toctree-l4"><a class="reference internal"
href="#pydolphinscheduler.tasks.sql.SqlType"><code class="docutils literal
notranslate"><span class="pre">SqlType</span></code></a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal"
href="#yaml-file-example">YAML file example</a></li>
@@ -124,12 +113,129 @@
<div role="main" class="document" itemscope="itemscope"
itemtype="http://schema.org/Article";>
<div itemprop="articleBody">
- <section id="module-pydolphinscheduler.tasks.sql">
-<span id="sql"></span><h1>SQL<a class="headerlink"
href="#module-pydolphinscheduler.tasks.sql" title="Permalink to this
heading"></a></h1>
+ <section id="sql">
+<h1>SQL<a class="headerlink" href="#sql" title="Permalink to this
heading"></a></h1>
+<p>An example for SQL task, including how to use it and the detail of it
parameters.</p>
+<p>This task type can execute multiple type of database SQL, which includes</p>
+<ul class="simple">
+<li><p>MySQL</p></li>
+<li><p>PostgreSQL</p></li>
+<li><p>Oracle</p></li>
+<li><p>SQL Server</p></li>
+<li><p>DB2</p></li>
+<li><p>Hive</p></li>
+<li><p>Presto</p></li>
+<li><p>Trino</p></li>
+<li><p>ClickHouse</p></li>
+</ul>
+<section id="example">
+<h2>Example<a class="headerlink" href="#example" title="Permalink to this
heading"></a></h2>
+<div class="highlight-default notranslate"><div
class="highlight"><pre><span></span>
+<span class="sd">"""A example workflow for task
SQL."""</span>
+<span class="kn">from</span> <span class="nn">pathlib</span> <span
class="kn">import</span> <span class="n">Path</span>
+
+<span class="kn">from</span> <span
class="nn">pydolphinscheduler.core.workflow</span> <span
class="kn">import</span> <span class="n">Workflow</span>
+<span class="kn">from</span> <span
class="nn">pydolphinscheduler.resources_plugin</span> <span
class="kn">import</span> <span class="n">Local</span>
+<span class="kn">from</span> <span
class="nn">pydolphinscheduler.tasks.sql</span> <span class="kn">import</span>
<span class="n">Sql</span><span class="p">,</span> <span
class="n">SqlType</span>
+
+<span class="k">with</span> <span class="n">Workflow</span><span
class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"task_sql_example"</span><span class="p">,</span>
+<span class="p">)</span> <span class="k">as</span> <span
class="n">workflow</span><span class="p">:</span>
+ <span class="c1"># [start bare_sql_desc]</span>
+ <span class="n">bare_sql</span> <span class="o">=</span> <span
class="n">Sql</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"bare_sql"</span><span class="p">,</span>
+ <span class="n">datasource_name</span><span class="o">=</span><span
class="s2">"metadata"</span><span class="p">,</span>
+ <span class="n">sql</span><span class="o">=</span><span
class="s2">"select * from t_ds_version"</span><span class="p">,</span>
+ <span class="p">)</span>
+ <span class="c1"># [end bare_sql_desc]</span>
+ <span class="c1"># [start sql_file_desc]</span>
+ <span class="n">sql_file</span> <span class="o">=</span> <span
class="n">Sql</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"sql_file"</span><span class="p">,</span>
+ <span class="n">datasource_name</span><span class="o">=</span><span
class="s2">"metadata"</span><span class="p">,</span>
+ <span class="n">sql</span><span class="o">=</span><span
class="s2">"ext/example.sql"</span><span class="p">,</span>
+ <span class="n">sql_type</span><span class="o">=</span><span
class="n">SqlType</span><span class="o">.</span><span
class="n">SELECT</span><span class="p">,</span>
+ <span class="n">resource_plugin</span><span class="o">=</span><span
class="n">Local</span><span class="p">(</span><span
class="n">prefix</span><span class="o">=</span><span class="nb">str</span><span
class="p">(</span><span class="n">Path</span><span class="p">(</span><span
class="vm">__file__</span><span class="p">)</span><span class="o">.</span><span
class="n">parent</span><span class="p">)),</span>
+ <span class="p">)</span>
+ <span class="c1"># [end sql_file_desc]</span>
+ <span class="c1"># [start sql_with_pre_post_desc]</span>
+ <span class="n">sql_with_pre_post</span> <span class="o">=</span> <span
class="n">Sql</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"sql_with_pre_post"</span><span class="p">,</span>
+ <span class="n">datasource_name</span><span class="o">=</span><span
class="s2">"metadata"</span><span class="p">,</span>
+ <span class="n">sql</span><span class="o">=</span><span
class="s2">"select * from t_ds_version"</span><span class="p">,</span>
+ <span class="n">pre_statements</span><span class="o">=</span><span
class="p">[</span>
+ <span class="s2">"update table_one set version =
'1.3.6'"</span><span class="p">,</span>
+ <span class="s2">"delete from table_two where version =
'1.3.6'"</span><span class="p">,</span>
+ <span class="p">],</span>
+ <span class="n">post_statements</span><span class="o">=</span><span
class="s2">"update table_one set version =
'3.0.0'"</span><span class="p">,</span>
+ <span class="p">)</span>
+ <span class="c1"># [end sql_with_pre_post_desc]</span>
+
+ <span class="n">bare_sql</span> <span class="o">>></span> <span
class="p">[</span>
+ <span class="n">sql_file</span><span class="p">,</span>
+ <span class="n">sql_with_pre_post</span><span class="p">,</span>
+ <span class="p">]</span>
+
+ <span class="n">workflow</span><span class="o">.</span><span
class="n">submit</span><span class="p">()</span>
+
+</pre></div>
+</div>
+<p>You can see that SQL task support three ways to declare SQL, which are</p>
+<ul>
+<li><p>Bare SQL: Put bare SQL statement in the <code class="docutils literal
notranslate"><span class="pre">sql</span></code> parameter, such as <code
class="docutils literal notranslate"><span class="pre">select</span> <span
class="pre">*</span> <span class="pre">from</span> <span
class="pre">table1</span></code>.</p>
+<blockquote>
+<div><div class="highlight-default notranslate"><div
class="highlight"><pre><span></span> <span class="n">bare_sql</span> <span
class="o">=</span> <span class="n">Sql</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"bare_sql"</span><span class="p">,</span>
+ <span class="n">datasource_name</span><span class="o">=</span><span
class="s2">"metadata"</span><span class="p">,</span>
+ <span class="n">sql</span><span class="o">=</span><span
class="s2">"select * from t_ds_version"</span><span class="p">,</span>
+ <span class="p">)</span>
+</pre></div>
+</div>
+</div></blockquote>
+</li>
+<li><p>SQL Files: .</p>
+<blockquote>
+<div><div class="highlight-default notranslate"><div
class="highlight"><pre><span></span> <span class="n">sql_file</span> <span
class="o">=</span> <span class="n">Sql</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"sql_file"</span><span class="p">,</span>
+ <span class="n">datasource_name</span><span class="o">=</span><span
class="s2">"metadata"</span><span class="p">,</span>
+ <span class="n">sql</span><span class="o">=</span><span
class="s2">"ext/example.sql"</span><span class="p">,</span>
+ <span class="n">sql_type</span><span class="o">=</span><span
class="n">SqlType</span><span class="o">.</span><span
class="n">SELECT</span><span class="p">,</span>
+ <span class="n">resource_plugin</span><span class="o">=</span><span
class="n">Local</span><span class="p">(</span><span
class="n">prefix</span><span class="o">=</span><span class="nb">str</span><span
class="p">(</span><span class="n">Path</span><span class="p">(</span><span
class="vm">__file__</span><span class="p">)</span><span class="o">.</span><span
class="n">parent</span><span class="p">)),</span>
+ <span class="p">)</span>
+</pre></div>
+</div>
+</div></blockquote>
+</li>
+</ul>
+<p>If you want to do some preparation before executing SQL, or do some clean
up after executing SQL, you can use
+<code class="docutils literal notranslate"><span
class="pre">pre_statements</span></code> and <code class="docutils literal
notranslate"><span class="pre">post_statements</span></code> parameters to do
that. Both <code class="docutils literal notranslate"><span
class="pre">pre_statements</span></code> and <code class="docutils literal
notranslate"><span class="pre">post_statements</span></code>
+support one or multiple statements, you can assign type sequence of SQL
statements to them if you want to execute
+multiple statements. But if you only need to execute one statement, you can
assign a string to them.</p>
+<blockquote>
+<div><div class="highlight-default notranslate"><div
class="highlight"><pre><span></span> <span
class="n">sql_with_pre_post</span> <span class="o">=</span> <span
class="n">Sql</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"sql_with_pre_post"</span><span class="p">,</span>
+ <span class="n">datasource_name</span><span class="o">=</span><span
class="s2">"metadata"</span><span class="p">,</span>
+ <span class="n">sql</span><span class="o">=</span><span
class="s2">"select * from t_ds_version"</span><span class="p">,</span>
+ <span class="n">pre_statements</span><span class="o">=</span><span
class="p">[</span>
+ <span class="s2">"update table_one set version =
'1.3.6'"</span><span class="p">,</span>
+ <span class="s2">"delete from table_two where version =
'1.3.6'"</span><span class="p">,</span>
+ <span class="p">],</span>
+ <span class="n">post_statements</span><span class="o">=</span><span
class="s2">"update table_one set version =
'3.0.0'"</span><span class="p">,</span>
+ <span class="p">)</span>
+</pre></div>
+</div>
+</div></blockquote>
+<div class="admonition note">
+<p class="admonition-title">Note</p>
+<p>Parameter <code class="docutils literal notranslate"><span
class="pre">pre_statements</span></code> and <code class="docutils literal
notranslate"><span class="pre">post_statements</span></code> only support not
query statements, such as <code class="docutils literal notranslate"><span
class="pre">create</span> <span class="pre">table</span></code>,
+<code class="docutils literal notranslate"><span class="pre">drop</span> <span
class="pre">table</span></code>, <code class="docutils literal
notranslate"><span class="pre">update</span> <span
class="pre">table</span></code> currently. And also it only support bare SQL
instead of SQL files now.</p>
+</div>
+</section>
+<section id="module-pydolphinscheduler.tasks.sql">
+<span id="dive-into"></span><h2>Dive Into<a class="headerlink"
href="#module-pydolphinscheduler.tasks.sql" title="Permalink to this
heading"></a></h2>
<p>Task sql.</p>
<dl class="py class">
<dt class="sig sig-object py" id="pydolphinscheduler.tasks.sql.Sql">
-<em class="property"><span class="pre">class</span><span class="w">
</span></em><span class="sig-prename descclassname"><span
class="pre">pydolphinscheduler.tasks.sql.</span></span><span class="sig-name
descname"><span class="pre">Sql</span></span><span
class="sig-paren">(</span><em class="sig-param"><span class="n"><span
class="pre">name</span></span><span class="p"><span
class="pre">:</span></span><span class="w"> </span><span class="n"><span
class="pre">str</span></span></em>, <em cla [...]
+<em class="property"><span class="pre">class</span><span class="w">
</span></em><span class="sig-prename descclassname"><span
class="pre">pydolphinscheduler.tasks.sql.</span></span><span class="sig-name
descname"><span class="pre">Sql</span></span><span
class="sig-paren">(</span><em class="sig-param"><span class="n"><span
class="pre">name</span></span><span class="p"><span
class="pre">:</span></span><span class="w"> </span><span class="n"><span
class="pre">str</span></span></em>, <em cla [...]
<dd><p>Bases: <a class="reference internal"
href="../api.html#pydolphinscheduler.core.Task"
title="pydolphinscheduler.core.task.Task"><code class="xref py py-class
docutils literal notranslate"><span class="pre">Task</span></code></a></p>
<p>Task SQL object, declare behavior for SQL task to dolphinscheduler.</p>
<p>It should run sql job in multiply sql lik engine, such as:
@@ -143,6 +249,37 @@
- SQLServer
You provider datasource_name contain connection information, it decisions which
database type and database instance would run this sql.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>name</strong> – SQL task name</p></li>
+<li><p><strong>datasource_name</strong> – datasource name in dolphinscheduler,
the name must exists and must be <code class="docutils literal
notranslate"><span class="pre">online</span></code>
+datasource instead of <code class="docutils literal notranslate"><span
class="pre">test</span></code>.</p></li>
+<li><p><strong>sql</strong> – SQL statement, the sql script you want to run.
Support resource plugin in this parameter.</p></li>
+<li><p><strong>sql_type</strong> – SQL type, whether sql statement is select
query or not. If not provided, it will be auto
+detected according to sql statement using <a class="reference internal"
href="#pydolphinscheduler.tasks.sql.Sql.sql_type"
title="pydolphinscheduler.tasks.sql.Sql.sql_type"><code class="xref py py-func
docutils literal notranslate"><span
class="pre">pydolphinscheduler.tasks.sql.Sql.sql_type()</span></code></a>, and
you
+can also set it manually. by <code class="docutils literal notranslate"><span
class="pre">SqlType.SELECT</span></code> for query statement or <code
class="docutils literal notranslate"><span
class="pre">SqlType.NOT_SELECT</span></code> for not
+query statement.</p></li>
+<li><p><strong>pre_statements</strong> – SQL statements to be executed before
the main SQL statement.</p></li>
+<li><p><strong>post_statements</strong> – SQL statements to be executed after
the main SQL statement.</p></li>
+<li><p><strong>display_rows</strong> – The number of record rows number to be
displayed in the SQL task log, default is 10.</p></li>
+</ul>
+</dd>
+</dl>
+<dl class="py method">
+<dt class="sig sig-object py"
id="pydolphinscheduler.tasks.sql.Sql.get_stm_list">
+<em class="property"><span class="pre">static</span><span class="w">
</span></em><span class="sig-name descname"><span
class="pre">get_stm_list</span></span><span class="sig-paren">(</span><em
class="sig-param"><span class="n"><span class="pre">stm</span></span><span
class="p"><span class="pre">:</span></span><span class="w"> </span><span
class="n"><span class="pre">Union</span><span class="p"><span
class="pre">[</span></span><span class="pre">str</span><span class="p"><span
class="pre"> [...]
+<dd><p>Convert statement to str of list.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>stm</strong> – statements string</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>statements list</p>
+</dd>
+</dl>
+</dd></dl>
+
<dl class="py attribute">
<dt class="sig sig-object py"
id="pydolphinscheduler.tasks.sql.Sql._downstream_task_codes">
<span class="sig-name descname"><span
class="pre">_downstream_task_codes</span></span><em class="property"><span
class="p"><span class="pre">:</span></span><span class="w"> </span><span
class="pre">Set</span><span class="p"><span class="pre">[</span></span><span
class="pre">int</span><span class="p"><span class="pre">]</span></span></em><a
class="headerlink"
href="#pydolphinscheduler.tasks.sql.Sql._downstream_task_codes"
title="Permalink to this definition"></a></dt>
@@ -214,6 +351,7 @@ directly set as python property, so we Override
Task.task_params here.</p>
</dd></dl>
+</section>
<section id="yaml-file-example">
<h2>YAML file example<a class="headerlink" href="#yaml-file-example"
title="Permalink to this heading"></a></h2>
<div class="highlight-yaml notranslate"><div
class="highlight"><pre><span></span><span class="c1"># Define the
workflow</span><span class="w"></span>
diff --git a/python/main/tasks/sub_workflow.html
b/python/main/tasks/sub_workflow.html
index 4c96f393b5..80d3cec77e 100644
--- a/python/main/tasks/sub_workflow.html
+++ b/python/main/tasks/sub_workflow.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tasks/switch.html b/python/main/tasks/switch.html
index 9bb7c8832f..15c46a5ae4 100644
--- a/python/main/tasks/switch.html
+++ b/python/main/tasks/switch.html
@@ -49,7 +49,7 @@
<li class="toctree-l1"><a class="reference internal"
href="../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal"
href="../concept.html">Concepts</a></li>
<li class="toctree-l1 current"><a class="reference internal"
href="index.html">Tasks</a><ul class="current">
-<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Wrapper</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="func_wrap.html">Python Function Decorate</a></li>
<li class="toctree-l2"><a class="reference internal"
href="shell.html">Shell</a></li>
<li class="toctree-l2"><a class="reference internal"
href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal"
href="python.html">Python</a></li>
diff --git a/python/main/tutorial.html b/python/main/tutorial.html
index ebe2c37126..d4d2446f9f 100644
--- a/python/main/tutorial.html
+++ b/python/main/tutorial.html
@@ -160,30 +160,44 @@ We can find more YAML file examples in <a
class="reference external" href="https
<span class="c1"># [end package_import]</span>
+<span class="n">scope_global</span> <span class="o">=</span> <span
class="s2">"global-var"</span>
+
<span class="c1"># [start task_declare]</span>
<span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_parent</span><span
class="p">():</span>
+<span class="k">def</span> <span class="nf">print_something</span><span
class="p">():</span>
<span class="sd">"""First task in this
workflow."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo hello pydolphinscheduler"</span><span
class="p">)</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="s2">"hello python function wrap task"</span><span
class="p">)</span>
<span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_child_one</span><span
class="p">():</span>
- <span class="sd">"""Child task will be run parallel after
task ``task_parent`` finished."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo 'child one'"</span><span class="p">)</span>
+<span class="k">def</span> <span class="nf">depend_import</span><span
class="p">():</span>
+ <span class="sd">"""Depend on import
module."""</span>
+ <span class="n">time</span><span class="o">.</span><span
class="n">sleep</span><span class="p">(</span><span class="mi">2</span><span
class="p">)</span>
<span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_child_two</span><span
class="p">():</span>
- <span class="sd">"""Child task will be run parallel after
task ``task_parent`` finished."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo 'child two'"</span><span class="p">)</span>
+<span class="k">def</span> <span class="nf">depend_global_var</span><span
class="p">():</span>
+ <span class="sd">"""Depend on global
var."""</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="sa">f</span><span class="s2">"Use global variable </span><span
class="si">{</span><span class="n">scope_global</span><span
class="si">}</span><span class="s2">"</span><span class="p">)</span>
<span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_union</span><span
class="p">():</span>
- <span class="sd">"""Last task in this
workflow."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo union"</span><span class="p">)</span>
+<span class="k">def</span> <span class="nf">depend_local_var</span><span
class="p">():</span>
+ <span class="sd">"""Depend on local
variable."""</span>
+ <span class="n">scope_global</span> <span class="o">=</span> <span
class="s2">"local"</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="sa">f</span><span class="s2">"Use local variable overwrite global
</span><span class="si">{</span><span class="n">scope_global</span><span
class="si">}</span><span class="s2">"</span><span class="p">)</span>
+
+
+<span class="k">def</span> <span class="nf">foo</span><span
class="p">():</span>
+ <span class="sd">"""Call in other
task."""</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="s2">"this is a global function"</span><span class="p">)</span>
+
+
+<span class="nd">@task</span>
+<span class="k">def</span> <span class="nf">depend_func</span><span
class="p">():</span>
+ <span class="sd">"""Depend on global
function."""</span>
+ <span class="n">foo</span><span class="p">()</span>
<span class="c1"># [end task_declare]</span>
@@ -198,14 +212,14 @@ We can find more YAML file examples in <a
class="reference external" href="https
<span class="c1"># [end workflow_declare]</span>
<span class="c1"># [start task_relation_declare]</span>
- <span class="n">task_group</span> <span class="o">=</span> <span
class="p">[</span><span class="n">task_child_one</span><span
class="p">(),</span> <span class="n">task_child_two</span><span
class="p">()]</span>
- <span class="n">task_parent</span><span class="p">()</span><span
class="o">.</span><span class="n">set_downstream</span><span
class="p">(</span><span class="n">task_group</span><span class="p">)</span>
+ <span class="n">task_group</span> <span class="o">=</span> <span
class="p">[</span><span class="n">depend_import</span><span
class="p">(),</span> <span class="n">depend_global_var</span><span
class="p">()]</span>
+ <span class="n">print_something</span><span class="p">()</span><span
class="o">.</span><span class="n">set_downstream</span><span
class="p">(</span><span class="n">task_group</span><span class="p">)</span>
- <span class="n">task_union</span><span class="p">()</span> <span
class="o"><<</span> <span class="n">task_group</span>
+ <span class="n">task_group</span> <span class="o">>></span> <span
class="n">depend_local_var</span><span class="p">()</span> <span
class="o">>></span> <span class="n">depend_func</span><span
class="p">()</span>
<span class="c1"># [end task_relation_declare]</span>
<span class="c1"># [start submit_or_run]</span>
- <span class="n">workflow</span><span class="o">.</span><span
class="n">run</span><span class="p">()</span>
+ <span class="n">workflow</span><span class="o">.</span><span
class="n">submit</span><span class="p">()</span>
<span class="c1"># [end submit_or_run]</span>
</pre></div>
</div>
@@ -340,27 +354,39 @@ if you are interested in it. For all arguments of object
workflow, you could fin
using <a class="reference internal"
href="tasks/func_wrap.html#pydolphinscheduler.tasks.func_wrap.task"
title="pydolphinscheduler.tasks.func_wrap.task"><code class="xref py py-func
docutils literal notranslate"><span
class="pre">pydolphinscheduler.tasks.func_wrap.task()</span></code></a>. All we
have to do is add a decorator named
<code class="code docutils literal notranslate"><span
class="pre">@task</span></code> to existing Python function, and then use
them inside <code class="xref py py-class docutils literal notranslate"><span
class="pre">pydolphinscheduler.core.workflow</span></code></p>
<div class="highlight-default notranslate"><div
class="highlight"><pre><span></span><span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_parent</span><span
class="p">():</span>
+<span class="k">def</span> <span class="nf">print_something</span><span
class="p">():</span>
<span class="sd">"""First task in this
workflow."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo hello pydolphinscheduler"</span><span
class="p">)</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="s2">"hello python function wrap task"</span><span
class="p">)</span>
<span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_child_one</span><span
class="p">():</span>
- <span class="sd">"""Child task will be run parallel after
task ``task_parent`` finished."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo 'child one'"</span><span class="p">)</span>
+<span class="k">def</span> <span class="nf">depend_import</span><span
class="p">():</span>
+ <span class="sd">"""Depend on import
module."""</span>
+ <span class="n">time</span><span class="o">.</span><span
class="n">sleep</span><span class="p">(</span><span class="mi">2</span><span
class="p">)</span>
<span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_child_two</span><span
class="p">():</span>
- <span class="sd">"""Child task will be run parallel after
task ``task_parent`` finished."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo 'child two'"</span><span class="p">)</span>
+<span class="k">def</span> <span class="nf">depend_global_var</span><span
class="p">():</span>
+ <span class="sd">"""Depend on global
var."""</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="sa">f</span><span class="s2">"Use global variable </span><span
class="si">{</span><span class="n">scope_global</span><span
class="si">}</span><span class="s2">"</span><span class="p">)</span>
<span class="nd">@task</span>
-<span class="k">def</span> <span class="nf">task_union</span><span
class="p">():</span>
- <span class="sd">"""Last task in this
workflow."""</span>
- <span class="nb">print</span><span class="p">(</span><span
class="s2">"echo union"</span><span class="p">)</span>
+<span class="k">def</span> <span class="nf">depend_local_var</span><span
class="p">():</span>
+ <span class="sd">"""Depend on local
variable."""</span>
+ <span class="n">scope_global</span> <span class="o">=</span> <span
class="s2">"local"</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="sa">f</span><span class="s2">"Use local variable overwrite global
</span><span class="si">{</span><span class="n">scope_global</span><span
class="si">}</span><span class="s2">"</span><span class="p">)</span>
+
+
+<span class="k">def</span> <span class="nf">foo</span><span
class="p">():</span>
+ <span class="sd">"""Call in other
task."""</span>
+ <span class="nb">print</span><span class="p">(</span><span
class="s2">"this is a global function"</span><span class="p">)</span>
+
+
+<span class="nd">@task</span>
+<span class="k">def</span> <span class="nf">depend_func</span><span
class="p">():</span>
+ <span class="sd">"""Depend on global
function."""</span>
+ <span class="n">foo</span><span class="p">()</span>
</pre></div>
@@ -413,10 +439,10 @@ and task <cite>task_child_two</cite> was done, because
both two task is <cite>ta
</div>
</div>
<input class="tab-input" id="tab-set--4-input--2" name="tab-set--4"
type="radio"><label class="tab-label" for="tab-set--4-input--2">Task
Decorator</label><div class="tab-content docutils container">
-<div class="highlight-default notranslate"><div
class="highlight"><pre><span></span> <span class="n">task_group</span> <span
class="o">=</span> <span class="p">[</span><span
class="n">task_child_one</span><span class="p">(),</span> <span
class="n">task_child_two</span><span class="p">()]</span>
- <span class="n">task_parent</span><span class="p">()</span><span
class="o">.</span><span class="n">set_downstream</span><span
class="p">(</span><span class="n">task_group</span><span class="p">)</span>
+<div class="highlight-default notranslate"><div
class="highlight"><pre><span></span> <span class="n">task_group</span> <span
class="o">=</span> <span class="p">[</span><span
class="n">depend_import</span><span class="p">(),</span> <span
class="n">depend_global_var</span><span class="p">()]</span>
+ <span class="n">print_something</span><span class="p">()</span><span
class="o">.</span><span class="n">set_downstream</span><span
class="p">(</span><span class="n">task_group</span><span class="p">)</span>
- <span class="n">task_union</span><span class="p">()</span> <span
class="o"><<</span> <span class="n">task_group</span>
+ <span class="n">task_group</span> <span class="o">>></span> <span
class="n">depend_local_var</span><span class="p">()</span> <span
class="o">>></span> <span class="n">depend_func</span><span
class="p">()</span>
</pre></div>
</div>
</div>
@@ -467,7 +493,7 @@ will create workflow definition as well as workflow
schedule.</p>
</div>
</div>
<input class="tab-input" id="tab-set--5-input--2" name="tab-set--5"
type="radio"><label class="tab-label" for="tab-set--5-input--2">Task
Decorator</label><div class="tab-content docutils container">
-<div class="highlight-default notranslate"><div
class="highlight"><pre><span></span> <span class="n">workflow</span><span
class="o">.</span><span class="n">run</span><span class="p">()</span>
+<div class="highlight-default notranslate"><div
class="highlight"><pre><span></span> <span class="n">workflow</span><span
class="o">.</span><span class="n">submit</span><span class="p">()</span>
</pre></div>
</div>
</div>
