This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler-website.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 77c26eb3d2 Automated deployment:
ab53994053261a99e7d6aacee296c36c07254683
77c26eb3d2 is described below
commit 77c26eb3d283ad4f6cd4079f6e322320fc968d74
Author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Jan 14 05:05:48 2024 +0000
Automated deployment: ab53994053261a99e7d6aacee296c36c07254683
---
python/2.0.5/.buildinfo | 2 +-
python/2.0.6/.buildinfo | 2 +-
python/2.0.7/.buildinfo | 2 +-
python/3.0.0/.buildinfo | 2 +-
python/3.0.1/.buildinfo | 2 +-
python/3.1.0/.buildinfo | 2 +-
python/4.0.0/.buildinfo | 2 +-
python/4.0.1/.buildinfo | 2 +-
python/4.0.2/.buildinfo | 2 +-
python/4.0.3/.buildinfo | 2 +-
python/4.0.4/.buildinfo | 2 +-
python/main/.buildinfo | 2 +-
.../_modules/pydolphinscheduler/tasks/datax.html | 53 +++++++++++++++--
python/main/api.html | 62 +++++++++++++++++--
python/main/searchindex.js | 2 +-
python/main/tasks/datax.html | 69 ++++++++++++++++++++--
python/main/tasks/shell.html | 10 +++-
python/main/tutorial.html | 23 ++++++--
18 files changed, 212 insertions(+), 31 deletions(-)
diff --git a/python/2.0.5/.buildinfo b/python/2.0.5/.buildinfo
index 4a0e7b0b3e..4d5eae6f35 100644
--- a/python/2.0.5/.buildinfo
+++ b/python/2.0.5/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 47c03025b7a4efbb8cc1003dc21203ab
+config: 028462528d0092b88afc0ef548299ce5
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/2.0.6/.buildinfo b/python/2.0.6/.buildinfo
index 6d8f728053..54f976951b 100644
--- a/python/2.0.6/.buildinfo
+++ b/python/2.0.6/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 0109ba55ff46d6555e11c54934b81e41
+config: fa5ea0bb23531315cf2b61c5e3122fad
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/2.0.7/.buildinfo b/python/2.0.7/.buildinfo
index f34c622056..cedae8fe83 100644
--- a/python/2.0.7/.buildinfo
+++ b/python/2.0.7/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 23852ad44c5c8c06634bfdbc3217588a
+config: 99a7589df6038adf00391bf87c7eed6f
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/3.0.0/.buildinfo b/python/3.0.0/.buildinfo
index 2cb14dcb90..51edd71fd9 100644
--- a/python/3.0.0/.buildinfo
+++ b/python/3.0.0/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 3b40b42e3479ff9f79d7f24d7a5e4b86
+config: 9d2582f587f53b3df9b73369d5719603
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/3.0.1/.buildinfo b/python/3.0.1/.buildinfo
index 9557662b3b..b8f8a910d9 100644
--- a/python/3.0.1/.buildinfo
+++ b/python/3.0.1/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 5875be8ee6a630bf071a2c7cf6de3e2e
+config: 4b4e4f7e44bfe70bf022966dacfdac42
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/3.1.0/.buildinfo b/python/3.1.0/.buildinfo
index 834b00869b..88d0caf35d 100644
--- a/python/3.1.0/.buildinfo
+++ b/python/3.1.0/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 39c2ae8fd51b649560b85e18f23d733f
+config: 4a29e286c5ad04b1ed04ea0b73eb905c
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/4.0.0/.buildinfo b/python/4.0.0/.buildinfo
index bd720ada74..14cf6c9ae6 100644
--- a/python/4.0.0/.buildinfo
+++ b/python/4.0.0/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: e28d82e8d526b59b81325d33b491499d
+config: ea53f9b21236c0a99ab9349e2a59189a
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/4.0.1/.buildinfo b/python/4.0.1/.buildinfo
index 4d9ffa4beb..2adb26fd7c 100644
--- a/python/4.0.1/.buildinfo
+++ b/python/4.0.1/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 8a1bf642b1e7682e9d4e516f03b7351f
+config: 278fbbe11c29ab286ba571b7c9db1946
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/4.0.2/.buildinfo b/python/4.0.2/.buildinfo
index 1ee96eecc4..73ca7ac673 100644
--- a/python/4.0.2/.buildinfo
+++ b/python/4.0.2/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: fddbf9bcec7da8b17630d49908392b50
+config: 08a610423d2defb10193db1f9af8ebd4
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/4.0.3/.buildinfo b/python/4.0.3/.buildinfo
index 8a0b24bb4b..ba39b5344c 100644
--- a/python/4.0.3/.buildinfo
+++ b/python/4.0.3/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 7f2819e3b08e58fc50aa98bfef17214a
+config: 87ceca6e430f234e8bcced92fe74e69e
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/4.0.4/.buildinfo b/python/4.0.4/.buildinfo
index 6bc8057058..8f7ca46e68 100644
--- a/python/4.0.4/.buildinfo
+++ b/python/4.0.4/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: f3e321cd2bac219561c736e7c23fbe20
+config: 2741aacc167ca6a694706d54e059e446
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/main/.buildinfo b/python/main/.buildinfo
index 20971fd415..8f78a2a7c4 100644
--- a/python/main/.buildinfo
+++ b/python/main/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it
is not found, a full rebuild will be done.
-config: 83c14bbb21971eda6a993dcda2c2ef2c
+config: bb37374974f439020de21714650b5dd5
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/python/main/_modules/pydolphinscheduler/tasks/datax.html
b/python/main/_modules/pydolphinscheduler/tasks/datax.html
index 48c107275b..9023445cdf 100644
--- a/python/main/_modules/pydolphinscheduler/tasks/datax.html
+++ b/python/main/_modules/pydolphinscheduler/tasks/datax.html
@@ -117,6 +117,38 @@
<span class="w"> </span><span class="sd">"""Task CustomDatax
object, declare behavior for custom DataX task to dolphinscheduler.</span>
<span class="sd"> You provider json template for DataX, it can synchronize
data according to the template you provided.</span>
+
+<span class="sd"> :param name: task name for this task</span>
+<span class="sd"> :param json: json template string, or json file path for
custom DataX task, :class:`CustomDataX` will not</span>
+<span class="sd"> format json template, you should format by
yourself.</span>
+
+<span class="sd"> * Use config string directly instead of json file
path</span>
+<span class="sd"> * should use :func:`json.dumps` to format it if
your json template is dict</span>
+
+<span class="sd"> .. code-block:: python</span>
+
+<span class="sd"> import json</span>
+
+<span class="sd"> custom = CustomDataX(</span>
+<span class="sd"> name="custom_datax",</span>
+<span class="sd"> json=json.dumps({"job":
{"content": [{"reader": {"name":
"mysqlreader"}}]}}),</span>
+<span class="sd"> )</span>
+
+<span class="sd"> * or format it by manual if your json template is
native str.</span>
+<span class="sd"> * Use json file path, the format it shows in web UI
is depended on your json file content.</span>
+
+<span class="sd"> .. code-block:: python</span>
+
+<span class="sd"> import json</span>
+
+<span class="sd"> custom = CustomDataX(</span>
+<span class="sd"> name="custom_datax",</span>
+<span class="sd"> # web UI datax config will show as json
file content</span>
+<span class="sd">
json="/path/to/datax.json",</span>
+<span class="sd"> )</span>
+
+<span class="sd"> :param xms: jvm param about min memory for task datax
running, default is 1g</span>
+<span class="sd"> :param xmx: jvm param about max memory for task datax
running, default is 1g</span>
<span class="sd"> """</span>
<span class="n">CUSTOM_CONFIG</span> <span class="o">=</span> <span
class="mi">1</span>
@@ -159,12 +191,25 @@
<span class="sd"> You provider datasource_name and datatarget_name contain
connection information, it decisions which</span>
<span class="sd"> database type and database instance would synchronous
data.</span>
-<span class="sd"> :param name: task name.</span>
-<span class="sd"> :param datasource_name: source database name for task
datax to extract data.</span>
-<span class="sd"> :param datatarget_name: target database name for task
datax to load data.</span>
+<span class="sd"> :param name: task name for this task</span>
+<span class="sd"> :param datasource_name: source database name for task
datax to extract data, it must exist in</span>
+<span class="sd"> dolphinscheduler's datasource center otherwise
task datax will raise exception.</span>
+<span class="sd"> :param datatarget_name: target database name for task
datax to load data, it must exist in</span>
+<span class="sd"> dolphinscheduler's datasource center otherwise
task datax will raise exception.</span>
<span class="sd"> :param sql: sql statement for task datax to extract data
form source database.</span>
<span class="sd"> :param target_table: target table name for task datax to
load data into target database.</span>
-<span class="sd"> :param datasource_type: source database type,
dolphinscheduler use</span>
+<span class="sd"> :param datasource_type: source database type,
dolphinscheduler use it to find :param:``datasource_name``</span>
+<span class="sd"> in datasource center.</span>
+<span class="sd"> :param datasource_type: target database type,
dolphinscheduler use it to find :param:``datatarget_name``</span>
+<span class="sd"> in datasource center.</span>
+<span class="sd"> :param job_speed_byte: task datax job speed byte, default
is 0. For more detail you can get from</span>
+<span class="sd"> :seealso: https://github.com/alibaba/DataX</span>
+<span class="sd"> :param job_speed_record: task datax job speed record,
default is 1000. For more detail you can get from</span>
+<span class="sd"> :seealso: https://github.com/alibaba/DataX</span>
+<span class="sd"> :param pre_statements: task datax job pre statements, it
will execute before task datax job start to load.</span>
+<span class="sd"> default is None.</span>
+<span class="sd"> :param post_statements: task datax job post statements,
it will execute after task datax job finish load.</span>
+<span class="sd"> default is None.</span>
<span class="sd"> """</span>
<span class="n">CUSTOM_CONFIG</span> <span class="o">=</span> <span
class="mi">0</span>
diff --git a/python/main/api.html b/python/main/api.html
index c95679eb33..aeeae48924 100644
--- a/python/main/api.html
+++ b/python/main/api.html
@@ -2737,6 +2737,47 @@ attribute <cite>_task_custom_attr</cite> due to avoid
attribute cover.</p>
<dd><p>Bases: <code class="xref py py-class docutils literal
notranslate"><span class="pre">WorkerResourceMixin</span></code>, <a
class="reference internal" href="#pydolphinscheduler.core.Task"
title="pydolphinscheduler.core.task.Task"><code class="xref py py-class
docutils literal notranslate"><span class="pre">Task</span></code></a></p>
<p>Task CustomDatax object, declare behavior for custom DataX task to
dolphinscheduler.</p>
<p>You provider json template for DataX, it can synchronize data according to
the template you provided.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>name</strong> – task name for this task</p></li>
+<li><p><strong>json</strong> – <p>json template string, or json file path for
custom DataX task, <a class="reference internal"
href="#pydolphinscheduler.tasks.CustomDataX"
title="pydolphinscheduler.tasks.CustomDataX"><code class="xref py py-class
docutils literal notranslate"><span class="pre">CustomDataX</span></code></a>
will not
+format json template, you should format by yourself.</p>
+<blockquote>
+<div><ul>
+<li><p>Use config string directly instead of json file path
+* should use <code class="xref py py-func docutils literal notranslate"><span
class="pre">json.dumps()</span></code> to format it if your json template is
dict</p>
+<div class="highlight-python notranslate"><div
class="highlight"><pre><span></span><span class="kn">import</span> <span
class="nn">json</span>
+
+<span class="n">custom</span> <span class="o">=</span> <span
class="n">CustomDataX</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"custom_datax"</span><span class="p">,</span>
+ <span class="n">json</span><span class="o">=</span><span
class="n">json</span><span class="o">.</span><span class="n">dumps</span><span
class="p">({</span><span class="s2">"job"</span><span
class="p">:</span> <span class="p">{</span><span
class="s2">"content"</span><span class="p">:</span> <span
class="p">[{</span><span class="s2">"reader"</span><span
class="p">:</span> <span class="p">{</span><span
class="s2">"name"</span><span class="p">:</sp [...]
+<span class="p">)</span>
+</pre></div>
+</div>
+<ul class="simple">
+<li><p>or format it by manual if your json template is native str.</p></li>
+</ul>
+</li>
+<li><p>Use json file path, the format it shows in web UI is depended on your
json file content.</p>
+<div class="highlight-python notranslate"><div
class="highlight"><pre><span></span><span class="kn">import</span> <span
class="nn">json</span>
+
+<span class="n">custom</span> <span class="o">=</span> <span
class="n">CustomDataX</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"custom_datax"</span><span class="p">,</span>
+ <span class="c1"># web UI datax config will show as json file
content</span>
+ <span class="n">json</span><span class="o">=</span><span
class="s2">"/path/to/datax.json"</span><span class="p">,</span>
+<span class="p">)</span>
+</pre></div>
+</div>
+</li>
+</ul>
+</div></blockquote>
+</p></li>
+<li><p><strong>xms</strong> – jvm param about min memory for task datax
running, default is 1g</p></li>
+<li><p><strong>xmx</strong> – jvm param about max memory for task datax
running, default is 1g</p></li>
+</ul>
+</dd>
+</dl>
<dl class="py method">
<dt class="sig sig-object py"
id="pydolphinscheduler.tasks.CustomDataX._get_attr">
<span class="sig-name descname"><span class="pre">_get_attr</span></span><span
class="sig-paren">(</span><span class="sig-paren">)</span> <span
class="sig-return"><span class="sig-return-icon">→</span> <span
class="sig-return-typehint"><span class="pre">set</span><span class="p"><span
class="pre">[</span></span><span class="pre">str</span><span class="p"><span
class="pre">]</span></span></span></span><a class="headerlink"
href="#pydolphinscheduler.tasks.CustomDataX._get_attr" titl [...]
@@ -3751,12 +3792,25 @@ database type and database instance would synchronous
data.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>name</strong> – task name.</p></li>
-<li><p><strong>datasource_name</strong> – source database name for task datax
to extract data.</p></li>
-<li><p><strong>datatarget_name</strong> – target database name for task datax
to load data.</p></li>
+<li><p><strong>name</strong> – task name for this task</p></li>
+<li><p><strong>datasource_name</strong> – source database name for task datax
to extract data, it must exist in
+dolphinscheduler’s datasource center otherwise task datax will raise
exception.</p></li>
+<li><p><strong>datatarget_name</strong> – target database name for task datax
to load data, it must exist in
+dolphinscheduler’s datasource center otherwise task datax will raise
exception.</p></li>
<li><p><strong>sql</strong> – sql statement for task datax to extract data
form source database.</p></li>
<li><p><strong>target_table</strong> – target table name for task datax to
load data into target database.</p></li>
-<li><p><strong>datasource_type</strong> – source database type,
dolphinscheduler use</p></li>
+<li><p><strong>datasource_type</strong> – source database type,
dolphinscheduler use it to find :param:<code class="docutils literal
notranslate"><span class="pre">datasource_name</span></code>
+in datasource center.</p></li>
+<li><p><strong>datasource_type</strong> – target database type,
dolphinscheduler use it to find :param:<code class="docutils literal
notranslate"><span class="pre">datatarget_name</span></code>
+in datasource center.</p></li>
+<li><p><strong>job_speed_byte</strong> – task datax job speed byte, default is
0. For more detail you can get from
+:seealso: <a class="reference external"
href="https://github.com/alibaba/DataX";>https://github.com/alibaba/DataX</a></p></li>
+<li><p><strong>job_speed_record</strong> – task datax job speed record,
default is 1000. For more detail you can get from
+:seealso: <a class="reference external"
href="https://github.com/alibaba/DataX";>https://github.com/alibaba/DataX</a></p></li>
+<li><p><strong>pre_statements</strong> – task datax job pre statements, it
will execute before task datax job start to load.
+default is None.</p></li>
+<li><p><strong>post_statements</strong> – task datax job post statements, it
will execute after task datax job finish load.
+default is None.</p></li>
</ul>
</dd>
</dl>
diff --git a/python/main/searchindex.js b/python/main/searchindex.js
index 611251e0e6..e42579235d 100644
--- a/python/main/searchindex.js
+++ b/python/main/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"docnames": ["api", "changelog", "cli", "concept", "config",
"howto/index", "howto/multi-resources", "howto/remote-submit", "index",
"resources_plugin/develop", "resources_plugin/github",
"resources_plugin/gitlab", "resources_plugin/index", "resources_plugin/local",
"resources_plugin/oss", "resources_plugin/resource-plugin",
"resources_plugin/s3", "start", "tasks/condition", "tasks/datax",
"tasks/dependent", "tasks/dvc", "tasks/flink", "tasks/func_wrap", "tasks/http",
"t [...]
\ No newline at end of file
+Search.setIndex({"docnames": ["api", "changelog", "cli", "concept", "config",
"howto/index", "howto/multi-resources", "howto/remote-submit", "index",
"resources_plugin/develop", "resources_plugin/github",
"resources_plugin/gitlab", "resources_plugin/index", "resources_plugin/local",
"resources_plugin/oss", "resources_plugin/resource-plugin",
"resources_plugin/s3", "start", "tasks/condition", "tasks/datax",
"tasks/dependent", "tasks/dvc", "tasks/flink", "tasks/func_wrap", "tasks/http",
"t [...]
\ No newline at end of file
diff --git a/python/main/tasks/datax.html b/python/main/tasks/datax.html
index 8ba3a2e9b7..b3855a8698 100644
--- a/python/main/tasks/datax.html
+++ b/python/main/tasks/datax.html
@@ -133,6 +133,8 @@
<span class="sd">It creates a task to synchronize datax from the source
database to the target database.</span>
<span class="sd">"""</span>
+<span class="kn">import</span> <span class="nn">json</span>
+
<span class="kn">from</span> <span
class="nn">pydolphinscheduler.core.workflow</span> <span
class="kn">import</span> <span class="n">Workflow</span>
<span class="kn">from</span> <span
class="nn">pydolphinscheduler.tasks.datax</span> <span class="kn">import</span>
<span class="n">CustomDataX</span><span class="p">,</span> <span
class="n">DataX</span>
@@ -197,7 +199,10 @@
<span class="c1"># You can custom json_template of datax to sync data.
This task create a new</span>
<span class="c1"># datax job same as task1, transfer record from
`first_mysql` to `second_mysql`</span>
- <span class="n">task2</span> <span class="o">=</span> <span
class="n">CustomDataX</span><span class="p">(</span><span
class="n">name</span><span class="o">=</span><span
class="s2">"task_custom_datax"</span><span class="p">,</span> <span
class="n">json</span><span class="o">=</span><span class="nb">str</span><span
class="p">(</span><span class="n">JSON_TEMPLATE</span><span class="p">))</span>
+ <span class="c1"># We should format the custom json config if we want to
format it in web UI</span>
+ <span class="n">task2</span> <span class="o">=</span> <span
class="n">CustomDataX</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"task_custom_datax"</span><span class="p">,</span> <span
class="n">json</span><span class="o">=</span><span class="n">json</span><span
class="o">.</span><span class="n">dumps</span><span class="p">(</span><span
class="n">JSON_TEMPLATE</span><span class="p">,</span> <span
class="n">indent</span><span class="o">=</span><span class="mi">4</span><span
class="p">)</span>
+ <span class="p">)</span>
<span class="c1"># [start resource_limit]</span>
<span class="n">resource_limit</span> <span class="o">=</span> <span
class="n">DataX</span><span class="p">(</span>
@@ -238,6 +243,47 @@
<dd><p>Bases: <code class="xref py py-class docutils literal
notranslate"><span class="pre">WorkerResourceMixin</span></code>, <a
class="reference internal" href="../api.html#pydolphinscheduler.core.Task"
title="pydolphinscheduler.core.task.Task"><code class="xref py py-class
docutils literal notranslate"><span class="pre">Task</span></code></a></p>
<p>Task CustomDatax object, declare behavior for custom DataX task to
dolphinscheduler.</p>
<p>You provider json template for DataX, it can synchronize data according to
the template you provided.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>name</strong> – task name for this task</p></li>
+<li><p><strong>json</strong> – <p>json template string, or json file path for
custom DataX task, <a class="reference internal"
href="#pydolphinscheduler.tasks.datax.CustomDataX"
title="pydolphinscheduler.tasks.datax.CustomDataX"><code class="xref py
py-class docutils literal notranslate"><span
class="pre">CustomDataX</span></code></a> will not
+format json template, you should format by yourself.</p>
+<blockquote>
+<div><ul>
+<li><p>Use config string directly instead of json file path
+* should use <code class="xref py py-func docutils literal notranslate"><span
class="pre">json.dumps()</span></code> to format it if your json template is
dict</p>
+<div class="highlight-python notranslate"><div
class="highlight"><pre><span></span><span class="kn">import</span> <span
class="nn">json</span>
+
+<span class="n">custom</span> <span class="o">=</span> <span
class="n">CustomDataX</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"custom_datax"</span><span class="p">,</span>
+ <span class="n">json</span><span class="o">=</span><span
class="n">json</span><span class="o">.</span><span class="n">dumps</span><span
class="p">({</span><span class="s2">"job"</span><span
class="p">:</span> <span class="p">{</span><span
class="s2">"content"</span><span class="p">:</span> <span
class="p">[{</span><span class="s2">"reader"</span><span
class="p">:</span> <span class="p">{</span><span
class="s2">"name"</span><span class="p">:</sp [...]
+<span class="p">)</span>
+</pre></div>
+</div>
+<ul class="simple">
+<li><p>or format it by manual if your json template is native str.</p></li>
+</ul>
+</li>
+<li><p>Use json file path, the format it shows in web UI is depended on your
json file content.</p>
+<div class="highlight-python notranslate"><div
class="highlight"><pre><span></span><span class="kn">import</span> <span
class="nn">json</span>
+
+<span class="n">custom</span> <span class="o">=</span> <span
class="n">CustomDataX</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"custom_datax"</span><span class="p">,</span>
+ <span class="c1"># web UI datax config will show as json file
content</span>
+ <span class="n">json</span><span class="o">=</span><span
class="s2">"/path/to/datax.json"</span><span class="p">,</span>
+<span class="p">)</span>
+</pre></div>
+</div>
+</li>
+</ul>
+</div></blockquote>
+</p></li>
+<li><p><strong>xms</strong> – jvm param about min memory for task datax
running, default is 1g</p></li>
+<li><p><strong>xmx</strong> – jvm param about max memory for task datax
running, default is 1g</p></li>
+</ul>
+</dd>
+</dl>
<dl class="py attribute">
<dt class="sig sig-object py"
id="pydolphinscheduler.tasks.datax.CustomDataX.CUSTOM_CONFIG">
<span class="sig-name descname"><span
class="pre">CUSTOM_CONFIG</span></span><em class="property"><span class="w">
</span><span class="p"><span class="pre">=</span></span><span class="w">
</span><span class="pre">1</span></em><a class="headerlink"
href="#pydolphinscheduler.tasks.datax.CustomDataX.CUSTOM_CONFIG" title="Link to
this definition"></a></dt>
@@ -297,12 +343,25 @@ database type and database instance would synchronous
data.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>name</strong> – task name.</p></li>
-<li><p><strong>datasource_name</strong> – source database name for task datax
to extract data.</p></li>
-<li><p><strong>datatarget_name</strong> – target database name for task datax
to load data.</p></li>
+<li><p><strong>name</strong> – task name for this task</p></li>
+<li><p><strong>datasource_name</strong> – source database name for task datax
to extract data, it must exist in
+dolphinscheduler’s datasource center otherwise task datax will raise
exception.</p></li>
+<li><p><strong>datatarget_name</strong> – target database name for task datax
to load data, it must exist in
+dolphinscheduler’s datasource center otherwise task datax will raise
exception.</p></li>
<li><p><strong>sql</strong> – sql statement for task datax to extract data
form source database.</p></li>
<li><p><strong>target_table</strong> – target table name for task datax to
load data into target database.</p></li>
-<li><p><strong>datasource_type</strong> – source database type,
dolphinscheduler use</p></li>
+<li><p><strong>datasource_type</strong> – source database type,
dolphinscheduler use it to find :param:<code class="docutils literal
notranslate"><span class="pre">datasource_name</span></code>
+in datasource center.</p></li>
+<li><p><strong>datasource_type</strong> – target database type,
dolphinscheduler use it to find :param:<code class="docutils literal
notranslate"><span class="pre">datatarget_name</span></code>
+in datasource center.</p></li>
+<li><p><strong>job_speed_byte</strong> – task datax job speed byte, default is
0. For more detail you can get from
+:seealso: <a class="reference external"
href="https://github.com/alibaba/DataX";>https://github.com/alibaba/DataX</a></p></li>
+<li><p><strong>job_speed_record</strong> – task datax job speed record,
default is 1000. For more detail you can get from
+:seealso: <a class="reference external"
href="https://github.com/alibaba/DataX";>https://github.com/alibaba/DataX</a></p></li>
+<li><p><strong>pre_statements</strong> – task datax job pre statements, it
will execute before task datax job start to load.
+default is None.</p></li>
+<li><p><strong>post_statements</strong> – task datax job post statements, it
will execute after task datax job finish load.
+default is None.</p></li>
</ul>
</dd>
</dl>
diff --git a/python/main/tasks/shell.html b/python/main/tasks/shell.html
index 0379394cba..8bff2d9ad6 100644
--- a/python/main/tasks/shell.html
+++ b/python/main/tasks/shell.html
@@ -131,7 +131,15 @@
<span class="c1"># [end workflow_declare]</span>
<span class="c1"># [start task_declare]</span>
<span class="n">task_parent</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span><span class="n">name</span><span
class="o">=</span><span class="s2">"task_parent"</span><span
class="p">,</span> <span class="n">command</span><span class="o">=</span><span
class="s2">"echo hello pydolphinscheduler"</span><span
class="p">)</span>
- <span class="n">task_child_one</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span><span class="n">name</span><span
class="o">=</span><span class="s2">"task_child_one"</span><span
class="p">,</span> <span class="n">command</span><span class="o">=</span><span
class="s2">"echo 'child one'"</span><span class="p">)</span>
+ <span class="n">task_child_one</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"task_child_one"</span><span class="p">,</span>
+ <span class="n">command</span><span class="o">=</span><span
class="s2">"""</span>
+<span class="s2"> echo "Executing line 1 with parameter str type
$</span><span class="si">{param1}</span><span class="s2">"</span>
+<span class="s2"> echo "Executing line 2 with parameter int type
$</span><span class="si">{param2}</span><span class="s2">"</span>
+<span class="s2"> echo "Executing line 3 with parameter build-in
parameter currently date $</span><span class="si">{param3}</span><span
class="s2">"</span>
+<span class="s2"> """</span><span class="p">,</span>
+ <span class="n">params</span><span class="o">=</span><span
class="p">{</span><span class="s2">"param1"</span><span
class="p">:</span> <span class="s2">"str1"</span><span
class="p">,</span> <span class="s2">"param2"</span><span
class="p">:</span> <span class="mi">123</span><span class="p">,</span> <span
class="s2">"param3"</span><span class="p">:</span> <span
class="s2">"$[yyyy-MM-dd]"</span><span class="p">},</span>
+ <span class="p">)</span>
<span class="n">task_child_two</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span><span class="n">name</span><span
class="o">=</span><span class="s2">"task_child_two"</span><span
class="p">,</span> <span class="n">command</span><span class="o">=</span><span
class="s2">"echo 'child two'"</span><span class="p">)</span>
<span class="n">task_union</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span><span class="n">name</span><span
class="o">=</span><span class="s2">"task_union"</span><span
class="p">,</span> <span class="n">command</span><span class="o">=</span><span
class="s2">"echo union"</span><span class="p">)</span>
diff --git a/python/main/tutorial.html b/python/main/tutorial.html
index 4ed596b43f..fce05b3066 100644
--- a/python/main/tutorial.html
+++ b/python/main/tutorial.html
@@ -137,7 +137,15 @@ We can find more YAML file examples in <a class="reference
external" href="https
<span class="c1"># [end workflow_declare]</span>
<span class="c1"># [start task_declare]</span>
<span class="n">task_parent</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span><span class="n">name</span><span
class="o">=</span><span class="s2">"task_parent"</span><span
class="p">,</span> <span class="n">command</span><span class="o">=</span><span
class="s2">"echo hello pydolphinscheduler"</span><span
class="p">)</span>
- <span class="n">task_child_one</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span><span class="n">name</span><span
class="o">=</span><span class="s2">"task_child_one"</span><span
class="p">,</span> <span class="n">command</span><span class="o">=</span><span
class="s2">"echo 'child one'"</span><span class="p">)</span>
+ <span class="n">task_child_one</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"task_child_one"</span><span class="p">,</span>
+ <span class="n">command</span><span class="o">=</span><span
class="s2">"""</span>
+<span class="s2"> echo "Executing line 1 with parameter str type
$</span><span class="si">{param1}</span><span class="s2">"</span>
+<span class="s2"> echo "Executing line 2 with parameter int type
$</span><span class="si">{param2}</span><span class="s2">"</span>
+<span class="s2"> echo "Executing line 3 with parameter build-in
parameter currently date $</span><span class="si">{param3}</span><span
class="s2">"</span>
+<span class="s2"> """</span><span class="p">,</span>
+ <span class="n">params</span><span class="o">=</span><span
class="p">{</span><span class="s2">"param1"</span><span
class="p">:</span> <span class="s2">"str1"</span><span
class="p">,</span> <span class="s2">"param2"</span><span
class="p">:</span> <span class="mi">123</span><span class="p">,</span> <span
class="s2">"param3"</span><span class="p">:</span> <span
class="s2">"$[yyyy-MM-dd]"</span><span class="p">},</span>
+ <span class="p">)</span>
<span class="n">task_child_two</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span><span class="n">name</span><span
class="o">=</span><span class="s2">"task_child_two"</span><span
class="p">,</span> <span class="n">command</span><span class="o">=</span><span
class="s2">"echo 'child two'"</span><span class="p">)</span>
<span class="n">task_union</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span><span class="n">name</span><span
class="o">=</span><span class="s2">"task_union"</span><span
class="p">,</span> <span class="n">command</span><span class="o">=</span><span
class="s2">"echo union"</span><span class="p">)</span>
@@ -356,7 +364,15 @@ if you are interested in it. For all arguments of object
workflow, you could fin
<cite>command</cite> with <code class="code docutils literal
notranslate"><span class="pre">echo</span></code> command, we also need to set
the argument <cite>name</cite> for each task
<em>(not only shell task, `name` is required for each type of task)</em>.</p>
<div class="highlight-default notranslate"><div
class="highlight"><pre><span></span> <span class="n">task_parent</span>
<span class="o">=</span> <span class="n">Shell</span><span
class="p">(</span><span class="n">name</span><span class="o">=</span><span
class="s2">"task_parent"</span><span class="p">,</span> <span
class="n">command</span><span class="o">=</span><span class="s2">"echo
hello pydolphinscheduler"</span><span class="p">)</span>
- <span class="n">task_child_one</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span><span class="n">name</span><span
class="o">=</span><span class="s2">"task_child_one"</span><span
class="p">,</span> <span class="n">command</span><span class="o">=</span><span
class="s2">"echo 'child one'"</span><span class="p">)</span>
+ <span class="n">task_child_one</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span>
+ <span class="n">name</span><span class="o">=</span><span
class="s2">"task_child_one"</span><span class="p">,</span>
+ <span class="n">command</span><span class="o">=</span><span
class="s2">"""</span>
+<span class="s2"> echo "Executing line 1 with parameter str type
$</span><span class="si">{param1}</span><span class="s2">"</span>
+<span class="s2"> echo "Executing line 2 with parameter int type
$</span><span class="si">{param2}</span><span class="s2">"</span>
+<span class="s2"> echo "Executing line 3 with parameter build-in
parameter currently date $</span><span class="si">{param3}</span><span
class="s2">"</span>
+<span class="s2"> """</span><span class="p">,</span>
+ <span class="n">params</span><span class="o">=</span><span
class="p">{</span><span class="s2">"param1"</span><span
class="p">:</span> <span class="s2">"str1"</span><span
class="p">,</span> <span class="s2">"param2"</span><span
class="p">:</span> <span class="mi">123</span><span class="p">,</span> <span
class="s2">"param3"</span><span class="p">:</span> <span
class="s2">"$[yyyy-MM-dd]"</span><span class="p">},</span>
+ <span class="p">)</span>
<span class="n">task_child_two</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span><span class="n">name</span><span
class="o">=</span><span class="s2">"task_child_two"</span><span
class="p">,</span> <span class="n">command</span><span class="o">=</span><span
class="s2">"echo 'child two'"</span><span class="p">)</span>
<span class="n">task_union</span> <span class="o">=</span> <span
class="n">Shell</span><span class="p">(</span><span class="n">name</span><span
class="o">=</span><span class="s2">"task_union"</span><span
class="p">,</span> <span class="n">command</span><span class="o">=</span><span
class="s2">"echo union"</span><span class="p">)</span>
@@ -545,8 +561,7 @@ the workflow schedule information. For more detail, you
could see <a class="refe
<p>After we run the tutorial code, you could log in DolphinScheduler web UI,
go and see the
<a class="reference external"
href="https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/project.html";>DolphinScheduler
project page</a>. They is a new workflow be created by
<em>PyDolphinScheduler</em> and it
named “tutorial” or “tutorial_decorator”. The task graph of workflow like
below:</p>
-<div class="highlight-text notranslate"><div
class="highlight"><pre><span></span> --> task_child_one
- / \
+<div class="highlight-text notranslate"><div
class="highlight"><pre><span></span> / \
task_parent --> --> task_union
\ /
--> task_child_two
