This is an automated email from the ASF dual-hosted git repository.
wanghailin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 05717ef20e [Feature][Core] Add a test case for dynamic custom
parameters (#7128)
05717ef20e is described below
commit 05717ef20e19c9ab32cc9ca40e2917bc15525b04
Author: Guangdong Liu <[email protected]>
AuthorDate: Tue Jul 9 23:26:43 2024 +0800
[Feature][Core] Add a test case for dynamic custom parameters (#7128)
---
docs/en/concept/config.md | 1 +
docs/zh/concept/config.md | 112 +++++++++++++++++++++
.../core/starter/command/AbstractCommandArgs.java | 4 +-
.../seatunnel/engine/e2e/UserVariableIT.java | 4 +-
.../test/resources/fake_to_console.variables.conf | 2 +-
5 files changed, 119 insertions(+), 4 deletions(-)
diff --git a/docs/en/concept/config.md b/docs/en/concept/config.md
index 6f17ea03b3..a8c58bae2d 100644
--- a/docs/en/concept/config.md
+++ b/docs/en/concept/config.md
@@ -315,6 +315,7 @@ Some Notes:
- quota with `'` if the value has special character (like `(`)
- if the replacement variables is in `"` or `'`, like `resName` and `nameVal`,
you need add `"`
- the value can't have space `' '`, like `-i jobName='this is a job name' `,
this will be replaced to `job.name = "this"`
+- If you want to use dynamic parameters,you can use the following format: -i
date=$(date +"%Y%m%d").
## What's More
diff --git a/docs/zh/concept/config.md b/docs/zh/concept/config.md
index baa9a7a715..8f4368a67f 100644
--- a/docs/zh/concept/config.md
+++ b/docs/zh/concept/config.md
@@ -203,6 +203,118 @@ sink模块，你可以快速高效地完成这个操作。Sink和source非常相
`result_table_name` 和 `source_table_name`
配置。但你会发现在上面的配置例子中，不是每个模块都配置了这些参数，因为在SeaTunnel中，
有一个默认的约定，如果这两个参数没有配置，则使用上一个节点的最后一个模块生成的数据。当只有一个source时这是非常方便的。
+## 配置变量替换
+
+在配置文件中,我们可以定义一些变量并在运行时替换它们。这仅支持 hocon 格式的文件。
+
+```hocon
+env {
+ job.mode = "BATCH"
+ job.name = ${jobName}
+ parallelism = 2
+}
+
+source {
+ FakeSource {
+ result_table_name = ${resName}
+ row.num = ${rowNum}
+ string.template = ${strTemplate}
+ int.template = [20, 21]
+ schema = {
+ fields {
+ name = ${nameType}
+ age = "int"
+ }
+ }
+ }
+}
+
+transform {
+ sql {
+ source_table_name = "fake"
+ result_table_name = "sql"
+ query = "select * from "${resName}" where name = '"${nameVal}"' "
+ }
+
+}
+
+sink {
+ Console {
+ source_table_name = "sql"
+ username = ${username}
+ password = ${password}
+ }
+}
+
+```
+
+在上述配置中,我们定义了一些变量,如 ${rowNum}、${resName}。
+我们可以使用以下 shell 命令替换这些参数:
+
+```shell
+./bin/seatunnel.sh -c <this_config_file>
+-i jobName='this_is_a_job_name'
+-i resName=fake
+-i rowNum=10
+-i strTemplate=['abc','d~f','hi']
+-i nameType=string
+-i nameVal=abc
+-i username=seatunnel=2.3.1
+-i password='$a^b%c.d~e0*9('
+-e local
+```
+
+然后最终提交的配置是:
+
+```hocon
+env {
+ job.mode = "BATCH"
+ job.name = "this_is_a_job_name"
+ parallelism = 2
+}
+
+source {
+ FakeSource {
+ result_table_name = "fake"
+ row.num = 10
+ string.template = ['abc','d~f','hi']
+ int.template = [20, 21]
+ schema = {
+ fields {
+ name = "string"
+ age = "int"
+ }
+ }
+ }
+}
+
+transform {
+ sql {
+ source_table_name = "fake"
+ result_table_name = "sql"
+ query = "select * from "fake" where name = 'abc' "
+ }
+
+}
+
+sink {
+ Console {
+ source_table_name = "sql"
+ username = "seatunnel=2.3.1"
+ password = "$a^b%c.d~e0*9("
+ }
+}
+
+```
+
+一些注意事项:
+
+- 如果值包含特殊字符(如`(`)，请使用`'`引号将其括起来。
+- 如果替换变量包含`"`或`'`(如`"resName"`和`"nameVal"`)，需要添加`"`。
+- 值不能包含空格`' '`。例如, `-i jobName='this is a job name'`将被替换为`job.name = "this"`。
+- 如果要使用动态参数,可以使用以下格式: `-i date=$(date +"%Y%m%d")`。
+
## 此外
如果你想了解更多关于格式配置的详细信息，请查看
[HOCON](https://github.com/lightbend/config/blob/main/HOCON.md)。
+
diff --git
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/AbstractCommandArgs.java
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/AbstractCommandArgs.java
index 13d969c326..5d620c96ee 100644
---
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/AbstractCommandArgs.java
+++
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/AbstractCommandArgs.java
@@ -44,7 +44,9 @@ public abstract class AbstractCommandArgs extends CommandArgs
{
splitter = ParameterSplitter.class,
description =
"Variable substitution, such as -i city=beijing, or -i
date=20190318."
- + "We use ',' as separator, when inside \"\", ','
are treated as normal characters instead of delimiters.")
+ + "We use ',' as separator, when inside \"\", ','
are treated as normal characters instead of delimiters."
+ + " For example, -i city=\"beijing,shanghai\". If
you want to use dynamic parameters,"
+ + " you can use the following format: -i
date=$(date +\"%Y%m%d\").")
protected List<String> variables = Collections.emptyList();
/** check config flag */
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/UserVariableIT.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/UserVariableIT.java
index 03455af2b0..87e05821b1 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/UserVariableIT.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/UserVariableIT.java
@@ -34,7 +34,7 @@ public class UserVariableIT extends TestSuiteBase {
public void userVariableTest(TestContainer container) throws IOException,
InterruptedException {
List<String> variables = new ArrayList<>();
String list = "[abc,def]";
- variables.add("resName=fake");
+ variables.add("resName=a$(date +\"%Y%m%d\")");
variables.add("rowNum=10");
variables.add("strTemplate=" + list);
variables.add("nameType=string");
@@ -42,6 +42,6 @@ public class UserVariableIT extends TestSuiteBase {
variables.add("sourceTableName=sql");
Container.ExecResult execResult =
container.executeJob("/fake_to_console.variables.conf",
variables);
- Assertions.assertEquals(0, execResult.getExitCode());
+ Assertions.assertEquals(0, execResult.getExitCode(),
execResult.getStderr());
}
}
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/fake_to_console.variables.conf
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/fake_to_console.variables.conf
index 48f7ec548b..41f5bbc77b 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/fake_to_console.variables.conf
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/fake_to_console.variables.conf
@@ -46,7 +46,7 @@ transform {
# If you would like to get more information about how to configure seatunnel
and see full list of transform plugins,
# please go to https://seatunnel.apache.org/docs/category/transform-v2
sql {
- source_table_name = "fake"
+ source_table_name = ${resName}
query = "select * from "${resName}" where name = '"${nameVal}"' "
result_table_name = "sql"
}
