This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 1af7d38 [Bug] [config] Fix "Can only specify option -i once" in
command line args parser (#1236)
1af7d38 is described below
commit 1af7d38a75f4cd9e331701cf6144008008c0fc3f
Author: CenterCode <[email protected]>
AuthorDate: Wed Mar 2 22:30:46 2022 +0800
[Bug] [config] Fix "Can only specify option -i once" in command line args
parser (#1236)
* [SeaTunnel #1092] [core] fix multiple variables substitution
* Update
seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/config/command/CommandSparkArgsTest.java
Co-authored-by: Wenjun Ruan <[email protected]>
* update CommandSparkArgsTest
Co-authored-by: Wenjun Ruan <[email protected]>
---
.../seatunnel/config/command/CommandFlinkArgs.java | 8 ++--
.../seatunnel/config/command/CommandLineArgs.java | 12 +++---
.../seatunnel/config/command/CommandLineUtils.java | 2 +-
.../seatunnel/config/command/CommandSparkArgs.java | 11 +++++-
.../config/command/CommandFlinkArgsTest.java} | 46 +++++++++-------------
.../config/command/CommandSparkArgsTest.java} | 40 ++++++-------------
.../apache/seatunnel/core/sql/SeatunnelSql.java | 2 +-
.../org/apache/seatunnel/core/sql/job/JobInfo.java | 9 ++++-
.../core/sql/SqlVariableSubstitutionTest.java | 2 +-
9 files changed, 64 insertions(+), 68 deletions(-)
diff --git
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandFlinkArgs.java
b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandFlinkArgs.java
index 58f17b8..95cf813 100644
---
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandFlinkArgs.java
+++
b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandFlinkArgs.java
@@ -19,6 +19,8 @@ package org.apache.seatunnel.config.command;
import com.beust.jcommander.Parameter;
+import java.util.List;
+
public class CommandFlinkArgs {
@Parameter(names = {"-c", "--config"},
@@ -28,7 +30,7 @@ public class CommandFlinkArgs {
@Parameter(names = {"-i", "--variable"},
description = "variable substitution, such as -i city=beijing, or -i
date=20190318")
- private String variable = null;
+ private List<String> variables = null;
@Parameter(names = {"-t", "--check"},
description = "check config")
@@ -42,7 +44,7 @@ public class CommandFlinkArgs {
return testConfig;
}
- public String getVariable() {
- return variable;
+ public List<String> getVariables() {
+ return variables;
}
}
diff --git
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandLineArgs.java
b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandLineArgs.java
index 009dc0f..5e9513b 100644
---
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandLineArgs.java
+++
b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandLineArgs.java
@@ -17,17 +17,19 @@
package org.apache.seatunnel.config.command;
+import java.util.List;
+
public class CommandLineArgs {
private String deployMode = "client";
private final String configFile;
private final boolean testConfig;
- private String variable;
+ private List<String> variables;
- public CommandLineArgs(String configFile, boolean testConfig, String
variable) {
+ public CommandLineArgs(String configFile, boolean testConfig, List<String>
variables) {
this.configFile = configFile;
this.testConfig = testConfig;
- this.variable = variable;
+ this.variables = variables;
}
public CommandLineArgs(String deployMode, String configFile, boolean
testConfig) {
@@ -48,8 +50,8 @@ public class CommandLineArgs {
return testConfig;
}
- public String getVariable() {
- return variable;
+ public List<String> getVariables() {
+ return variables;
}
}
diff --git
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandLineUtils.java
b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandLineUtils.java
index 41d7751..412f5d4 100644
---
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandLineUtils.java
+++
b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandLineUtils.java
@@ -48,7 +48,7 @@ public final class CommandLineUtils {
return new CommandLineArgs(
commandFlinkArgs.getConfigFile(),
commandFlinkArgs.isTestConfig(),
- commandFlinkArgs.getVariable()
+ commandFlinkArgs.getVariables()
);
}
diff --git
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandSparkArgs.java
b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandSparkArgs.java
index 1b1f689..35d7c16 100644
---
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandSparkArgs.java
+++
b/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandSparkArgs.java
@@ -19,6 +19,8 @@ package org.apache.seatunnel.config.command;
import com.beust.jcommander.Parameter;
+import java.util.List;
+
public class CommandSparkArgs {
@Parameter(names = {"-c", "--config"},
@@ -39,7 +41,7 @@ public class CommandSparkArgs {
@Parameter(names = {"-i", "--variable"},
description = "variable substitution, such as -i city=beijing, or -i
date=20190318")
- private String variable = null;
+ private List<String> variables = null;
@Parameter(names = {"-t", "--check"},
description = "check config")
@@ -57,4 +59,11 @@ public class CommandSparkArgs {
return testConfig;
}
+ public String getMaster() {
+ return master;
+ }
+
+ public List<String> getVariables() {
+ return variables;
+ }
}
diff --git
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandFlinkArgs.java
b/seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/config/command/CommandFlinkArgsTest.java
similarity index 54%
copy from
seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandFlinkArgs.java
copy to
seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/config/command/CommandFlinkArgsTest.java
index 58f17b8..e446bb4 100644
---
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandFlinkArgs.java
+++
b/seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/config/command/CommandFlinkArgsTest.java
@@ -17,32 +17,24 @@
package org.apache.seatunnel.config.command;
-import com.beust.jcommander.Parameter;
-
-public class CommandFlinkArgs {
-
- @Parameter(names = {"-c", "--config"},
- description = "config file",
- required = true)
- private String configFile = "application.conf";
-
- @Parameter(names = {"-i", "--variable"},
- description = "variable substitution, such as -i city=beijing, or -i
date=20190318")
- private String variable = null;
-
- @Parameter(names = {"-t", "--check"},
- description = "check config")
- private boolean testConfig = false;
-
- public String getConfigFile() {
- return configFile;
- }
-
- public boolean isTestConfig() {
- return testConfig;
- }
-
- public String getVariable() {
- return variable;
+import com.beust.jcommander.JCommander;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+public class CommandFlinkArgsTest {
+
+ @Test
+ public void testParseFlinkArgs() {
+ String[] args = {"-c", "app.conf", "-t", "-i", "city=shenyang", "-i",
"date=20200202"};
+ CommandFlinkArgs flinkArgs = new CommandFlinkArgs();
+ JCommander.newBuilder()
+ .addObject(flinkArgs)
+ .build()
+ .parse(args);
+ Assert.assertEquals("app.conf", flinkArgs.getConfigFile());
+ Assert.assertTrue(flinkArgs.isTestConfig());
+ Assert.assertEquals(Arrays.asList("city=shenyang", "date=20200202"),
flinkArgs.getVariables());
}
}
diff --git
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandLineUtils.java
b/seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/config/command/CommandSparkArgsTest.java
similarity index 52%
copy from
seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandLineUtils.java
copy to
seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/config/command/CommandSparkArgsTest.java
index 41d7751..0548f2e 100644
---
a/seatunnel-core/seatunnel-core-base/src/main/java/org/apache/seatunnel/config/command/CommandLineUtils.java
+++
b/seatunnel-core/seatunnel-core-base/src/test/java/org/apache/seatunnel/config/command/CommandSparkArgsTest.java
@@ -18,38 +18,24 @@
package org.apache.seatunnel.config.command;
import com.beust.jcommander.JCommander;
+import org.junit.Assert;
+import org.junit.Test;
-public final class CommandLineUtils {
+import java.util.Arrays;
- private CommandLineUtils() {
- }
+public class CommandSparkArgsTest {
- public static CommandLineArgs parseSparkArgs(String[] args) {
- CommandSparkArgs commandSparkArgs = new CommandSparkArgs();
+ @Test
+ public void testParseSparkArgs() {
+ String[] args = {"-c", "app.conf", "-e", "client", "-m", "yarn", "-i",
"city=shijiazhuang", "-i", "name=Tom"};
+ CommandSparkArgs sparkArgs = new CommandSparkArgs();
JCommander.newBuilder()
- .addObject(commandSparkArgs)
+ .addObject(sparkArgs)
.build()
.parse(args);
-
- return new CommandLineArgs(
- commandSparkArgs.getDeployMode(),
- commandSparkArgs.getConfigFile(),
- commandSparkArgs.isTestConfig()
- );
+ Assert.assertEquals("app.conf", sparkArgs.getConfigFile());
+ Assert.assertEquals("client", sparkArgs.getDeployMode());
+ Assert.assertEquals("yarn", sparkArgs.getMaster());
+ Assert.assertEquals(Arrays.asList("city=shijiazhuang", "name=Tom"),
sparkArgs.getVariables());
}
-
- public static CommandLineArgs parseFlinkArgs(String[] args) {
- CommandFlinkArgs commandFlinkArgs = new CommandFlinkArgs();
- JCommander.newBuilder()
- .addObject(commandFlinkArgs)
- .build()
- .parse(args);
-
- return new CommandLineArgs(
- commandFlinkArgs.getConfigFile(),
- commandFlinkArgs.isTestConfig(),
- commandFlinkArgs.getVariable()
- );
- }
-
}
diff --git
a/seatunnel-core/seatunnel-core-sql/src/main/java/org/apache/seatunnel/core/sql/SeatunnelSql.java
b/seatunnel-core/seatunnel-core-sql/src/main/java/org/apache/seatunnel/core/sql/SeatunnelSql.java
index 7b09e5a..9e67f1a 100644
---
a/seatunnel-core/seatunnel-core-sql/src/main/java/org/apache/seatunnel/core/sql/SeatunnelSql.java
+++
b/seatunnel-core/seatunnel-core-sql/src/main/java/org/apache/seatunnel/core/sql/SeatunnelSql.java
@@ -40,7 +40,7 @@ public class SeatunnelSql {
String configFilePath = flinkArgs.getConfigFile();
String jobContent = FileUtils.readFileToString(new
File(configFilePath), StandardCharsets.UTF_8);
JobInfo jobInfo = new JobInfo(jobContent);
- jobInfo.substitute(flinkArgs.getVariable());
+ jobInfo.substitute(flinkArgs.getVariables());
return jobInfo;
}
diff --git
a/seatunnel-core/seatunnel-core-sql/src/main/java/org/apache/seatunnel/core/sql/job/JobInfo.java
b/seatunnel-core/seatunnel-core-sql/src/main/java/org/apache/seatunnel/core/sql/job/JobInfo.java
index 0ed5a6f..6458551 100644
---
a/seatunnel-core/seatunnel-core-sql/src/main/java/org/apache/seatunnel/core/sql/job/JobInfo.java
+++
b/seatunnel-core/seatunnel-core-sql/src/main/java/org/apache/seatunnel/core/sql/job/JobInfo.java
@@ -17,6 +17,8 @@
package org.apache.seatunnel.core.sql.job;
+import java.util.List;
+
public class JobInfo {
private static final String JOB_NAME = "default sql job";
@@ -44,8 +46,11 @@ public class JobInfo {
return jobContent;
}
- public void substitute(String variable) {
- if (variable != null) {
+ public void substitute(List<String> variables) {
+ if (variables == null) {
+ return;
+ }
+ for (String variable : variables) {
String[] s = variable.split(DELIMITER);
if (s.length == 2) {
jobContent = jobContent.replaceAll(String.format(VAR_REGEX,
s[0].trim()), s[1].trim());
diff --git
a/seatunnel-core/seatunnel-core-sql/src/test/java/org/apache/seatunnel/core/sql/SqlVariableSubstitutionTest.java
b/seatunnel-core/seatunnel-core-sql/src/test/java/org/apache/seatunnel/core/sql/SqlVariableSubstitutionTest.java
index b639a73..30ada8a 100644
---
a/seatunnel-core/seatunnel-core-sql/src/test/java/org/apache/seatunnel/core/sql/SqlVariableSubstitutionTest.java
+++
b/seatunnel-core/seatunnel-core-sql/src/test/java/org/apache/seatunnel/core/sql/SqlVariableSubstitutionTest.java
@@ -42,7 +42,7 @@ public class SqlVariableSubstitutionTest {
JobInfo jobInfo = new JobInfo(jobContent);
Assert.assertFalse(jobInfo.getJobContent().contains("events"));
- jobInfo.substitute(flinkArgs.getVariable());
+ jobInfo.substitute(flinkArgs.getVariables());
Assert.assertTrue(jobInfo.getJobContent().contains("events"));
}
