This is an automated email from the ASF dual-hosted git repository.
wuchunfu pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 2f314043a6 [hotfix][zeta] conf property is null, print log bug fix
(#7487)
2f314043a6 is described below
commit 2f314043a6299021fd063f327660b537a9dc2f5d
Author: zhangdonghao <[email protected]>
AuthorDate: Sun Aug 25 00:57:57 2024 +0800
[hotfix][zeta] conf property is null, print log bug fix (#7487)
---
.../core/starter/utils/ConfigBuilder.java | 73 +++++++++++-----------
.../core/starter/utils/ConfigShadeTest.java | 27 +++++++-
.../src/test/resources/config.shade_caseNull.conf | 63 +++++++++++++++++++
3 files changed, 123 insertions(+), 40 deletions(-)
diff --git
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
index 0bd85213f5..57392f6a3f 100644
---
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
+++
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
@@ -35,6 +35,7 @@ import lombok.extern.slf4j.Slf4j;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -85,7 +86,7 @@ public class ConfigBuilder {
boolean isJson = filePath.getFileName().toString().endsWith(".json");
log.info(
"Parsed config file: \n{}",
- mapToString(configDesensitization(config.root().unwrapped()),
isJson));
+ mapToString(configDesensitization(config.root().unwrapped())));
return config;
}
@@ -107,45 +108,45 @@ public class ConfigBuilder {
}
log.info(
"Parsed config file: \n{}",
- mapToString(configDesensitization(config.root().unwrapped()),
isJson));
+ mapToString(configDesensitization(config.root().unwrapped())));
return config;
}
public static Map<String, Object> configDesensitization(Map<String,
Object> configMap) {
return configMap.entrySet().stream()
.collect(
- Collectors.toMap(
- Map.Entry::getKey,
- entry -> {
- String key = entry.getKey();
- if
(Arrays.asList(DEFAULT_SENSITIVE_KEYWORDS)
- .contains(key.toLowerCase())) {
- return "******";
- }
- Object value = entry.getValue();
- if (value instanceof Map) {
- if ("schema".equals(key)) {
- return value;
- }
- return
configDesensitization((Map<String, Object>) value);
- } else if (value instanceof List) {
- return ((List<?>) value)
- .stream()
- .map(
- v -> {
- if (v
instanceof Map) {
- return
configDesensitization(
-
(Map<
-
String,
-
Object>)
-
v);
- }
+ HashMap::new,
+ (m, p) -> {
+ String key = p.getKey();
+ Object value = p.getValue();
+ if (Arrays.asList(DEFAULT_SENSITIVE_KEYWORDS)
+ .contains(key.toLowerCase())) {
+ m.put(key, "******");
+ } else {
+ if (value instanceof Map<?, ?>) {
+ m.put(key,
configDesensitization((Map<String, Object>) value));
+ } else if (value instanceof List<?>) {
+ List<?> listValue = (List<?>) value;
+ List<Object> newList =
+ listValue.stream()
+ .map(
+ v -> {
+ if (v
instanceof Map<?, ?>) {
+ return
configDesensitization(
+
(Map<String, Object>)
+
v);
+ } else {
return v;
- })
-
.collect(Collectors.toList());
- }
- return value;
- }));
+ }
+ })
+
.collect(Collectors.toList());
+ m.put(key, newList);
+ } else {
+ m.put(key, value);
+ }
+ }
+ },
+ HashMap::putAll);
}
public static Config of(
@@ -185,9 +186,7 @@ public class ConfigBuilder {
return config;
}
- public static String mapToString(Map<String, Object> configMap, boolean
isJson) {
- ConfigRenderOptions configRenderOptions =
-
ConfigRenderOptions.concise().setFormatted(true).setJson(isJson);
+ public static String mapToString(Map<String, Object> configMap) {
ConfigParseOptions configParseOptions =
ConfigParseOptions.defaults().setSyntax(ConfigSyntax.JSON);
Config config =
@@ -196,6 +195,6 @@ public class ConfigBuilder {
.resolveWith(
ConfigFactory.systemProperties(),
ConfigResolveOptions.defaults().setAllowUnresolved(true));
- return config.root().render(configRenderOptions);
+ return config.root().render(CONFIG_RENDER_OPTIONS);
}
}
diff --git
a/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
b/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
index 4463597e9a..db5df31866 100644
---
a/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
+++
b/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
@@ -87,8 +87,8 @@ public class ConfigShadeTest {
config.getConfigList("source").get(0).getString("username"),
"******");
Assertions.assertEquals(
config.getConfigList("source").get(0).getString("password"),
"******");
- String conf = ConfigBuilder.mapToString(config.root().unwrapped(),
false);
- Assertions.assertTrue(conf.contains("username=\"******\""));
+ String conf = ConfigBuilder.mapToString(config.root().unwrapped());
+ Assertions.assertTrue(conf.contains("\"password\" : \"******\""));
}
@Test
@@ -107,10 +107,31 @@ public class ConfigShadeTest {
config.getConfigList("source").get(0).getString("username"),
"******");
Assertions.assertEquals(
config.getConfigList("source").get(0).getString("password"),
"******");
- String json = ConfigBuilder.mapToString(config.root().unwrapped(),
true);
+ String json = ConfigBuilder.mapToString(config.root().unwrapped());
Assertions.assertTrue(json.contains("\"password\" : \"******\""));
}
+ @Test
+ public void testConfNull() throws URISyntaxException {
+ URL resource =
ConfigShadeTest.class.getResource("/config.shade_caseNull.conf");
+ Assertions.assertNotNull(resource);
+ Config config = ConfigBuilder.of(Paths.get(resource.toURI()),
Lists.newArrayList());
+ config =
+ ConfigFactory.parseMap(
+
ConfigBuilder.configDesensitization(config.root().unwrapped()))
+
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
+ .resolveWith(
+ ConfigFactory.systemProperties(),
+
ConfigResolveOptions.defaults().setAllowUnresolved(true));
+ Assertions.assertEquals(
+ config.getConfigList("source").get(0).getString("username"),
"******");
+ Assertions.assertEquals(
+ config.getConfigList("source").get(0).getString("password"),
"******");
+ String conf = ConfigBuilder.mapToString(config.root().unwrapped());
+ Assertions.assertTrue(conf.contains("\"password\" : \"******\""));
+ Assertions.assertTrue(conf.contains("\"test\" : null"));
+ }
+
@Test
public void testVariableReplacement() throws URISyntaxException {
String jobName = "seatunnel variable test job";
diff --git
a/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_caseNull.conf
b/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_caseNull.conf
new file mode 100644
index 0000000000..f0a926807d
--- /dev/null
+++
b/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_caseNull.conf
@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ shade.identifier = "base64"
+}
+
+source {
+ MySQL-CDC {
+ schema {
+ fields {
+ name = string
+ age = int
+ sex = boolean
+ }
+ }
+ result_table_name = "fake"
+ parallelism = 1
+ server-id = 5656
+ port = 56725
+ hostname = "127.0.0.1"
+ username = "c2VhdHVubmVs"
+ password = "c2VhdHVubmVsX3Bhc3N3b3Jk"
+ database-name = "inventory_vwyw0n"
+ table-name = "products"
+ base-url = "jdbc:mysql://localhost:56725"
+ test = null
+ }
+}
+
+transform {
+}
+
+sink {
+ # choose stdout output plugin to output data to console
+ Clickhouse {
+ host = "localhost:8123"
+ database = "default"
+ table = "fake_all"
+ username = "c2VhdHVubmVs"
+ password = "c2VhdHVubmVsX3Bhc3N3b3Jk"
+
+ # cdc options
+ primary_key = "id"
+ support_upsert = true
+ test = null
+ }
+}
