This is an automated email from the ASF dual-hosted git repository.
wanghailin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 6145da9628 [Improve][Zeta] Handle user privacy when submitting a task
print config logs (#7247)
6145da9628 is described below
commit 6145da96283f2115f18c19c53269be37000e71b2
Author: zhangdonghao <[email protected]>
AuthorDate: Sat Aug 24 16:13:05 2024 +0800
[Improve][Zeta] Handle user privacy when submitting a task print config
logs (#7247)
---
config/seatunnel.yaml | 2 +-
.../core/starter/utils/ConfigBuilder.java | 69 +++++++++++++++++++++-
.../core/starter/utils/ConfigShadeUtils.java | 6 +-
.../core/starter/utils/ConfigShadeTest.java | 43 ++++++++++++++
.../src/test/resources/config.shade.json | 41 +++++++++++++
.../seatunnel/engine/server/utils/RestUtil.java | 2 +-
6 files changed, 156 insertions(+), 7 deletions(-)
diff --git a/config/seatunnel.yaml b/config/seatunnel.yaml
index 5961c83923..6b81b92ea3 100644
--- a/config/seatunnel.yaml
+++ b/config/seatunnel.yaml
@@ -33,4 +33,4 @@ seatunnel:
plugin-config:
namespace: /tmp/seatunnel/checkpoint_snapshot
storage.type: hdfs
- fs.defaultFS: file:///tmp/ # Ensure that the directory has written
permission
\ No newline at end of file
+ fs.defaultFS: file:///tmp/ # Ensure that the directory has written
permission
diff --git
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
index 57d26ee0e7..0bd85213f5 100644
---
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
+++
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
@@ -22,9 +22,11 @@ import
org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigParseOptions;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigSyntax;
import org.apache.seatunnel.shade.com.typesafe.config.impl.Parseable;
import org.apache.seatunnel.api.configuration.ConfigAdapter;
+import org.apache.seatunnel.common.utils.JsonUtils;
import org.apache.seatunnel.common.utils.ParserException;
import lombok.NonNull;
@@ -32,10 +34,14 @@ import lombok.extern.slf4j.Slf4j;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static
org.apache.seatunnel.core.starter.utils.ConfigShadeUtils.DEFAULT_SENSITIVE_KEYWORDS;
/** Used to build the {@link Config} from config file. */
@Slf4j
@@ -76,14 +82,19 @@ public class ConfigBuilder {
adapterSupplier
.map(adapter -> of(adapter, filePath, variables))
.orElseGet(() -> ofInner(filePath, variables));
+ boolean isJson = filePath.getFileName().toString().endsWith(".json");
+ log.info(
+ "Parsed config file: \n{}",
+ mapToString(configDesensitization(config.root().unwrapped()),
isJson));
return config;
}
public static Config of(@NonNull Map<String, Object> objectMap) {
- return of(objectMap, false);
+ return of(objectMap, false, false);
}
- public static Config of(@NonNull Map<String, Object> objectMap, boolean
isEncrypt) {
+ public static Config of(
+ @NonNull Map<String, Object> objectMap, boolean isEncrypt, boolean
isJson) {
log.info("Loading config file from objectMap");
Config config =
ConfigFactory.parseMap(objectMap)
@@ -94,9 +105,49 @@ public class ConfigBuilder {
if (!isEncrypt) {
config = ConfigShadeUtils.decryptConfig(config);
}
+ log.info(
+ "Parsed config file: \n{}",
+ mapToString(configDesensitization(config.root().unwrapped()),
isJson));
return config;
}
+ public static Map<String, Object> configDesensitization(Map<String,
Object> configMap) {
+ return configMap.entrySet().stream()
+ .collect(
+ Collectors.toMap(
+ Map.Entry::getKey,
+ entry -> {
+ String key = entry.getKey();
+ if
(Arrays.asList(DEFAULT_SENSITIVE_KEYWORDS)
+ .contains(key.toLowerCase())) {
+ return "******";
+ }
+ Object value = entry.getValue();
+ if (value instanceof Map) {
+ if ("schema".equals(key)) {
+ return value;
+ }
+ return
configDesensitization((Map<String, Object>) value);
+ } else if (value instanceof List) {
+ return ((List<?>) value)
+ .stream()
+ .map(
+ v -> {
+ if (v
instanceof Map) {
+ return
configDesensitization(
+
(Map<
+
String,
+
Object>)
+
v);
+ }
+ return v;
+ })
+
.collect(Collectors.toList());
+ }
+ return value;
+ }));
+ }
+
public static Config of(
@NonNull ConfigAdapter configAdapter, @NonNull Path filePath,
List<String> variables) {
log.info("With config adapter spi {}",
configAdapter.getClass().getName());
@@ -133,4 +184,18 @@ public class ConfigBuilder {
}
return config;
}
+
+ public static String mapToString(Map<String, Object> configMap, boolean
isJson) {
+ ConfigRenderOptions configRenderOptions =
+
ConfigRenderOptions.concise().setFormatted(true).setJson(isJson);
+ ConfigParseOptions configParseOptions =
+ ConfigParseOptions.defaults().setSyntax(ConfigSyntax.JSON);
+ Config config =
+ ConfigFactory.parseString(JsonUtils.toJsonString(configMap),
configParseOptions)
+
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
+ .resolveWith(
+ ConfigFactory.systemProperties(),
+
ConfigResolveOptions.defaults().setAllowUnresolved(true));
+ return config.root().render(configRenderOptions);
+ }
}
diff --git
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java
index a1e3dffe01..3269ab8cd8 100644
---
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java
+++
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java
@@ -47,8 +47,8 @@ public final class ConfigShadeUtils {
private static final String SHADE_IDENTIFIER_OPTION = "shade.identifier";
- private static final String[] DEFAULT_SENSITIVE_OPTIONS =
- new String[] {"password", "username", "auth"};
+ public static final String[] DEFAULT_SENSITIVE_KEYWORDS =
+ new String[] {"password", "username", "auth", "token"};
private static final Map<String, ConfigShade> CONFIG_SHADES = new
HashMap<>();
@@ -126,7 +126,7 @@ public final class ConfigShadeUtils {
@SuppressWarnings("unchecked")
private static Config processConfig(String identifier, Config config,
boolean isDecrypted) {
ConfigShade configShade = CONFIG_SHADES.getOrDefault(identifier,
DEFAULT_SHADE);
- List<String> sensitiveOptions = new
ArrayList<>(Arrays.asList(DEFAULT_SENSITIVE_OPTIONS));
+ List<String> sensitiveOptions = new
ArrayList<>(Arrays.asList(DEFAULT_SENSITIVE_KEYWORDS));
sensitiveOptions.addAll(Arrays.asList(configShade.sensitiveOptions()));
BiFunction<String, Object, String> processFunction =
(key, value) -> {
diff --git
a/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
b/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
index 9382c68663..4463597e9a 100644
---
a/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
+++
b/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
@@ -19,7 +19,9 @@ package org.apache.seatunnel.core.starter.utils;
import
org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions;
import org.apache.seatunnel.api.configuration.ConfigShade;
import org.apache.seatunnel.common.utils.JsonUtils;
@@ -27,6 +29,7 @@ import org.apache.seatunnel.common.utils.JsonUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import com.beust.jcommander.internal.Lists;
import lombok.extern.slf4j.Slf4j;
import java.net.URISyntaxException;
@@ -68,6 +71,46 @@ public class ConfigShadeTest {
config.getConfigList("source").get(0).getString("password"),
PASSWORD);
}
+ @Test
+ public void testUsePrivacyHandlerHocon() throws URISyntaxException {
+ URL resource = ConfigShadeTest.class.getResource("/config.shade.conf");
+ Assertions.assertNotNull(resource);
+ Config config = ConfigBuilder.of(Paths.get(resource.toURI()),
Lists.newArrayList());
+ config =
+ ConfigFactory.parseMap(
+
ConfigBuilder.configDesensitization(config.root().unwrapped()))
+
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
+ .resolveWith(
+ ConfigFactory.systemProperties(),
+
ConfigResolveOptions.defaults().setAllowUnresolved(true));
+ Assertions.assertEquals(
+ config.getConfigList("source").get(0).getString("username"),
"******");
+ Assertions.assertEquals(
+ config.getConfigList("source").get(0).getString("password"),
"******");
+ String conf = ConfigBuilder.mapToString(config.root().unwrapped(),
false);
+ Assertions.assertTrue(conf.contains("username=\"******\""));
+ }
+
+ @Test
+ public void testUsePrivacyHandlerJson() throws URISyntaxException {
+ URL resource = ConfigShadeTest.class.getResource("/config.shade.json");
+ Assertions.assertNotNull(resource);
+ Config config = ConfigBuilder.of(Paths.get(resource.toURI()),
Lists.newArrayList());
+ config =
+ ConfigFactory.parseMap(
+
ConfigBuilder.configDesensitization(config.root().unwrapped()))
+
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
+ .resolveWith(
+ ConfigFactory.systemProperties(),
+
ConfigResolveOptions.defaults().setAllowUnresolved(true));
+ Assertions.assertEquals(
+ config.getConfigList("source").get(0).getString("username"),
"******");
+ Assertions.assertEquals(
+ config.getConfigList("source").get(0).getString("password"),
"******");
+ String json = ConfigBuilder.mapToString(config.root().unwrapped(),
true);
+ Assertions.assertTrue(json.contains("\"password\" : \"******\""));
+ }
+
@Test
public void testVariableReplacement() throws URISyntaxException {
String jobName = "seatunnel variable test job";
diff --git
a/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade.json
b/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade.json
new file mode 100644
index 0000000000..594c3db8fa
--- /dev/null
+++ b/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade.json
@@ -0,0 +1,41 @@
+{
+ "env" : {
+ "shade.identifier" : "base64",
+ "parallelism" : 1
+ },
+ "source" : [
+ {
+ "plugin_name" : "MySQL-CDC",
+ "base-url" : "jdbc:mysql://localhost:56725",
+ "username" : "c2VhdHVubmVs",
+ "password" : "c2VhdHVubmVsX3Bhc3N3b3Jk",
+ "hostname" : "127.0.0.1",
+ "port" : 56725,
+ "database-name" : "inventory_vwyw0n",
+ "parallelism" : 1,
+ "table-name" : "products",
+ "server-id" : 5656,
+ "schema" : {
+ "fields" : {
+ "name" : "string",
+ "age" : "int",
+ "sex" : "boolean"
+ }
+ },
+ "result_table_name" : "fake"
+ }
+ ],
+ "transform" : [],
+ "sink" : [
+ {
+ "plugin_name" : "Clickhouse",
+ "host" : "localhost:8123",
+ "username" : "c2VhdHVubmVs",
+ "password" : "c2VhdHVubmVsX3Bhc3N3b3Jk",
+ "database" : "default",
+ "table" : "fake_all",
+ "support_upsert" : true,
+ "primary_key" : "id"
+ }
+ ]
+}
diff --git
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/utils/RestUtil.java
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/utils/RestUtil.java
index c2e92c1948..9aaa8cd595 100644
---
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/utils/RestUtil.java
+++
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/utils/RestUtil.java
@@ -67,6 +67,6 @@ public class RestUtil {
public static Config buildConfig(JsonNode jsonNode, boolean isEncrypt) {
Map<String, Object> objectMap = JsonUtils.toMap(jsonNode);
- return ConfigBuilder.of(objectMap, isEncrypt);
+ return ConfigBuilder.of(objectMap, isEncrypt, true);
}
}
