This is an automated email from the ASF dual-hosted git repository.
liugddx pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new aec9a93412 [Feature][Http] Http paramter support custom encryption
(#5727)
aec9a93412 is described below
commit aec9a93412cecff71d53cd4c900ef7567c9c2b3b
Author: fang <[email protected]>
AuthorDate: Wed Nov 8 11:47:08 2023 +0800
[Feature][Http] Http paramter support custom encryption (#5727)
---
docs/en/seatunnel-engine/rest-api.md | 89 ++++++++++++++++++++++
.../core/starter/utils/ConfigBuilder.java | 17 ++++-
.../org/apache/seatunnel/engine/e2e/RestApiIT.java | 54 +++++++++++++
.../seatunnel/engine/server/rest/RestConstant.java | 1 +
.../server/rest/RestHttpPostCommandProcessor.java | 18 ++++-
.../seatunnel/engine/server/utils/RestUtil.java | 4 +-
6 files changed, 178 insertions(+), 5 deletions(-)
diff --git a/docs/en/seatunnel-engine/rest-api.md
b/docs/en/seatunnel-engine/rest-api.md
index 3f8cf910ea..f899b6b711 100644
--- a/docs/en/seatunnel-engine/rest-api.md
+++ b/docs/en/seatunnel-engine/rest-api.md
@@ -262,3 +262,92 @@ network:
</details>
+------------------------------------------------------------------------------------------
+
+### Encrypt Config.
+
+<details>
+<summary><code>POST</code>
<code><b>/hazelcast/rest/maps/encrypt-config</b></code> <code>(Returns the
encrypted config if config is encrypted successfully.)</code></summary>
+For more information about customize encryption, please refer to the
documentation
[config-encryption-decryption](../connector-v2/Config-Encryption-Decryption.md).
+
+#### Body
+
+```json
+{
+ "env": {
+ "execution.parallelism": 1,
+ "shade.identifier":"base64"
+ },
+ "source": [
+ {
+ "plugin_name": "MySQL-CDC",
+ "schema" : {
+ "fields": {
+ "name": "string",
+ "age": "int"
+ }
+ },
+ "result_table_name": "fake",
+ "parallelism": 1,
+ "hostname": "127.0.0.1",
+ "username": "seatunnel",
+ "password": "seatunnel_password",
+ "table-name": "inventory_vwyw0n"
+ }
+ ],
+ "transform": [
+ ],
+ "sink": [
+ {
+ "plugin_name": "Clickhouse",
+ "host": "localhost:8123",
+ "database": "default",
+ "table": "fake_all",
+ "username": "seatunnel",
+ "password": "seatunnel_password"
+ }
+ ]
+}
+```
+
+#### Responses
+
+```json
+{
+ "env": {
+ "execution.parallelism": 1,
+ "shade.identifier": "base64"
+ },
+ "source": [
+ {
+ "plugin_name": "MySQL-CDC",
+ "schema": {
+ "fields": {
+ "name": "string",
+ "age": "int"
+ }
+ },
+ "result_table_name": "fake",
+ "parallelism": 1,
+ "hostname": "127.0.0.1",
+ "username": "c2VhdHVubmVs",
+ "password": "c2VhdHVubmVsX3Bhc3N3b3Jk",
+ "table-name": "inventory_vwyw0n"
+ }
+ ],
+ "transform": [],
+ "sink": [
+ {
+ "plugin_name": "Clickhouse",
+ "host": "localhost:8123",
+ "database": "default",
+ "table": "fake_all",
+ "username": "c2VhdHVubmVs",
+ "password": "c2VhdHVubmVsX3Bhc3N3b3Jk"
+ }
+ ]
+}
+```
+
+</details>
+
diff --git
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
index ad063acac8..9ef0c2020d 100644
---
a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
+++
b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigBuilder.java
@@ -70,9 +70,22 @@ public class ConfigBuilder {
}
public static Config of(@NonNull Map<String, Object> objectMap) {
+ return of(objectMap, false);
+ }
+
+ public static Config of(@NonNull Map<String, Object> objectMap, boolean
isEncrypt) {
log.info("Loading config file from objectMap");
- Config config = ConfigFactory.parseMap(objectMap);
- return ConfigShadeUtils.decryptConfig(config);
+ Config config =
+ ConfigFactory.parseMap(objectMap)
+
.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))
+ .resolveWith(
+ ConfigFactory.systemProperties(),
+
ConfigResolveOptions.defaults().setAllowUnresolved(true));
+ if (!isEncrypt) {
+ config = ConfigShadeUtils.decryptConfig(config);
+ }
+ log.info("Parsed config file: {}",
config.root().render(CONFIG_RENDER_OPTIONS));
+ return config;
}
public static Config of(@NonNull ConfigAdapter configAdapter, @NonNull
Path filePath) {
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/RestApiIT.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/RestApiIT.java
index c74e18578f..eefefd80f9 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/RestApiIT.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/RestApiIT.java
@@ -253,6 +253,60 @@ public class RestApiIT {
.body("message", equalTo("Please provide jobId when start with
save point."));
}
+ @Test
+ public void testEncryptConfig() {
+ String config =
+ "{\n"
+ + " \"env\": {\n"
+ + " \"execution.parallelism\": 1,\n"
+ + " \"shade.identifier\":\"base64\"\n"
+ + " },\n"
+ + " \"source\": [\n"
+ + " {\n"
+ + " \"plugin_name\": \"MySQL-CDC\",\n"
+ + " \"schema\" : {\n"
+ + " \"fields\": {\n"
+ + " \"name\": \"string\",\n"
+ + " \"age\": \"int\"\n"
+ + " }\n"
+ + " },\n"
+ + " \"result_table_name\": \"fake\",\n"
+ + " \"parallelism\": 1,\n"
+ + " \"hostname\": \"127.0.0.1\",\n"
+ + " \"username\": \"seatunnel\",\n"
+ + " \"password\": \"seatunnel_password\",\n"
+ + " \"table-name\": \"inventory_vwyw0n\"\n"
+ + " }\n"
+ + " ],\n"
+ + " \"transform\": [\n"
+ + " ],\n"
+ + " \"sink\": [\n"
+ + " {\n"
+ + " \"plugin_name\": \"Clickhouse\",\n"
+ + " \"host\": \"localhost:8123\",\n"
+ + " \"database\": \"default\",\n"
+ + " \"table\": \"fake_all\",\n"
+ + " \"username\": \"seatunnel\",\n"
+ + " \"password\": \"seatunnel_password\"\n"
+ + " }\n"
+ + " ]\n"
+ + "}";
+ given().body(config)
+ .post(
+ HOST
+ + hazelcastInstance
+ .getCluster()
+ .getLocalMember()
+ .getAddress()
+ .getPort()
+ + RestConstant.ENCRYPT_CONFIG)
+ .then()
+ .statusCode(200)
+ .body("source[0].result_table_name", equalTo("fake"))
+ .body("source[0].username", equalTo("c2VhdHVubmVs"))
+ .body("source[0].password",
equalTo("c2VhdHVubmVsX3Bhc3N3b3Jk"));
+ }
+
@AfterEach
void afterClass() {
if (hazelcastInstance != null) {
diff --git
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestConstant.java
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestConstant.java
index c3178e3672..0b9d0dc15d 100644
---
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestConstant.java
+++
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestConstant.java
@@ -43,6 +43,7 @@ public class RestConstant {
public static final String RUNNING_JOBS_URL =
"/hazelcast/rest/maps/running-jobs";
public static final String RUNNING_JOB_URL =
"/hazelcast/rest/maps/running-job";
public static final String SUBMIT_JOB_URL =
"/hazelcast/rest/maps/submit-job";
+ public static final String ENCRYPT_CONFIG =
"/hazelcast/rest/maps/encrypt-config";
public static final String SYSTEM_MONITORING_INFORMATION =
"/hazelcast/rest/maps/system-monitoring-information";
diff --git
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestHttpPostCommandProcessor.java
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestHttpPostCommandProcessor.java
index e8b2337ef7..02158bbf34 100644
---
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestHttpPostCommandProcessor.java
+++
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestHttpPostCommandProcessor.java
@@ -19,8 +19,10 @@ package org.apache.seatunnel.engine.server.rest;
import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
+import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions;
import org.apache.seatunnel.common.utils.JsonUtils;
+import org.apache.seatunnel.core.starter.utils.ConfigShadeUtils;
import org.apache.seatunnel.engine.common.Constant;
import org.apache.seatunnel.engine.common.config.JobConfig;
import org.apache.seatunnel.engine.common.utils.PassiveCompletableFuture;
@@ -33,6 +35,7 @@ import org.apache.seatunnel.engine.server.utils.RestUtil;
import com.hazelcast.internal.ascii.TextCommandService;
import com.hazelcast.internal.ascii.rest.HttpCommandProcessor;
import com.hazelcast.internal.ascii.rest.HttpPostCommand;
+import com.hazelcast.internal.json.Json;
import com.hazelcast.internal.json.JsonObject;
import com.hazelcast.internal.serialization.Data;
@@ -42,10 +45,12 @@ import java.util.Map;
import static com.hazelcast.internal.ascii.rest.HttpStatusCode.SC_400;
import static com.hazelcast.internal.ascii.rest.HttpStatusCode.SC_500;
+import static
org.apache.seatunnel.engine.server.rest.RestConstant.ENCRYPT_CONFIG;
import static
org.apache.seatunnel.engine.server.rest.RestConstant.STOP_JOB_URL;
import static
org.apache.seatunnel.engine.server.rest.RestConstant.SUBMIT_JOB_URL;
public class RestHttpPostCommandProcessor extends
HttpCommandProcessor<HttpPostCommand> {
+
private final Log4j2HttpPostCommandProcessor original;
public RestHttpPostCommandProcessor(TextCommandService textCommandService)
{
@@ -69,6 +74,8 @@ public class RestHttpPostCommandProcessor extends
HttpCommandProcessor<HttpPostC
handleSubmitJob(httpPostCommand, uri);
} else if (uri.startsWith(STOP_JOB_URL)) {
handleStopJob(httpPostCommand, uri);
+ } else if (uri.startsWith(ENCRYPT_CONFIG)) {
+ handleEncrypt(httpPostCommand);
} else {
original.handle(httpPostCommand);
}
@@ -92,7 +99,7 @@ public class RestHttpPostCommandProcessor extends
HttpCommandProcessor<HttpPostC
throws IllegalArgumentException {
Map<String, String> requestParams = new HashMap<>();
RestUtil.buildRequestParams(requestParams, uri);
- Config config = RestUtil.buildConfig(requestHandle(httpPostCommand));
+ Config config = RestUtil.buildConfig(requestHandle(httpPostCommand),
false);
JobConfig jobConfig = new JobConfig();
jobConfig.setName(requestParams.get(RestConstant.JOB_NAME));
@@ -153,6 +160,15 @@ public class RestHttpPostCommandProcessor extends
HttpCommandProcessor<HttpPostC
new JsonObject().add(RestConstant.JOB_ID,
map.get(RestConstant.JOB_ID).toString()));
}
+ private void handleEncrypt(HttpPostCommand httpPostCommand) {
+ Config config = RestUtil.buildConfig(requestHandle(httpPostCommand),
true);
+ Config encryptConfig = ConfigShadeUtils.encryptConfig(config);
+ String encryptString =
+
encryptConfig.root().render(ConfigRenderOptions.concise().setJson(true));
+ JsonObject jsonObject = Json.parse(encryptString).asObject();
+ this.prepareResponse(httpPostCommand, jsonObject);
+ }
+
@Override
public void handleRejection(HttpPostCommand httpPostCommand) {
handle(httpPostCommand);
diff --git
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/utils/RestUtil.java
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/utils/RestUtil.java
index d2ac601890..508db2f56a 100644
---
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/utils/RestUtil.java
+++
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/utils/RestUtil.java
@@ -63,8 +63,8 @@ public class RestUtil {
}
}
- public static Config buildConfig(JsonNode jsonNode) {
+ public static Config buildConfig(JsonNode jsonNode, boolean isEncrypt) {
Map<String, Object> objectMap = JsonUtils.toMap(jsonNode);
- return ConfigBuilder.of(objectMap);
+ return ConfigBuilder.of(objectMap, isEncrypt);
}
}
