This is an automated email from the ASF dual-hosted git repository.
arshad pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/seatunnel-web.git
The following commit(s) were added to refs/heads/main by this push:
new 46c8a841 [Feature] Add Support for encrypting config fields (#278)
46c8a841 is described below
commit 46c8a841ad29967fe670d77dc37ece40b467f23a
Author: Shashwat Tiwari <[email protected]>
AuthorDate: Tue Feb 11 14:47:00 2025 +0530
[Feature] Add Support for encrypting config fields (#278)
Co-authored-by: BilwaST <[email protected]>
---
.../org/apache/seatunnel/app/common/Constants.java | 2 +
.../seatunnel/app/config/EncryptionConfig.java | 36 +++++++++
.../app/service/impl/DatasourceServiceImpl.java | 19 ++++-
.../app/service/impl/JobInstanceServiceImpl.java | 18 +++++
.../seatunnel/app/utils/ConfigShadeUtil.java | 89 ++++++++++++++++++++++
.../src/main/resources/application.yml | 8 +-
.../src/test/resources/application.yml | 8 ++
7 files changed, 177 insertions(+), 3 deletions(-)
diff --git
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/Constants.java
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/Constants.java
index 886c1f70..22c01834 100644
---
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/Constants.java
+++
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/Constants.java
@@ -215,6 +215,7 @@ public final class Constants {
public static final String COMMON_TASK_TYPE = "common";
public static final String DEFAULT = "default";
+ public static final String ENCRYPTION_TYPE_NONE = "none";
public static final String PASSWORD = "password";
public static final String XXXXXX = "******";
public static final String NULL = "NULL";
@@ -658,4 +659,5 @@ public final class Constants {
public static final String AUTHENTICATION_PROVIDER_LDAP = "LDAP";
public static final String AUTHENTICATION_PROVIDER_DB = "DB";
+ public static final String ENCRYPTION_IDENTIFIER_KEY = "shade.identifier";
}
diff --git
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/config/EncryptionConfig.java
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/config/EncryptionConfig.java
new file mode 100644
index 00000000..4e887321
--- /dev/null
+++
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/config/EncryptionConfig.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.config;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+import lombok.Data;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.seatunnel.app.common.Constants.ENCRYPTION_TYPE_NONE;
+
+@Data
+@Configuration
+@ConfigurationProperties(prefix = "seatunnel-web.datasource.encryption")
+public class EncryptionConfig {
+ private String type = ENCRYPTION_TYPE_NONE;
+ private Set<String> keysToEncrypt = new HashSet<>();
+}
diff --git
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/DatasourceServiceImpl.java
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/DatasourceServiceImpl.java
index 12b1a4f1..ce9cbc65 100644
---
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/DatasourceServiceImpl.java
+++
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/DatasourceServiceImpl.java
@@ -37,6 +37,7 @@ import org.apache.seatunnel.app.service.IJobDefinitionService;
import org.apache.seatunnel.app.service.ITableSchemaService;
import org.apache.seatunnel.app.thirdparty.datasource.DataSourceClientFactory;
import
org.apache.seatunnel.app.thirdparty.framework.SeaTunnelOptionRuleWrapper;
+import org.apache.seatunnel.app.utils.ConfigShadeUtil;
import org.apache.seatunnel.app.utils.ServletUtils;
import org.apache.seatunnel.common.utils.JsonUtils;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
@@ -94,6 +95,8 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
protected static final String DEFAULT_DATASOURCE_PLUGIN_VERSION = "1.0.0";
+ @Autowired private ConfigShadeUtil configShadeUtil;
+
@Override
public String createDatasource(
String datasourceName,
@@ -114,6 +117,7 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
throw new SeatunnelException(
SeatunnelErrorEnum.DATASOURCE_PRAM_NOT_ALLOWED_NULL,
"datasourceConfig");
}
+ configShadeUtil.encryptData(datasourceConfig);
String datasourceConfigStr = JsonUtils.toJsonString(datasourceConfig);
Datasource datasource =
Datasource.builder()
@@ -171,6 +175,7 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
datasource.setUpdateTime(new Date());
datasource.setDescription(description);
if (MapUtils.isNotEmpty(datasourceConfig)) {
+ configShadeUtil.encryptData(datasourceConfig);
String configJson = JsonUtils.toJsonString(datasourceConfig);
datasource.setDatasourceConfig(configJson);
}
@@ -226,6 +231,7 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
String configJson = datasource.getDatasourceConfig();
Map<String, String> datasourceConfig =
JsonUtils.toMap(configJson, String.class, String.class);
+ configShadeUtil.decryptData(datasourceConfig);
String pluginName = datasource.getPluginName();
return DataSourceClientFactory.getDataSourceClient()
.checkDataSourceConnectivity(pluginName, datasourceConfig);
@@ -274,6 +280,7 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
Map<String, String> datasourceConfig =
JsonUtils.toMap(config, String.class, String.class);
+ configShadeUtil.decryptData(datasourceConfig);
return DataSourceClientFactory.getDataSourceClient()
.getDatabases(pluginName, datasourceConfig);
}
@@ -305,6 +312,7 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
options.put("filterName", filterName);
String pluginName = datasource.getPluginName();
if (BooleanUtils.isNotTrue(checkIsSupportVirtualTable(pluginName))) {
+ configShadeUtil.decryptData(datasourceConfig);
return DataSourceClientFactory.getDataSourceClient()
.getTables(pluginName, databaseName, datasourceConfig,
options);
}
@@ -324,6 +332,7 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
Map<String, String> options = new HashMap<>();
String pluginName = datasource.getPluginName();
if (BooleanUtils.isNotTrue(checkIsSupportVirtualTable(pluginName))) {
+ configShadeUtil.decryptData(datasourceConfig);
return DataSourceClientFactory.getDataSourceClient()
.getTables(pluginName, databaseName, datasourceConfig,
options);
}
@@ -345,6 +354,7 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
ITableSchemaService tableSchemaService =
(ITableSchemaService)
applicationContext.getBean("tableSchemaServiceImpl");
if (BooleanUtils.isNotTrue(checkIsSupportVirtualTable(pluginName))) {
+ configShadeUtil.decryptData(datasourceConfig);
List<TableField> tableFields =
DataSourceClientFactory.getDataSourceClient()
.getTableFields(pluginName, datasourceConfig,
databaseName, tableName);
@@ -434,6 +444,7 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
datasource.getDatasourceConfig(),
String.class,
String.class);
+
configShadeUtil.decryptData(datasourceConfig);
datasourceRes.setDatasourceConfig(datasourceConfig);
datasourceRes.setCreateUserId(datasource.getCreateUserId());
datasourceRes.setUpdateUserId(datasource.getUpdateUserId());
@@ -503,7 +514,10 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
throw new
SeatunnelException(SeatunnelErrorEnum.DATASOURCE_NOT_FOUND, datasourceId);
}
String configJson = datasource.getDatasourceConfig();
- return JsonUtils.toMap(configJson, String.class, String.class);
+ Map<String, String> datasourceConfig =
+ JsonUtils.toMap(configJson, String.class, String.class);
+ configShadeUtil.decryptData(datasourceConfig);
+ return datasourceConfig;
}
@Override
@@ -591,7 +605,7 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
return getDatasourceDetailRes(datasource);
}
- private static DatasourceDetailRes getDatasourceDetailRes(Datasource
datasource) {
+ private DatasourceDetailRes getDatasourceDetailRes(Datasource datasource) {
DatasourceDetailRes datasourceDetailRes = new DatasourceDetailRes();
datasourceDetailRes.setId(datasource.getId().toString());
datasourceDetailRes.setDatasourceName(datasource.getDatasourceName());
@@ -603,6 +617,7 @@ public class DatasourceServiceImpl extends
SeatunnelBaseServiceImpl
Map<String, String> datasourceConfig =
JsonUtils.toMap(datasource.getDatasourceConfig(),
String.class, String.class);
+ configShadeUtil.decryptData(datasourceConfig);
// convert option rule
datasourceDetailRes.setDatasourceConfig(datasourceConfig);
return datasourceDetailRes;
diff --git
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
index e7883395..cd453048 100644
---
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
+++
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java
@@ -31,6 +31,7 @@ import org.apache.seatunnel.api.configuration.util.OptionRule;
import org.apache.seatunnel.api.env.ParsingMode;
import org.apache.seatunnel.app.bean.connector.ConnectorCache;
import org.apache.seatunnel.app.config.ConnectorDataSourceMapperConfig;
+import org.apache.seatunnel.app.config.EncryptionConfig;
import org.apache.seatunnel.app.dal.dao.IJobDefinitionDao;
import org.apache.seatunnel.app.dal.dao.IJobInstanceDao;
import org.apache.seatunnel.app.dal.dao.IJobLineDao;
@@ -60,6 +61,7 @@ import org.apache.seatunnel.app.service.IJobMetricsService;
import org.apache.seatunnel.app.service.IVirtualTableService;
import
org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcherUtils;
import
org.apache.seatunnel.app.thirdparty.transfrom.TransformConfigSwitcherUtils;
+import org.apache.seatunnel.app.utils.ConfigShadeUtil;
import org.apache.seatunnel.app.utils.JobUtils;
import org.apache.seatunnel.app.utils.SeaTunnelConfigUtil;
import org.apache.seatunnel.app.utils.ServletUtils;
@@ -74,6 +76,7 @@ import org.apache.seatunnel.server.common.SeatunnelException;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -95,6 +98,8 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import static com.google.common.base.Preconditions.checkArgument;
+import static
org.apache.seatunnel.app.common.Constants.ENCRYPTION_IDENTIFIER_KEY;
+import static org.apache.seatunnel.app.common.Constants.ENCRYPTION_TYPE_NONE;
import static
org.apache.seatunnel.app.utils.TaskOptionUtils.getTransformOption;
@Service
@@ -124,6 +129,10 @@ public class JobInstanceServiceImpl extends
SeatunnelBaseServiceImpl
@Resource private IJobMetricsService jobMetricsService;
+ @Autowired private ConfigShadeUtil configShadeUtil;
+
+ @Autowired private EncryptionConfig encryptionConfig;
+
@Override
public JobExecutorRes createExecuteResource(
@NonNull Long jobDefineId, JobExecParam executeParam) {
@@ -324,6 +333,14 @@ public class JobInstanceServiceImpl extends
SeatunnelBaseServiceImpl
if (sinkMap.size() > 0) {
sinks = getConnectorConfig(sinkMap);
}
+
+ if (!encryptionConfig.getType().equals(ENCRYPTION_TYPE_NONE)) {
+ envConfig =
+ envConfig.withValue(
+ ENCRYPTION_IDENTIFIER_KEY,
+
ConfigValueFactory.fromAnyRef(encryptionConfig.getType()));
+ }
+
String env =
envConfig
.root()
@@ -575,6 +592,7 @@ public class JobInstanceServiceImpl extends
SeatunnelBaseServiceImpl
String connectorType,
Map<String, String> config,
OptionRule optionRule) {
+ configShadeUtil.encryptData(config);
return parseConfigWithOptionRule(
pluginType, connectorType, ConfigFactory.parseMap(config),
optionRule);
}
diff --git
a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/ConfigShadeUtil.java
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/ConfigShadeUtil.java
new file mode 100644
index 00000000..ae2db772
--- /dev/null
+++
b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/utils/ConfigShadeUtil.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.app.utils;
+
+import org.apache.seatunnel.app.config.EncryptionConfig;
+import org.apache.seatunnel.core.starter.utils.ConfigShadeUtils;
+import org.apache.seatunnel.server.common.SeatunnelErrorEnum;
+import org.apache.seatunnel.server.common.SeatunnelException;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Map;
+
+import static org.apache.seatunnel.app.common.Constants.ENCRYPTION_TYPE_NONE;
+
+@Slf4j
+@Component
+public class ConfigShadeUtil {
+
+ @Autowired private EncryptionConfig encryptionConfig;
+
+ public void encryptData(Map<String, String> datasourceConfig) {
+ if (encryptionConfig.getType().equals(ENCRYPTION_TYPE_NONE)) {
+ return;
+ }
+ for (String key : encryptionConfig.getKeysToEncrypt()) {
+ String value = datasourceConfig.get(key);
+ if (StringUtils.isNotEmpty(value)) {
+ try {
+ String processedValue =
+
ConfigShadeUtils.encryptOption(encryptionConfig.getType(), value);
+ datasourceConfig.replace(key, processedValue);
+ } catch (IllegalArgumentException ex) {
+ log.error("encryption for key {} failed", key);
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG,
+ String.format(
+ "encryption failed for key: %s, check if
the keys were persisted in expected format",
+ key),
+ ex);
+ }
+ }
+ }
+ }
+
+ public void decryptData(Map<String, String> datasourceConfig) {
+ if (encryptionConfig.getType().equals(ENCRYPTION_TYPE_NONE)) {
+ return;
+ }
+ for (String key : encryptionConfig.getKeysToEncrypt()) {
+ String value = datasourceConfig.get(key);
+ if (StringUtils.isNotEmpty(value)) {
+ try {
+ String processedValue =
+
ConfigShadeUtils.decryptOption(encryptionConfig.getType(), value);
+ datasourceConfig.replace(key, processedValue);
+ } catch (IllegalArgumentException ex) {
+ log.error("decryption for key {} failed", key);
+ throw new SeatunnelException(
+ SeatunnelErrorEnum.ERROR_CONFIG,
+ String.format(
+ "decryption failed for key: %s, check if
the keys were persisted in expected format",
+ key),
+ ex);
+ }
+ }
+ }
+ }
+}
diff --git a/seatunnel-server/seatunnel-app/src/main/resources/application.yml
b/seatunnel-server/seatunnel-app/src/main/resources/application.yml
index 5a6e3a06..a90a6366 100644
--- a/seatunnel-server/seatunnel-app/src/main/resources/application.yml
+++ b/seatunnel-server/seatunnel-app/src/main/resources/application.yml
@@ -52,7 +52,13 @@ jwt:
secretKey:
algorithm: HS256
-
+seatunnel-web:
+ datasource:
+ encryption:
+ type: none
+ keys-to-encrypt:
+ - password
+ - auth
---
spring:
config:
diff --git a/seatunnel-web-it/src/test/resources/application.yml
b/seatunnel-web-it/src/test/resources/application.yml
index ef285655..752fbc7b 100644
--- a/seatunnel-web-it/src/test/resources/application.yml
+++ b/seatunnel-web-it/src/test/resources/application.yml
@@ -48,6 +48,14 @@ jwt:
secretKey: https://github.com/apache/seatunnel
algorithm: HS256
+seatunnel-web:
+ datasource:
+ encryption:
+ type: none
+ keys-to-encrypt:
+ - password
+ - auth
+
---
spring:
application:
