This is an automated email from the ASF dual-hosted git repository.
klion26 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 66a5254e5 [AMORO-3335] Add interface ConfigShade to support encryption
of sensitive configuration items and provide a base64 encoding implementation
(#3396)
66a5254e5 is described below
commit 66a5254e56cc62c27e4d6f4a890237059ee11756
Author: Jzjsnow <[email protected]>
AuthorDate: Wed Feb 5 10:09:16 2025 +0800
[AMORO-3335] Add interface ConfigShade to support encryption of sensitive
configuration items and provide a base64 encoding implementation (#3396)
* [AMORO-3335] Add interface ConfigShade to support encryption of sensitive
configuration items and provide a base64 encoding implementation
---
.../apache/amoro/server/AmoroServiceContainer.java | 6 +-
.../amoro/server/config/TestConfigShade.java | 137 +++++++++++++++++++++
.../resources/configs/config-base64-shade.yaml | 38 ++++++
.../resources/configs/config-default-shade.yaml | 37 ++++++
.../org/apache/amoro/config/shade/ConfigShade.java | 48 ++++++++
.../amoro/config/shade/impl/Base64ConfigShade.java | 41 ++++++
.../amoro/config/shade/utils/ConfigShadeUtils.java | 137 +++++++++++++++++++++
.../org.apache.amoro.config.shade.ConfigShade | 20 +++
charts/amoro/templates/amoro-configmap.yaml | 13 +-
charts/amoro/tests/amoro-configmap_test.yaml | 28 +++++
charts/amoro/values.yaml | 6 +
dist/src/main/amoro-bin/conf/config.yaml | 5 +
12 files changed, 514 insertions(+), 2 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java
b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java
index f2880ebf6..a751eeea1 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java
@@ -27,6 +27,7 @@ import org.apache.amoro.api.AmoroTableMetastore;
import org.apache.amoro.api.OptimizingService;
import org.apache.amoro.config.ConfigHelpers;
import org.apache.amoro.config.Configurations;
+import org.apache.amoro.config.shade.utils.ConfigShadeUtils;
import org.apache.amoro.exception.AmoroRuntimeException;
import org.apache.amoro.server.catalog.CatalogManager;
import org.apache.amoro.server.catalog.DefaultCatalogManager;
@@ -459,6 +460,8 @@ public class AmoroServiceContainer {
// If same configurations in files and environment variables,
environment variables have
// higher priority.
expandedConfigurationMap.putAll(envConfig);
+ // Decrypt the sensitive configurations if specified
+ expandedConfigurationMap =
ConfigShadeUtils.decryptConfig(expandedConfigurationMap);
serviceConfig = Configurations.fromObjectMap(expandedConfigurationMap);
AmoroManagementConfValidator.validateConfig(serviceConfig);
dataSource = DataSourceFactory.createDataSource(serviceConfig);
@@ -533,7 +536,8 @@ public class AmoroServiceContainer {
}
@SuppressWarnings("unchecked")
- private void expandConfigMap(
+ @VisibleForTesting
+ public static void expandConfigMap(
Map<String, Object> config, String prefix, Map<String, Object> result) {
for (Map.Entry<String, Object> entry : config.entrySet()) {
String key = entry.getKey();
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/config/TestConfigShade.java
b/amoro-ams/src/test/java/org/apache/amoro/server/config/TestConfigShade.java
new file mode 100644
index 000000000..92530936f
--- /dev/null
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/config/TestConfigShade.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.config;
+
+import static org.apache.amoro.server.AmoroServiceContainer.expandConfigMap;
+
+import org.apache.amoro.config.shade.impl.Base64ConfigShade;
+import org.apache.amoro.config.shade.utils.ConfigShadeUtils;
+import org.apache.amoro.server.AmoroManagementConf;
+import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap;
+import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
+import org.apache.amoro.shade.guava32.com.google.common.io.Resources;
+import
org.apache.amoro.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import org.apache.amoro.utils.JacksonUtil;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.yaml.snakeyaml.Yaml;
+
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Base64;
+import java.util.Map;
+
+public class TestConfigShade {
+ private static final String USERNAME = "admin";
+ private static final String PASSWORD = "password";
+
+ private static final String BASE64_CONFIG_SHADE_IDENTIFIER =
+ new Base64ConfigShade().getIdentifier();
+
+ @Test
+ public void testDecryptOptions() {
+ String encryptUsername = getBase64EncodedText(USERNAME);
+ String encryptPassword = getBase64EncodedText(PASSWORD);
+ Assertions.assertEquals(encryptUsername, "YWRtaW4=");
+ Assertions.assertEquals(encryptPassword, "cGFzc3dvcmQ=");
+
+ String decryptUsername =
+ ConfigShadeUtils.decryptOption(BASE64_CONFIG_SHADE_IDENTIFIER,
encryptUsername);
+ String decryptPassword =
+ ConfigShadeUtils.decryptOption(BASE64_CONFIG_SHADE_IDENTIFIER,
encryptPassword);
+ Assertions.assertEquals(decryptUsername, USERNAME);
+ Assertions.assertEquals(decryptPassword, PASSWORD);
+ }
+
+ private String getBase64EncodedText(String plaintext) {
+ return
Base64.getEncoder().encodeToString(plaintext.getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void testDecryptServiceConfigWithDefaultShade() throws Exception {
+ URL resource = Resources.getResource("configs/config-default-shade.yaml");
+ JsonNode yamlConfig =
+ JacksonUtil.fromObjects(
+ new
Yaml().loadAs(Files.newInputStream(Paths.get(resource.toURI())), Map.class));
+ Map<String, Object> systemConfig =
+ JacksonUtil.getMap(
+ yamlConfig,
+ AmoroManagementConf.SYSTEM_CONFIG,
+ new TypeReference<Map<String, Object>>() {});
+ Map<String, Object> expandedConfigurationMap = Maps.newHashMap();
+ expandConfigMap(systemConfig, "", expandedConfigurationMap);
+ expandedConfigurationMap =
ConfigShadeUtils.decryptConfig(expandedConfigurationMap);
+
+ Assertions.assertEquals(decryptedServiceConfigWithDefaultShade,
expandedConfigurationMap);
+ }
+
+ @Test
+ void testDecryptServiceConfigWithBase64Shade() throws Exception {
+ URL resource = Resources.getResource("configs/config-base64-shade.yaml");
+ JsonNode yamlConfig =
+ JacksonUtil.fromObjects(
+ new
Yaml().loadAs(Files.newInputStream(Paths.get(resource.toURI())), Map.class));
+ Map<String, Object> systemConfig =
+ JacksonUtil.getMap(
+ yamlConfig,
+ AmoroManagementConf.SYSTEM_CONFIG,
+ new TypeReference<Map<String, Object>>() {});
+ Map<String, Object> expandedConfigurationMap = Maps.newHashMap();
+ expandConfigMap(systemConfig, "", expandedConfigurationMap);
+ expandedConfigurationMap =
ConfigShadeUtils.decryptConfig(expandedConfigurationMap);
+
+ Assertions.assertEquals(decryptedServiceConfigWithBase64Shade,
expandedConfigurationMap);
+ }
+
+ private final Map<String, String> decryptedServiceConfigWithDefaultShade =
+ ImmutableMap.<String, String>builder()
+ .put("admin-username", "admin")
+ .put("admin-password", "admin")
+ .put("server-bind-host", "0.0.0.0")
+ .put("server-expose-host", "127.0.0.1")
+ .put("shade.identifier", "default")
+ .put("database.type", "mysql")
+ .put("database.jdbc-driver-class", "com.mysql.cj.jdbc.Driver")
+ .put(
+ "database.url",
+
"jdbc:mysql://127.0.0.1:3306/amoro?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false")
+ .put("database.username", "root")
+ .put("database.password", "password")
+ .build();
+
+ private final Map<String, String> decryptedServiceConfigWithBase64Shade =
+ ImmutableMap.<String, String>builder()
+ .put("admin-username", "admin")
+ .put("admin-password", "admin")
+ .put("server-bind-host", "0.0.0.0")
+ .put("server-expose-host", "127.0.0.1")
+ .put("shade.identifier", "base64")
+ .put("shade.sensitive-keywords", "admin-password;database.password")
+ .put("database.type", "mysql")
+ .put("database.jdbc-driver-class", "com.mysql.cj.jdbc.Driver")
+ .put(
+ "database.url",
+
"jdbc:mysql://127.0.0.1:3306/amoro?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false")
+ .put("database.username", "root")
+ .put("database.password", "password")
+ .build();
+}
diff --git a/amoro-ams/src/test/resources/configs/config-base64-shade.yaml
b/amoro-ams/src/test/resources/configs/config-base64-shade.yaml
new file mode 100644
index 000000000..9338ecbb9
--- /dev/null
+++ b/amoro-ams/src/test/resources/configs/config-base64-shade.yaml
@@ -0,0 +1,38 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+ams:
+ admin-username: admin
+ admin-password: YWRtaW4=
+ server-bind-host: "0.0.0.0"
+ server-expose-host: "127.0.0.1"
+
+ shade:
+ identifier: base64
+ sensitive-keywords: admin-password;database.password
+
+ database:
+ type: mysql
+ jdbc-driver-class: com.mysql.cj.jdbc.Driver
+ url:
jdbc:mysql://127.0.0.1:3306/amoro?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false
+ username: root
+ password: cGFzc3dvcmQ=
+
+containers:
+ - name: localContainer
+ container-impl: org.apache.amoro.server.manager.LocalOptimizerContainer
\ No newline at end of file
diff --git a/amoro-ams/src/test/resources/configs/config-default-shade.yaml
b/amoro-ams/src/test/resources/configs/config-default-shade.yaml
new file mode 100644
index 000000000..c512b891f
--- /dev/null
+++ b/amoro-ams/src/test/resources/configs/config-default-shade.yaml
@@ -0,0 +1,37 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+ams:
+ admin-username: admin
+ admin-password: admin
+ server-bind-host: "0.0.0.0"
+ server-expose-host: "127.0.0.1"
+
+ shade:
+ identifier: default
+
+ database:
+ type: mysql
+ jdbc-driver-class: com.mysql.cj.jdbc.Driver
+ url:
jdbc:mysql://127.0.0.1:3306/amoro?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false
+ username: root
+ password: password
+
+containers:
+ - name: localContainer
+ container-impl: org.apache.amoro.server.manager.LocalOptimizerContainer
\ No newline at end of file
diff --git
a/amoro-common/src/main/java/org/apache/amoro/config/shade/ConfigShade.java
b/amoro-common/src/main/java/org/apache/amoro/config/shade/ConfigShade.java
new file mode 100644
index 000000000..07d223ebb
--- /dev/null
+++ b/amoro-common/src/main/java/org/apache/amoro/config/shade/ConfigShade.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.config.shade;
+
+import org.apache.amoro.config.Configurations;
+
+/**
+ * The interface that provides the ability to decrypt {@link
+ * org.apache.amoro.config.Configurations}.
+ */
+public interface ConfigShade {
+ /**
+ * Initializes the custom instance using the AMS configuration.
+ *
+ * <p>This method can be useful when decryption requires an external file
(e.g. a key file)
+ * defined in the service configs.
+ */
+ default void initialize(Configurations serviceConfig) throws Exception {}
+
+ /**
+ * The unique identifier of the current interface, used it to select the
expected {@link
+ * ConfigShade}.
+ */
+ String getIdentifier();
+
+ /**
+ * Decrypt the content.
+ *
+ * @param content The content to decrypt
+ */
+ String decrypt(String content);
+}
diff --git
a/amoro-common/src/main/java/org/apache/amoro/config/shade/impl/Base64ConfigShade.java
b/amoro-common/src/main/java/org/apache/amoro/config/shade/impl/Base64ConfigShade.java
new file mode 100644
index 000000000..0446e626c
--- /dev/null
+++
b/amoro-common/src/main/java/org/apache/amoro/config/shade/impl/Base64ConfigShade.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.config.shade.impl;
+
+import org.apache.amoro.config.shade.ConfigShade;
+
+import java.util.Base64;
+
+/** Base64 ConfigShade. */
+public class Base64ConfigShade implements ConfigShade {
+
+ private static final Base64.Decoder DECODER = Base64.getDecoder();
+
+ private static final String IDENTIFIER = "base64";
+
+ @Override
+ public String getIdentifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public String decrypt(String content) {
+ return new String(DECODER.decode(content));
+ }
+}
diff --git
a/amoro-common/src/main/java/org/apache/amoro/config/shade/utils/ConfigShadeUtils.java
b/amoro-common/src/main/java/org/apache/amoro/config/shade/utils/ConfigShadeUtils.java
new file mode 100644
index 000000000..53ef1d740
--- /dev/null
+++
b/amoro-common/src/main/java/org/apache/amoro/config/shade/utils/ConfigShadeUtils.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.config.shade.utils;
+
+import org.apache.amoro.config.ConfigOption;
+import org.apache.amoro.config.ConfigOptions;
+import org.apache.amoro.config.Configurations;
+import org.apache.amoro.config.shade.ConfigShade;
+import
org.apache.amoro.shade.guava32.com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.function.BiFunction;
+
+/** Config shade utilities. */
+public final class ConfigShadeUtils {
+ private static final Logger LOG =
LoggerFactory.getLogger(ConfigShadeUtils.class);
+
+ private static final Map<String, ConfigShade> CONFIG_SHADES = new
HashMap<>();
+
+ private static final ConfigShade DEFAULT_SHADE = new DefaultConfigShade();
+
+ public static final ConfigOption<String> SHADE_IDENTIFIER_OPTION =
+ ConfigOptions.key("shade.identifier")
+ .stringType()
+ .defaultValue("default")
+ .withDescription(
+ "The identifier of the encryption method for decryption.
Defaults to \"default\", indicating no encryption");
+
+ public static final ConfigOption<List<String>> SHADE_SENSITIVE_KEYWORDS =
+ ConfigOptions.key("shade.sensitive-keywords")
+ .stringType()
+ .asList()
+ .defaultValues("admin-password", "database.password")
+ .withDescription(
+ "A semicolon-separated list of keywords for the configuration
items to be decrypted.");
+
+ static {
+ ServiceLoader<ConfigShade> serviceLoader =
ServiceLoader.load(ConfigShade.class);
+ Iterator<ConfigShade> it = serviceLoader.iterator();
+ it.forEachRemaining(
+ configShade -> {
+ CONFIG_SHADES.put(configShade.getIdentifier(), configShade);
+ LOG.info(
+ "Load config shade spi [{}] from {}",
+ configShade.getIdentifier(),
+ configShade.getClass());
+ });
+ }
+
+ @VisibleForTesting
+ public static String decryptOption(String identifier, String content) {
+ ConfigShade configShade = CONFIG_SHADES.getOrDefault(identifier,
DEFAULT_SHADE);
+ return configShade.decrypt(content);
+ }
+
+ public static Map<String, Object> decryptConfig(Map<String, Object>
configMap) throws Exception {
+ Configurations serviceConfig = Configurations.fromObjectMap(configMap);
+ String identifier = serviceConfig.get(SHADE_IDENTIFIER_OPTION);
+ List<String> sensitiveOptions =
serviceConfig.get(SHADE_SENSITIVE_KEYWORDS);
+ return decryptConfig(identifier, configMap, sensitiveOptions,
serviceConfig);
+ }
+
+ public static Map<String, Object> decryptConfig(
+ String identifier,
+ Map<String, Object> configMap,
+ List<String> sensitiveOptions,
+ Configurations serviceConfig)
+ throws Exception {
+ ConfigShade configShade = CONFIG_SHADES.get(identifier);
+ if (configShade == null) {
+ LOG.error("Can not find config shade: {}", identifier);
+ throw new IllegalStateException("Can not find config shade: " +
identifier);
+ }
+ configShade.initialize(serviceConfig);
+
+ if (DEFAULT_SHADE.getIdentifier().equals(configShade.getIdentifier())) {
+ return configMap;
+ }
+
+ LOG.info("Use config shade: {}", identifier);
+ BiFunction<String, Object, String> processFunction =
+ (key, value) -> configShade.decrypt(value.toString());
+
+ for (String sensitiveOption : sensitiveOptions) {
+ try {
+ configMap.computeIfPresent(sensitiveOption, processFunction);
+ } catch (Exception e) {
+ LOG.error("Failed to decrypt sensitive option {}:", sensitiveOption,
e);
+ throw e;
+ }
+ }
+ LOG.info(
+ "Sensitive option{} {} {} been decrypted and refreshed",
+ sensitiveOptions.size() > 1 ? "s" : "",
+ sensitiveOptions,
+ sensitiveOptions.size() > 1 ? "have" : "has");
+
+ return configMap;
+ }
+
+ /** Default ConfigShade. */
+ public static class DefaultConfigShade implements ConfigShade {
+ private static final String IDENTIFIER = "default";
+
+ @Override
+ public String getIdentifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public String decrypt(String content) {
+ return content;
+ }
+ }
+}
diff --git
a/amoro-common/src/main/resources/META-INF/services/org.apache.amoro.config.shade.ConfigShade
b/amoro-common/src/main/resources/META-INF/services/org.apache.amoro.config.shade.ConfigShade
new file mode 100644
index 000000000..8d3697dc0
--- /dev/null
+++
b/amoro-common/src/main/resources/META-INF/services/org.apache.amoro.config.shade.ConfigShade
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+# *
+# http://www.apache.org/licenses/LICENSE-2.0
+# *
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.amoro.config.shade.utils.ConfigShadeUtils$DefaultConfigShade
+org.apache.amoro.config.shade.impl.Base64ConfigShade
\ No newline at end of file
diff --git a/charts/amoro/templates/amoro-configmap.yaml
b/charts/amoro/templates/amoro-configmap.yaml
index bf06c4038..ec4cb9c83 100644
--- a/charts/amoro/templates/amoro-configmap.yaml
+++ b/charts/amoro/templates/amoro-configmap.yaml
@@ -40,7 +40,18 @@ data:
ams:
server-bind-host: "0.0.0.0"
server-expose-host: "127.0.0.1"
-
+
+ {{- if or .Values.amoroConf.shade.identifier
.Values.amoroConf.shade.sensitiveKeywords}}
+
+ shade:
+ {{- if .Values.amoroConf.shade.identifier }}
+ identifier: {{ .Values.amoroConf.shade.identifier }}
+ {{- end }}
+ {{- if .Values.amoroConf.shade.sensitiveKeywords }}
+ sensitive-keywords: {{ .Values.amoroConf.shade.sensitiveKeywords }}
+ {{- end }}
+ {{- end }}
+
thrift-server:
max-message-size: 104857600 # 100MB
selector-thread-count: 2
diff --git a/charts/amoro/tests/amoro-configmap_test.yaml
b/charts/amoro/tests/amoro-configmap_test.yaml
index 73a5fe97c..66d674fc8 100644
--- a/charts/amoro/tests/amoro-configmap_test.yaml
+++ b/charts/amoro/tests/amoro-configmap_test.yaml
@@ -73,6 +73,34 @@ tests:
pattern:
|
kyuubi.jdbc.url: "jdbc:hive2://127.0.0.1:10009/"
+ - it: Amoro configMap should show shade configs if shade.identifier is set
+ set:
+ amoroConf:
+ shade:
+ identifier: base64
+ asserts:
+ - matchRegex:
+ path: data["config.yaml"]
+ pattern: "shade:\\s*identifier: base64"
+ - it: Amoro configMap should show shade configs if shade.sensitiveKeywords
is set
+ set:
+ amoroConf:
+ shade:
+ sensitiveKeywords: admin-password;database.password
+ asserts:
+ - matchRegex:
+ path: data["config.yaml"]
+ pattern: "shade:\\s*sensitive-keywords:
admin-password;database.password"
+ - it: Amoro configMap should show shade configs if both shade.identifier and
shade.sensitiveKeywords are set
+ set:
+ amoroConf:
+ shade:
+ identifier: base64
+ sensitiveKeywords: admin-password;database.password
+ asserts:
+ - matchRegex:
+ path: data["config.yaml"]
+ pattern: "shade:\\s*identifier: base64\\s*sensitive-keywords:
admin-password;database.password"
- it: Amoro configMap should show flink if flink optimizer enabled is set
true
set:
optimizer:
diff --git a/charts/amoro/values.yaml b/charts/amoro/values.yaml
index bcb058565..312fefaea 100644
--- a/charts/amoro/values.yaml
+++ b/charts/amoro/values.yaml
@@ -144,6 +144,12 @@ amoroConf:
adminUsername: admin
adminPassword: admin
+ ## AMS config shade properties
+ ##
+ shade:
+ identifier: ~
+ sensitiveKeywords: ~
+
## AMS database properties, default value is derby. For production
environment, suggest to use mysql
##
database:
diff --git a/dist/src/main/amoro-bin/conf/config.yaml
b/dist/src/main/amoro-bin/conf/config.yaml
index 0e8c6f5cb..6f3f4849d 100644
--- a/dist/src/main/amoro-bin/conf/config.yaml
+++ b/dist/src/main/amoro-bin/conf/config.yaml
@@ -94,6 +94,11 @@ ams:
catalog-meta-cache:
expiration-interval: 60s
+ # Support for encrypted sensitive configuration items
+ shade:
+ identifier: default # Built-in support for default/base64. Defaults to
"default", indicating no encryption
+ sensitive-keywords: admin-password;database.password
+
database:
type: derby
jdbc-driver-class: org.apache.derby.jdbc.EmbeddedDriver
