This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 570ab71c9 IMPALA-14287: Resolve environment variables in REST server
configurations
570ab71c9 is described below
commit 570ab71c9d91f034ad8dc59d48b482a52b28fb50
Author: Peter Rozsa <[email protected]>
AuthorDate: Wed Sep 24 13:42:46 2025 +0200
IMPALA-14287: Resolve environment variables in REST server configurations
This change adds a step to REST server configuration loading that
resolves environment variables noted as ${ENV:VARIABLE_NAME} format. If
the environment variable is not set, then the reference text remains the
same and Impala logs an error.
Tests:
- unit tests added
Change-Id: I3faccc15d012c389703c58371a4d38cca82bef60
Reviewed-on: http://gerrit.cloudera.org:8080/23457
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
.../service/catalogmanager/ConfigLoader.java | 44 ++++++++++
.../service/catalogmanager/ConfigLoaderTest.java | 96 +++++++++++++++++++++-
2 files changed, 139 insertions(+), 1 deletion(-)
diff --git
a/fe/src/main/java/org/apache/impala/service/catalogmanager/ConfigLoader.java
b/fe/src/main/java/org/apache/impala/service/catalogmanager/ConfigLoader.java
index 0a7bf1283..bf72e7366 100644
---
a/fe/src/main/java/org/apache/impala/service/catalogmanager/ConfigLoader.java
+++
b/fe/src/main/java/org/apache/impala/service/catalogmanager/ConfigLoader.java
@@ -25,16 +25,29 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Properties;
+import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.impala.common.ImpalaRuntimeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ConfigLoader {
+ private static final Logger LOG =
LoggerFactory.getLogger(ConfigLoader.class);
+
private final File configFolder_;
+ private final UnaryOperator<String> envVarResolver_;
public ConfigLoader(File configFolder) {
+ this(configFolder, System::getenv);
+ }
+
+ public ConfigLoader(File configFolder, UnaryOperator<String> envVarResolver)
{
this.configFolder_ = configFolder;
+ this.envVarResolver_ = envVarResolver;
}
public List<Properties> loadConfigs() throws ImpalaRuntimeException {
@@ -68,9 +81,40 @@ public class ConfigLoader {
try (InputStream in = Files.newInputStream(file.toPath())) {
props.load(in);
}
+ this.resolveEnvVars(props);
return props;
}
+ private void resolveEnvVars(Properties props) {
+ for (String key : props.stringPropertyNames()) {
+ String value = props.getProperty(key);
+ String resolved = resolveEnvVar(value);
+ props.setProperty(key, resolved);
+ }
+ }
+
+ private String resolveEnvVar(String input) {
+ if (input == null || input.isEmpty()) return input;
+ Pattern pattern = Pattern.compile("\\$\\{ENV:([a-zA-Z][a-zA-Z0-9_-]*)}");
+ Matcher matcher = pattern.matcher(input);
+ StringBuffer sb = new StringBuffer();
+ while (matcher.find()) {
+ String envVarName = matcher.group(1);
+ String envVarValue = this.envVarResolver_.apply(envVarName);
+ String replacement;
+ if (envVarValue == null) {
+ LOG.error("Unable to resolve environment variable: '{}'", envVarName);
+ replacement = matcher.group(0);
+ }
+ else {
+ replacement = envVarValue;
+ }
+ matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement));
+ }
+ matcher.appendTail(sb);
+ return sb.toString();
+ }
+
private void checkPropertyValue(String configFile, Properties props, String
key,
String expectedValue) {
if (!props.containsKey(key)) {
diff --git
a/fe/src/test/java/org/apache/impala/service/catalogmanager/ConfigLoaderTest.java
b/fe/src/test/java/org/apache/impala/service/catalogmanager/ConfigLoaderTest.java
index a1ef153b8..326940c98 100644
---
a/fe/src/test/java/org/apache/impala/service/catalogmanager/ConfigLoaderTest.java
+++
b/fe/src/test/java/org/apache/impala/service/catalogmanager/ConfigLoaderTest.java
@@ -18,6 +18,7 @@ package org.apache.impala.service.catalogmanager;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -124,4 +125,97 @@ public class ConfigLoaderTest {
List<Properties> configs = loader.loadConfigs();
assertTrue(configs.isEmpty());
}
-}
\ No newline at end of file
+
+ @Test
+ public void testEnvVarResolutionResolved() throws Exception {
+ String pathValue = System.getenv("PATH");
+ createConfigFile("resolved.properties",
+
"path=${ENV:PATH}\nconnector.name=iceberg\niceberg.catalog.type=rest\n");
+ ConfigLoader loader = new ConfigLoader(tempDir);
+ List<Properties> configs = loader.loadConfigs();
+ assertEquals(1, configs.size());
+ assertNotNull(pathValue);
+ assertEquals(pathValue, configs.get(0).getProperty("path"));
+ }
+
+ @Test
+ public void testEnvVarResolutionUnresolved() throws Exception {
+ createConfigFile("unresolved.properties",
+
"secret=${ENV:FAKE_SECRET}\nconnector.name=iceberg\niceberg.catalog.type=rest\n");
+ ConfigLoader loader = new ConfigLoader(tempDir);
+ List<Properties> configs = loader.loadConfigs();
+ assertEquals(1, configs.size());
+ assertEquals("${ENV:FAKE_SECRET}", configs.get(0).getProperty("secret"));
+ }
+
+ @Test
+ public void testEnvVarInvalidSyntax() throws Exception {
+ createConfigFile("invalid.properties",
+
"invalid=${SOME_VAR}\nconnector.name=iceberg\niceberg.catalog.type=rest\n");
+ ConfigLoader loader = new ConfigLoader(tempDir);
+ List<Properties> configs = loader.loadConfigs();
+ assertEquals(1, configs.size());
+ assertEquals("${SOME_VAR}", configs.get(0).getProperty("invalid"));
+ }
+
+ @Test
+ public void testMultipleEnvVars() throws Exception {
+ String pathValue = System.getenv("PATH");
+ createConfigFile("multiple.properties",
+ "path=${ENV:PATH}/dir ${ENV:FAKE}\n"
+ + "connector.name=iceberg\niceberg.catalog.type=rest\n");
+ ConfigLoader loader = new ConfigLoader(tempDir);
+ List<Properties> configs = loader.loadConfigs();
+ assertEquals(1, configs.size());
+ String resolvedPath = configs.get(0).getProperty("path");
+ String expected = pathValue + "/dir ${ENV:FAKE}";
+ assertEquals(expected, resolvedPath);
+ }
+
+ @Test
+ public void testMultipleValidEnvVars() throws Exception {
+ String pathValue = System.getenv("PATH");
+ String userValue = System.getenv("USER");
+ String shellValue = System.getenv("SHELL");
+
+ createConfigFile("multiple_env_vars_with_specific_names.properties",
+ "path=${ENV:PATH}\n"
+ + "user=${ENV:USER}\n"
+ + "shell=${ENV:SHELL}\n"
+ + "connector.name=iceberg\niceberg.catalog.type=rest\n");
+
+ ConfigLoader loader = new ConfigLoader(tempDir);
+ List<Properties> configs = loader.loadConfigs();
+
+ assertEquals(1, configs.size());
+ Properties config = configs.get(0);
+
+ assertNotNull(pathValue);
+ assertEquals(pathValue, config.getProperty("path"));
+
+ assertNotNull(userValue);
+ assertEquals(userValue, config.getProperty("user"));
+
+ assertNotNull(shellValue);
+ assertEquals(shellValue, config.getProperty("shell"));
+ }
+
+ @Test
+ public void testEnvVarWithNewLines() throws Exception {
+ createConfigFile("newline.properties",
+ "key=${ENV:CUSTOM_VAR}\n"
+ + "connector.name=iceberg\niceberg.catalog.type=rest\n");
+
+ ConfigLoader loader = new ConfigLoader(tempDir, envVar -> {
+ if (envVar.equals("CUSTOM_VAR")) {
+ return "VALUE\nWITH\nNEWLINE";
+ }
+ return null;
+ });
+
+ List<Properties> configs = loader.loadConfigs();
+ assertEquals(1, configs.size());
+ Properties config = configs.get(0);
+ assertEquals("VALUE\nWITH\nNEWLINE", config.getProperty("key"));
+ }
+}
