Repository: flume Updated Branches: refs/heads/trunk 2ff2dbbd1 -> 5e52ac4ad
FLUME-2993. Add support for environment variables in configuration files Flume does not currently support environment variable interpolation in the properties file configuration. Enabling it helps with: * removing security credentials from config files * copy-pastes in configuration files when defining multiple agents This closes #97 Reviewers: Lior Zeno, Jeff Holoman, Shang Wu (Bessenyei Balázs Donát via Bessenyei Balázs Donát) Project: http://git-wip-us.apache.org/repos/asf/flume/repo Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/5e52ac4a Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/5e52ac4a Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/5e52ac4a Branch: refs/heads/trunk Commit: 5e52ac4ad18a7776f4c74499c1314fbfd3513160 Parents: 2ff2dbb Author: Bessenyei Balázs Donát <[email protected]> Authored: Fri Jan 6 18:32:50 2017 +0000 Committer: Bessenyei Balázs Donát <[email protected]> Committed: Fri Jan 6 18:37:14 2017 +0000 ---------------------------------------------------------------------- flume-ng-doc/sphinx/FlumeUserGuide.rst | 19 ++++++ flume-ng-node/pom.xml | 7 ++ .../flume/node/EnvVarResolverProperties.java | 60 +++++++++++++++++ .../PropertiesFileConfigurationProvider.java | 13 +++- .../node/TestEnvVarResolverProperties.java | 69 ++++++++++++++++++++ .../flume-conf-with-envvars.properties | 35 ++++++++++ 6 files changed, 202 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-doc/sphinx/FlumeUserGuide.rst ---------------------------------------------------------------------- diff --git a/flume-ng-doc/sphinx/FlumeUserGuide.rst b/flume-ng-doc/sphinx/FlumeUserGuide.rst index 3c316c6..6bc9dba 100644 --- a/flume-ng-doc/sphinx/FlumeUserGuide.rst +++ b/flume-ng-doc/sphinx/FlumeUserGuide.rst @@ -234,6 +234,25 @@ The original Flume terminal will output the event in a log message. Congratulations - you've successfully configured and deployed a Flume agent! Subsequent sections cover agent configuration in much more detail. +Using environment variables in configuration files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Flume has the ability to substitute environment variables in the configuration. For example:: + + a1.sources = r1 + a1.sources.r1.type = netcat + a1.sources.r1.bind = 0.0.0.0 + a1.sources.r1.port = ${NC_PORT} + a1.sources.r1.channels = c1 + +NB: it currently works for values only, not for keys. (Ie. only on the "right side" of the `=` mark of the config lines.) + +This can be enabled via Java system properties on agent invocation by setting `propertiesImplementation = org.apache.flume.node.EnvVarResolverProperties`. + +For example:: + $ NC_PORT=44444 bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger=INFO,console -DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties + +Note the above is just an example, environment variables can be configured in other ways, including being set in `conf/flume-env.sh`. + Logging raw data ~~~~~~~~~~~~~~~~ http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/pom.xml ---------------------------------------------------------------------- diff --git a/flume-ng-node/pom.xml b/flume-ng-node/pom.xml index 6d1067d..f4fb958 100644 --- a/flume-ng-node/pom.xml +++ b/flume-ng-node/pom.xml @@ -145,6 +145,13 @@ </dependency> <dependency> + <groupId>com.github.stefanbirkner</groupId> + <artifactId>system-rules</artifactId> + <version>1.16.0</version> + <scope>test</scope> + </dependency> + + <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-core-asl</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java ---------------------------------------------------------------------- diff --git a/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java b/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java new file mode 100644 index 0000000..e0b0d22 --- /dev/null +++ b/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flume.node; + +import com.google.common.base.Preconditions; + +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * A class that extends the Java built-in Properties overriding + * {@link java.util.Properties#getProperty(String)} to allow ${ENV_VAR_NAME}-style environment + * variable inclusions + */ +public class EnvVarResolverProperties extends Properties { + /** + * @param input The input string with ${ENV_VAR_NAME}-style environment variable names + * @return The output string with ${ENV_VAR_NAME} replaced with their environment variable values + */ + protected static String resolveEnvVars(String input) { + Preconditions.checkNotNull(input); + // match ${ENV_VAR_NAME} + Pattern p = Pattern.compile("\\$\\{(\\w+)\\}"); + Matcher m = p.matcher(input); + StringBuffer sb = new StringBuffer(); + while (m.find()) { + String envVarName = m.group(1); + String envVarValue = System.getenv(envVarName); + m.appendReplacement(sb, null == envVarValue ? "" : envVarValue); + } + m.appendTail(sb); + return sb.toString(); + } + + /** + * @param key the property key + * @return the value of the property key with ${ENV_VAR_NAME}-style environment variables replaced + */ + @Override + public String getProperty(String key) { + return resolveEnvVars(super.getProperty(key)); + } +} http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java b/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java index b428c9e..75f45db 100644 --- a/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java +++ b/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java @@ -171,6 +171,7 @@ public class PropertiesFileConfigurationProvider extends private static final Logger LOGGER = LoggerFactory .getLogger(PropertiesFileConfigurationProvider.class); + private static final String DEFAULT_PROPERTIES_IMPLEMENTATION = "java.util.Properties"; private final File file; @@ -184,12 +185,22 @@ public class PropertiesFileConfigurationProvider extends BufferedReader reader = null; try { reader = new BufferedReader(new FileReader(file)); - Properties properties = new Properties(); + String resolverClassName = System.getProperty("propertiesImplementation", + DEFAULT_PROPERTIES_IMPLEMENTATION); + Class<? extends Properties> propsclass = Class.forName(resolverClassName) + .asSubclass(Properties.class); + Properties properties = propsclass.newInstance(); properties.load(reader); return new FlumeConfiguration(toMap(properties)); } catch (IOException ex) { LOGGER.error("Unable to load file:" + file + " (I/O failure) - Exception follows.", ex); + } catch (ClassNotFoundException e) { + LOGGER.error("Configuration resolver class not found", e); + } catch (InstantiationException e) { + LOGGER.error("Instantiation exception", e); + } catch (IllegalAccessException e) { + LOGGER.error("Illegal access exception", e); } finally { if (reader != null) { try { http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java ---------------------------------------------------------------------- diff --git a/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java b/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java new file mode 100644 index 0000000..27aeef1 --- /dev/null +++ b/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flume.node; + +import junit.framework.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.contrib.java.lang.system.EnvironmentVariables; + +import java.io.File; + +public class TestEnvVarResolverProperties { + private static final File TESTFILE = new File( + TestEnvVarResolverProperties.class.getClassLoader() + .getResource("flume-conf-with-envvars.properties").getFile()); + + @Rule + public final EnvironmentVariables environmentVariables = new EnvironmentVariables(); + private PropertiesFileConfigurationProvider provider; + + @Before + public void setUp() throws Exception { + provider = new PropertiesFileConfigurationProvider("a1", TESTFILE); + } + + @Test + public void resolveEnvVar() throws Exception { + environmentVariables.set("VARNAME", "varvalue"); + String resolved = EnvVarResolverProperties.resolveEnvVars("padding ${VARNAME} padding"); + Assert.assertEquals("padding varvalue padding", resolved); + } + + @Test + public void resolveEnvVars() throws Exception { + environmentVariables.set("VARNAME1", "varvalue1"); + environmentVariables.set("VARNAME2", "varvalue2"); + String resolved = EnvVarResolverProperties + .resolveEnvVars("padding ${VARNAME1} ${VARNAME2} padding"); + Assert.assertEquals("padding varvalue1 varvalue2 padding", resolved); + } + + @Test + public void getProperty() throws Exception { + String NC_PORT = "6667"; + environmentVariables.set("NC_PORT", NC_PORT); + System.setProperty("propertiesImplementation", + "org.apache.flume.node.EnvVarResolverProperties"); + + Assert.assertEquals(NC_PORT, provider.getFlumeConfiguration() + .getConfigurationFor("a1") + .getSourceContext().get("r1").getParameters().get("port")); + } +} http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties ---------------------------------------------------------------------- diff --git a/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties b/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties new file mode 100644 index 0000000..536eb02 --- /dev/null +++ b/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +a1.sources = r1 +a1.sources.r1.type = netcat +a1.sources.r1.bind = 0.0.0.0 +a1.sources.r1.port = ${NC_PORT} +a1.sources.r1.channels = c1 + +a1.channels = c1 +a1.channels.c1.type = memory +a1.channels.c1.capacity = 10000 +a1.channels.c1.transactionCapacity = 10000 +a1.channels.c1.byteCapacityBufferPercentage = 20 +a1.channels.c1.byteCapacity = 800000 + +a1.channels = c1 +a1.sinks = k1 +a1.sinks.k1.type = logger +a1.sinks.k1.channel = c1
