This is an automated email from the ASF dual-hosted git repository. rgoers pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/flume.git
The following commit(s) were added to refs/heads/trunk by this push: new e98013ff FLUME-3433 - Limit the Lookups used by MapResolver e98013ff is described below commit e98013ffeca50ddd912a76d0ae94b0462152517a Author: Ralph Goers <rgo...@apache.org> AuthorDate: Mon Aug 8 23:09:02 2022 -0700 FLUME-3433 - Limit the Lookups used by MapResolver --- .../java/org/apache/flume/node/MapResolver.java | 97 +++++++++++++++++++++- .../org/apache/flume/node/TestMapResolver.java | 83 ++++++++++++++++++ .../org/apache/flume/node/lookup/TestLookup.java | 29 +++++++ .../src/test/resources/map-resolver.properties | 4 + .../src/test/resources/test-lookups.properties | 4 + 5 files changed, 216 insertions(+), 1 deletion(-) diff --git a/flume-ng-node/src/main/java/org/apache/flume/node/MapResolver.java b/flume-ng-node/src/main/java/org/apache/flume/node/MapResolver.java index f7571b90..ea933ec0 100644 --- a/flume-ng-node/src/main/java/org/apache/flume/node/MapResolver.java +++ b/flume-ng-node/src/main/java/org/apache/flume/node/MapResolver.java @@ -17,13 +17,22 @@ */ package org.apache.flume.node; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; import java.util.HashMap; +import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Properties; import org.apache.commons.text.StringSubstitutor; +import org.apache.commons.text.lookup.DefaultStringLookup; import org.apache.commons.text.lookup.StringLookup; import org.apache.commons.text.lookup.StringLookupFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Resolves replaceable tokens to create a Map. @@ -32,15 +41,26 @@ import org.apache.commons.text.lookup.StringLookupFactory; */ final class MapResolver { + private static final Logger LOGGER = LoggerFactory.getLogger(MapResolver.class); + private static final String DEFAULT_LOOKUPS = "lookups.properties"; + private static final String CUSTOM_LOOKUPS_KEY = "lookups"; private static final String PROPS_IMPL_KEY = "propertiesImplementation"; private static final String ENV_VAR_PROPERTY = "org.apache.flume.node.EnvVarResolverProperties"; + private static final String LOOKUP = "org.apache.commons.text.lookup.DefaultStringLookup."; + private static final LookupEntry[] LOOKUP_ENTRIES = { + new LookupEntry("sys", DefaultStringLookup.SYSTEM_PROPERTIES.getStringLookup()), + new LookupEntry("env", DefaultStringLookup.ENVIRONMENT.getStringLookup()) , + new LookupEntry("java", DefaultStringLookup.JAVA.getStringLookup()), + new LookupEntry("date", DefaultStringLookup.DATE.getStringLookup()) + }; public static Map<String, String> resolveProperties(Properties properties) { Map<String, String> map = new HashMap<>(); boolean useEnvVars = ENV_VAR_PROPERTY.equals(System.getProperty(PROPS_IMPL_KEY)); StringLookup defaultLookup = useEnvVars ? new DefaultLookup(map) : StringLookupFactory.INSTANCE.mapStringLookup(map); - StringLookup lookup = StringLookupFactory.INSTANCE.interpolatorStringLookup(defaultLookup); + StringLookup lookup = StringLookupFactory.INSTANCE.interpolatorStringLookup(createLookupMap(), + defaultLookup, false); StringSubstitutor substitutor = new StringSubstitutor(lookup); substitutor.setEnableSubstitutionInVariables(true); properties.stringPropertyNames().forEach((k) -> map.put(k, @@ -48,6 +68,43 @@ final class MapResolver { return map; } + private static Map<String, StringLookup> createLookupMap() { + Map<String, StringLookup> map = new HashMap<>(); + Properties properties = loadProperties(); + if (properties == null) { + Arrays.stream(LOOKUP_ENTRIES).forEach((e) -> { + map.put(e.key, e.lookup); + }); + } else { + properties.forEach((k, v) -> { + String key = Objects.toString(k); + String value = Objects.toString(v); + if (value.startsWith(LOOKUP)) { + String lookupEnum = value.substring(LOOKUP.length()); + try { + StringLookup stringLookup = DefaultStringLookup.valueOf(lookupEnum).getStringLookup(); + map.put(key.toLowerCase(Locale.ROOT), stringLookup); + } catch (IllegalArgumentException ex) { + LOGGER.warn("{} is not a DefaultStringLookup enum value, ignoring", key); + } + } else { + try { + Class<?> clazz = Class.forName(Objects.toString(v)); + if (StringLookup.class.isAssignableFrom(clazz)) { + StringLookup stringLookup = (StringLookup) clazz.newInstance(); + map.put(k.toString().toLowerCase(Locale.ROOT), stringLookup); + } else { + LOGGER.warn("{} is not a StringLookup, ignoring", v); + } + } catch (Exception ex) { + LOGGER.warn("Unable to load {} due to {}, ignoring", v, ex.getMessage()); + } + } + }); + } + return map; + } + private static class DefaultLookup implements StringLookup { private final Map<String, String> properties; @@ -67,4 +124,42 @@ final class MapResolver { properties.get(key) : System.getenv(key); } } + + private static class LookupEntry { + private final String key; + private final StringLookup lookup; + + public LookupEntry(String key, StringLookup lookup) { + this.key = key; + this.lookup = lookup; + } + } + + private static Properties loadProperties() { + final Properties properties = new Properties(); + String fileName = System.getProperty(CUSTOM_LOOKUPS_KEY); + if (fileName != null) { + try (InputStream inputStream = new FileInputStream(fileName)) { + properties.load(inputStream); + } catch (final IOException e) { + try (InputStream inputStream = ClassLoader.getSystemResourceAsStream(fileName)) { + properties.load(inputStream); + } catch (final IOException ex) { + LOGGER.warn("Unable to load {} due to {}", fileName, ex.getMessage()); + } + } + } + if (properties.size() == 0) { + try (InputStream inputStream = ClassLoader.getSystemResourceAsStream(DEFAULT_LOOKUPS)) { + if (inputStream != null) { + properties.load(inputStream); + } else { + return null; + } + } catch (final IOException e) { + return null; + } + } + return properties; + } } diff --git a/flume-ng-node/src/test/java/org/apache/flume/node/TestMapResolver.java b/flume-ng-node/src/test/java/org/apache/flume/node/TestMapResolver.java new file mode 100644 index 00000000..9ba57ed8 --- /dev/null +++ b/flume-ng-node/src/test/java/org/apache/flume/node/TestMapResolver.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.apache.flume.node; + +import java.io.FileInputStream; +import java.util.Map; +import java.util.Properties; + +import org.junit.After; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Tests the MapResolver. + */ +public class TestMapResolver { + + public static final String TEST_CONST = "Apache Flume"; + private static final String TEST_PROPS = "target/test-classes/map-resolver.properties"; + private static final String NAME_VALUE = "FLUME"; + + @After + public void after() { + System.clearProperty("lookups"); + } + + @Test + public void testDefaultResolver() throws Exception { + Properties props = new Properties(); + props.load(new FileInputStream(TEST_PROPS)); + System.setProperty("name", NAME_VALUE); + Map<String, String> properties = MapResolver.resolveProperties(props); + String name = properties.get("name"); + assertNotNull("No name property", name); + assertEquals("Incorrect system property resolution", NAME_VALUE, name); + String testStr = properties.get("const"); + assertNotNull("No const property", testStr); + assertTrue("Constant was resolved", testStr.contains("${const:")); + String version = properties.get("version"); + assertNotNull("No Java property", version); + assertFalse("Java lookup was not resolved", version.contains("${java:")); + } + + @Test + public void testCustomResolver() throws Exception { + Properties props = new Properties(); + props.load(new FileInputStream(TEST_PROPS)); + System.setProperty("name", NAME_VALUE); + System.setProperty("lookups", "test-lookups.properties"); + Map<String, String> properties = MapResolver.resolveProperties(props); + String name = properties.get("name"); + assertNotNull("No name property", name); + assertEquals("Incorrect system property resolution", NAME_VALUE, name); + String testStr = properties.get("const"); + assertNotNull("No const property", testStr); + assertTrue("Constant was resolved", testStr.contains("${const:")); + String version = properties.get("version"); + assertNotNull("No Java property", version); + assertFalse("Java lookup was not resolved", version.contains("${java:")); + String test = properties.get("test"); + assertNotNull("No Test property", version); + assertEquals("Test lookup was not resolved", "Value", test); + } + +} diff --git a/flume-ng-node/src/test/java/org/apache/flume/node/lookup/TestLookup.java b/flume-ng-node/src/test/java/org/apache/flume/node/lookup/TestLookup.java new file mode 100644 index 00000000..42a004a8 --- /dev/null +++ b/flume-ng-node/src/test/java/org/apache/flume/node/lookup/TestLookup.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.apache.flume.node.lookup; + +import org.apache.commons.text.lookup.StringLookup; + +/** + * Test Lookup. + */ +public class TestLookup implements StringLookup { + @Override + public String lookup(String key) { + return key; + } +} diff --git a/flume-ng-node/src/test/resources/map-resolver.properties b/flume-ng-node/src/test/resources/map-resolver.properties new file mode 100644 index 00000000..a856b074 --- /dev/null +++ b/flume-ng-node/src/test/resources/map-resolver.properties @@ -0,0 +1,4 @@ +name = ${sys:name} +const = ${const:org.apache.flume.node.TestMapResolver.TEST_CONST} +version = ${java:version} +test = ${test:Value} \ No newline at end of file diff --git a/flume-ng-node/src/test/resources/test-lookups.properties b/flume-ng-node/src/test/resources/test-lookups.properties new file mode 100644 index 00000000..b8ba3a61 --- /dev/null +++ b/flume-ng-node/src/test/resources/test-lookups.properties @@ -0,0 +1,4 @@ +sys = org.apache.commons.text.lookup.DefaultStringLookup.SYSTEM_PROPERTIES +env = org.apache.commons.text.lookup.DefaultStringLookup.ENVIRONMENT +java = org.apache.commons.text.lookup.DefaultStringLookup.JAVA +test = org.apache.flume.node.lookup.TestLookup \ No newline at end of file