This is an automated email from the ASF dual-hosted git repository.
rgoers pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/flume.git
The following commit(s) were added to refs/heads/trunk by this push:
new e98013ff FLUME-3433 - Limit the Lookups used by MapResolver
e98013ff is described below
commit e98013ffeca50ddd912a76d0ae94b0462152517a
Author: Ralph Goers <[email protected]>
AuthorDate: Mon Aug 8 23:09:02 2022 -0700
FLUME-3433 - Limit the Lookups used by MapResolver
---
.../java/org/apache/flume/node/MapResolver.java | 97 +++++++++++++++++++++-
.../org/apache/flume/node/TestMapResolver.java | 83 ++++++++++++++++++
.../org/apache/flume/node/lookup/TestLookup.java | 29 +++++++
.../src/test/resources/map-resolver.properties | 4 +
.../src/test/resources/test-lookups.properties | 4 +
5 files changed, 216 insertions(+), 1 deletion(-)
diff --git a/flume-ng-node/src/main/java/org/apache/flume/node/MapResolver.java
b/flume-ng-node/src/main/java/org/apache/flume/node/MapResolver.java
index f7571b90..ea933ec0 100644
--- a/flume-ng-node/src/main/java/org/apache/flume/node/MapResolver.java
+++ b/flume-ng-node/src/main/java/org/apache/flume/node/MapResolver.java
@@ -17,13 +17,22 @@
*/
package org.apache.flume.node;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Properties;
import org.apache.commons.text.StringSubstitutor;
+import org.apache.commons.text.lookup.DefaultStringLookup;
import org.apache.commons.text.lookup.StringLookup;
import org.apache.commons.text.lookup.StringLookupFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Resolves replaceable tokens to create a Map.
@@ -32,15 +41,26 @@ import org.apache.commons.text.lookup.StringLookupFactory;
*/
final class MapResolver {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(MapResolver.class);
+ private static final String DEFAULT_LOOKUPS = "lookups.properties";
+ private static final String CUSTOM_LOOKUPS_KEY = "lookups";
private static final String PROPS_IMPL_KEY = "propertiesImplementation";
private static final String ENV_VAR_PROPERTY =
"org.apache.flume.node.EnvVarResolverProperties";
+ private static final String LOOKUP =
"org.apache.commons.text.lookup.DefaultStringLookup.";
+ private static final LookupEntry[] LOOKUP_ENTRIES = {
+ new LookupEntry("sys",
DefaultStringLookup.SYSTEM_PROPERTIES.getStringLookup()),
+ new LookupEntry("env", DefaultStringLookup.ENVIRONMENT.getStringLookup())
,
+ new LookupEntry("java", DefaultStringLookup.JAVA.getStringLookup()),
+ new LookupEntry("date", DefaultStringLookup.DATE.getStringLookup())
+ };
public static Map<String, String> resolveProperties(Properties properties) {
Map<String, String> map = new HashMap<>();
boolean useEnvVars =
ENV_VAR_PROPERTY.equals(System.getProperty(PROPS_IMPL_KEY));
StringLookup defaultLookup = useEnvVars ? new DefaultLookup(map) :
StringLookupFactory.INSTANCE.mapStringLookup(map);
- StringLookup lookup =
StringLookupFactory.INSTANCE.interpolatorStringLookup(defaultLookup);
+ StringLookup lookup =
StringLookupFactory.INSTANCE.interpolatorStringLookup(createLookupMap(),
+ defaultLookup, false);
StringSubstitutor substitutor = new StringSubstitutor(lookup);
substitutor.setEnableSubstitutionInVariables(true);
properties.stringPropertyNames().forEach((k) -> map.put(k,
@@ -48,6 +68,43 @@ final class MapResolver {
return map;
}
+ private static Map<String, StringLookup> createLookupMap() {
+ Map<String, StringLookup> map = new HashMap<>();
+ Properties properties = loadProperties();
+ if (properties == null) {
+ Arrays.stream(LOOKUP_ENTRIES).forEach((e) -> {
+ map.put(e.key, e.lookup);
+ });
+ } else {
+ properties.forEach((k, v) -> {
+ String key = Objects.toString(k);
+ String value = Objects.toString(v);
+ if (value.startsWith(LOOKUP)) {
+ String lookupEnum = value.substring(LOOKUP.length());
+ try {
+ StringLookup stringLookup =
DefaultStringLookup.valueOf(lookupEnum).getStringLookup();
+ map.put(key.toLowerCase(Locale.ROOT), stringLookup);
+ } catch (IllegalArgumentException ex) {
+ LOGGER.warn("{} is not a DefaultStringLookup enum value,
ignoring", key);
+ }
+ } else {
+ try {
+ Class<?> clazz = Class.forName(Objects.toString(v));
+ if (StringLookup.class.isAssignableFrom(clazz)) {
+ StringLookup stringLookup = (StringLookup) clazz.newInstance();
+ map.put(k.toString().toLowerCase(Locale.ROOT), stringLookup);
+ } else {
+ LOGGER.warn("{} is not a StringLookup, ignoring", v);
+ }
+ } catch (Exception ex) {
+ LOGGER.warn("Unable to load {} due to {}, ignoring", v,
ex.getMessage());
+ }
+ }
+ });
+ }
+ return map;
+ }
+
private static class DefaultLookup implements StringLookup {
private final Map<String, String> properties;
@@ -67,4 +124,42 @@ final class MapResolver {
properties.get(key) : System.getenv(key);
}
}
+
+ private static class LookupEntry {
+ private final String key;
+ private final StringLookup lookup;
+
+ public LookupEntry(String key, StringLookup lookup) {
+ this.key = key;
+ this.lookup = lookup;
+ }
+ }
+
+ private static Properties loadProperties() {
+ final Properties properties = new Properties();
+ String fileName = System.getProperty(CUSTOM_LOOKUPS_KEY);
+ if (fileName != null) {
+ try (InputStream inputStream = new FileInputStream(fileName)) {
+ properties.load(inputStream);
+ } catch (final IOException e) {
+ try (InputStream inputStream =
ClassLoader.getSystemResourceAsStream(fileName)) {
+ properties.load(inputStream);
+ } catch (final IOException ex) {
+ LOGGER.warn("Unable to load {} due to {}", fileName,
ex.getMessage());
+ }
+ }
+ }
+ if (properties.size() == 0) {
+ try (InputStream inputStream =
ClassLoader.getSystemResourceAsStream(DEFAULT_LOOKUPS)) {
+ if (inputStream != null) {
+ properties.load(inputStream);
+ } else {
+ return null;
+ }
+ } catch (final IOException e) {
+ return null;
+ }
+ }
+ return properties;
+ }
}
diff --git
a/flume-ng-node/src/test/java/org/apache/flume/node/TestMapResolver.java
b/flume-ng-node/src/test/java/org/apache/flume/node/TestMapResolver.java
new file mode 100644
index 00000000..9ba57ed8
--- /dev/null
+++ b/flume-ng-node/src/test/java/org/apache/flume/node/TestMapResolver.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+package org.apache.flume.node;
+
+import java.io.FileInputStream;
+import java.util.Map;
+import java.util.Properties;
+
+import org.junit.After;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests the MapResolver.
+ */
+public class TestMapResolver {
+
+ public static final String TEST_CONST = "Apache Flume";
+ private static final String TEST_PROPS =
"target/test-classes/map-resolver.properties";
+ private static final String NAME_VALUE = "FLUME";
+
+ @After
+ public void after() {
+ System.clearProperty("lookups");
+ }
+
+ @Test
+ public void testDefaultResolver() throws Exception {
+ Properties props = new Properties();
+ props.load(new FileInputStream(TEST_PROPS));
+ System.setProperty("name", NAME_VALUE);
+ Map<String, String> properties = MapResolver.resolveProperties(props);
+ String name = properties.get("name");
+ assertNotNull("No name property", name);
+ assertEquals("Incorrect system property resolution", NAME_VALUE, name);
+ String testStr = properties.get("const");
+ assertNotNull("No const property", testStr);
+ assertTrue("Constant was resolved", testStr.contains("${const:"));
+ String version = properties.get("version");
+ assertNotNull("No Java property", version);
+ assertFalse("Java lookup was not resolved", version.contains("${java:"));
+ }
+
+ @Test
+ public void testCustomResolver() throws Exception {
+ Properties props = new Properties();
+ props.load(new FileInputStream(TEST_PROPS));
+ System.setProperty("name", NAME_VALUE);
+ System.setProperty("lookups", "test-lookups.properties");
+ Map<String, String> properties = MapResolver.resolveProperties(props);
+ String name = properties.get("name");
+ assertNotNull("No name property", name);
+ assertEquals("Incorrect system property resolution", NAME_VALUE, name);
+ String testStr = properties.get("const");
+ assertNotNull("No const property", testStr);
+ assertTrue("Constant was resolved", testStr.contains("${const:"));
+ String version = properties.get("version");
+ assertNotNull("No Java property", version);
+ assertFalse("Java lookup was not resolved", version.contains("${java:"));
+ String test = properties.get("test");
+ assertNotNull("No Test property", version);
+ assertEquals("Test lookup was not resolved", "Value", test);
+ }
+
+}
diff --git
a/flume-ng-node/src/test/java/org/apache/flume/node/lookup/TestLookup.java
b/flume-ng-node/src/test/java/org/apache/flume/node/lookup/TestLookup.java
new file mode 100644
index 00000000..42a004a8
--- /dev/null
+++ b/flume-ng-node/src/test/java/org/apache/flume/node/lookup/TestLookup.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache license, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the license for the specific language governing permissions and
+ * limitations under the license.
+ */
+package org.apache.flume.node.lookup;
+
+import org.apache.commons.text.lookup.StringLookup;
+
+/**
+ * Test Lookup.
+ */
+public class TestLookup implements StringLookup {
+ @Override
+ public String lookup(String key) {
+ return key;
+ }
+}
diff --git a/flume-ng-node/src/test/resources/map-resolver.properties
b/flume-ng-node/src/test/resources/map-resolver.properties
new file mode 100644
index 00000000..a856b074
--- /dev/null
+++ b/flume-ng-node/src/test/resources/map-resolver.properties
@@ -0,0 +1,4 @@
+name = ${sys:name}
+const = ${const:org.apache.flume.node.TestMapResolver.TEST_CONST}
+version = ${java:version}
+test = ${test:Value}
\ No newline at end of file
diff --git a/flume-ng-node/src/test/resources/test-lookups.properties
b/flume-ng-node/src/test/resources/test-lookups.properties
new file mode 100644
index 00000000..b8ba3a61
--- /dev/null
+++ b/flume-ng-node/src/test/resources/test-lookups.properties
@@ -0,0 +1,4 @@
+sys = org.apache.commons.text.lookup.DefaultStringLookup.SYSTEM_PROPERTIES
+env = org.apache.commons.text.lookup.DefaultStringLookup.ENVIRONMENT
+java = org.apache.commons.text.lookup.DefaultStringLookup.JAVA
+test = org.apache.flume.node.lookup.TestLookup
\ No newline at end of file
