This is an automated email from the ASF dual-hosted git repository.
rhauch pushed a commit to branch 2.3
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/2.3 by this push:
new b85707c KAFKA-8774: Regex can be found anywhere in config value
(#7197)
b85707c is described below
commit b85707c0fdb4c11448cdb5a1938e0488ddc91102
Author: Arjun Satish <wickni...@users.noreply.github.com>
AuthorDate: Tue Aug 13 07:40:12 2019 -0700
KAFKA-8774: Regex can be found anywhere in config value (#7197)
Corrected the AbstractHerder to correctly identify task configs that
contain variables for externalized secrets. The original method incorrectly
used `matcher.matches()` instead of `matcher.find()`. The former method expects
the entire string to match the regex, whereas the second one can find a pattern
anywhere within the input string (which fits this use case more correctly).
Added unit tests to cover various cases of a config with externalized
secrets, and updated system tests to cover case where config value contains
additional characters besides secret that requires regex pattern to be found
anywhere in the string (as opposed to complete match).
Author: Arjun Satish <ar...@confluent.io>
Reviewer: Randall Hauch <rha...@gmail.com>
---
.../kafka/connect/runtime/AbstractHerder.java | 5 ++--
.../kafka/connect/runtime/AbstractHerderTest.java | 28 ++++++++++++++++++++++
tests/kafkatest/tests/connect/connect_rest_test.py | 2 +-
3 files changed, 32 insertions(+), 3 deletions(-)
diff --git
a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
index f66029c..83d81bb 100644
---
a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
+++
b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java
@@ -598,12 +598,13 @@ public abstract class AbstractHerder implements Herder,
TaskStatus.Listener, Con
return result;
}
- private static Set<String> keysWithVariableValues(Map<String, String>
rawConfig, Pattern pattern) {
+ // Visible for testing
+ static Set<String> keysWithVariableValues(Map<String, String> rawConfig,
Pattern pattern) {
Set<String> keys = new HashSet<>();
for (Map.Entry<String, String> config : rawConfig.entrySet()) {
if (config.getValue() != null) {
Matcher matcher = pattern.matcher(config.getValue());
- if (matcher.matches()) {
+ if (matcher.find()) {
keys.add(config.getKey());
}
}
diff --git
a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
index 2c341bc..8018916 100644
---
a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
+++
b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java
@@ -20,6 +20,7 @@ import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.config.ConfigException;
+import org.apache.kafka.common.config.ConfigTransformer;
import org.apache.kafka.common.config.SaslConfigs;
import
org.apache.kafka.common.security.oauthbearer.internals.unsecured.OAuthBearerUnsecuredLoginCallbackHandler;
import org.apache.kafka.connect.connector.ConnectRecord;
@@ -62,6 +63,7 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import static
org.apache.kafka.connect.runtime.AbstractHerder.keysWithVariableValues;
import static org.powermock.api.easymock.PowerMock.verifyAll;
import static org.powermock.api.easymock.PowerMock.replayAll;
import static org.easymock.EasyMock.strictMock;
@@ -474,6 +476,32 @@ public class AbstractHerderTest {
assertFalse(reverseTransformed.get(0).containsKey(TEST_KEY3));
}
+ @Test
+ public void testConfigProviderRegex() {
+ testConfigProviderRegex("\"${::}\"");
+ testConfigProviderRegex("${::}");
+ testConfigProviderRegex("\"${:/a:somevar}\"");
+ testConfigProviderRegex("\"${file::somevar}\"");
+ testConfigProviderRegex("${file:/a/b/c:}");
+ testConfigProviderRegex("${file:/tmp/somefile.txt:somevar}");
+ testConfigProviderRegex("\"${file:/tmp/somefile.txt:somevar}\"");
+ testConfigProviderRegex("plain.PlainLoginModule required
username=\"${file:/tmp/somefile.txt:somevar}\"");
+ testConfigProviderRegex("plain.PlainLoginModule required
username=${file:/tmp/somefile.txt:somevar}");
+ testConfigProviderRegex("plain.PlainLoginModule required
username=${file:/tmp/somefile.txt:somevar} not null");
+ testConfigProviderRegex("plain.PlainLoginModule required
username=${file:/tmp/somefile.txt:somevar}
password=${file:/tmp/somefile.txt:othervar}");
+ testConfigProviderRegex("plain.PlainLoginModule required username",
false);
+ }
+
+ private void testConfigProviderRegex(String rawConnConfig) {
+ testConfigProviderRegex(rawConnConfig, true);
+ }
+
+ private void testConfigProviderRegex(String rawConnConfig, boolean
expected) {
+ Set<String> keys =
keysWithVariableValues(Collections.singletonMap("key", rawConnConfig),
ConfigTransformer.DEFAULT_PATTERN);
+ boolean actual = keys != null && !keys.isEmpty() &&
keys.contains("key");
+ assertEquals(String.format("%s should have matched regex",
rawConnConfig), expected, actual);
+ }
+
private AbstractHerder createConfigValidationHerder(Class<? extends
Connector> connectorClass,
ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy) {
diff --git a/tests/kafkatest/tests/connect/connect_rest_test.py
b/tests/kafkatest/tests/connect/connect_rest_test.py
index 13dcf04..67c0417 100644
--- a/tests/kafkatest/tests/connect/connect_rest_test.py
+++ b/tests/kafkatest/tests/connect/connect_rest_test.py
@@ -44,7 +44,7 @@ class ConnectRestApiTest(KafkaTest):
INPUT_FILE2 = "/mnt/connect.input2"
OUTPUT_FILE = "/mnt/connect.output"
- TOPIC = "${file:%s:topic.external}" %
ConnectServiceBase.EXTERNAL_CONFIGS_FILE
+ TOPIC = "topic-${file:%s:topic.external}" %
ConnectServiceBase.EXTERNAL_CONFIGS_FILE
TOPIC_TEST = "test"
DEFAULT_BATCH_SIZE = "2000"
