This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new a5d4b117252 NIFI-15790 Added batch secret resolution in
StandardConnectorConfigurationContext (#11098)
a5d4b117252 is described below
commit a5d4b1172520277e3d362fffdf4f812b4a09d908
Author: Pierre Villard <[email protected]>
AuthorDate: Tue Apr 7 15:43:23 2026 +0200
NIFI-15790 Added batch secret resolution in
StandardConnectorConfigurationContext (#11098)
Signed-off-by: David Handermann <[email protected]>
---
.../StandardConnectorConfigurationContext.java | 104 ++++-------
.../TestStandardConnectorConfigurationContext.java | 199 ++++++++++++++++++---
2 files changed, 213 insertions(+), 90 deletions(-)
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorConfigurationContext.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorConfigurationContext.java
index c37a246933e..437a66f15fa 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorConfigurationContext.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorConfigurationContext.java
@@ -19,18 +19,14 @@ package org.apache.nifi.components.connector;
import org.apache.nifi.asset.Asset;
import org.apache.nifi.asset.AssetManager;
-import org.apache.nifi.components.connector.secrets.SecretProvider;
import org.apache.nifi.components.connector.secrets.SecretsManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.IOException;
-import java.io.UncheckedIOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -176,28 +172,36 @@ public class StandardConnectorConfigurationContext
implements MutableConnectorCo
}
private StepConfiguration resolvePropertyValues(final Map<String,
ConnectorValueReference> propertyValues) {
- final Map<String, ConnectorValueReference> resolvedProperties = new
HashMap<>();
- for (final Map.Entry<String, ConnectorValueReference> entry :
propertyValues.entrySet()) {
- final ConnectorValueReference resolved = resolve(entry.getValue());
- resolvedProperties.put(entry.getKey(), resolved);
+ final Set<SecretReference> secretReferences = new HashSet<>();
+ for (final ConnectorValueReference ref : propertyValues.values()) {
+ if (ref instanceof SecretReference secretRef) {
+ secretReferences.add(secretRef);
+ }
}
- return new StepConfiguration(resolvedProperties);
- }
- private ConnectorValueReference resolve(final ConnectorValueReference
reference) {
- if (reference == null) {
- return null;
- }
+ final Map<SecretReference, Secret> resolvedSecrets =
secretReferences.isEmpty() ? Map.of() :
secretsManager.getSecrets(secretReferences);
+ return resolvePropertyValues(propertyValues, resolvedSecrets);
+ }
- try {
- return switch (reference) {
- case StringLiteralValue stringLiteral -> stringLiteral;
- case AssetReference assetReference ->
resolveAssetReferences(assetReference);
- case SecretReference secretReference -> new
StringLiteralValue(getSecretValue(secretReference));
- };
- } catch (final IOException ioe) {
- throw new UncheckedIOException("Unable to obtain Secrets from
Secret Manager", ioe);
+ private StepConfiguration resolvePropertyValues(final Map<String,
ConnectorValueReference> propertyValues, final Map<SecretReference, Secret>
resolvedSecrets) {
+ final Map<String, ConnectorValueReference> resolvedProperties = new
HashMap<>();
+ for (final Map.Entry<String, ConnectorValueReference> entry :
propertyValues.entrySet()) {
+ final ConnectorValueReference reference = entry.getValue();
+ if (reference == null) {
+ resolvedProperties.put(entry.getKey(), null);
+ } else {
+ final ConnectorValueReference resolved = switch (reference) {
+ case StringLiteralValue stringLiteral -> stringLiteral;
+ case AssetReference assetReference ->
resolveAssetReferences(assetReference);
+ case SecretReference secretReference -> {
+ final Secret secret =
resolvedSecrets.get(secretReference);
+ yield new StringLiteralValue(secret == null ? null :
secret.getValue());
+ }
+ };
+ resolvedProperties.put(entry.getKey(), resolved);
+ }
}
+ return new StepConfiguration(resolvedProperties);
}
private StringLiteralValue resolveAssetReferences(final AssetReference
assetReference) {
@@ -213,43 +217,6 @@ public class StandardConnectorConfigurationContext
implements MutableConnectorCo
return new StringLiteralValue(String.join(",", resolvedAssetValues));
}
- private String getSecretValue(final SecretReference secretReference)
throws IOException {
- final SecretProvider provider = getSecretProvider(secretReference);
- if (provider == null) {
- return null;
- }
-
- final List<Secret> secrets =
provider.getSecrets(List.of(secretReference.getFullyQualifiedName()));
- return secrets.isEmpty() ? null : secrets.getFirst().getValue();
- }
-
- private SecretProvider getSecretProvider(final SecretReference
secretReference) {
- final Set<SecretProvider> providers =
secretsManager.getSecretProviders();
- for (final SecretProvider provider : providers) {
- if (Objects.equals(provider.getProviderId(),
secretReference.getProviderId())) {
- return provider;
- }
- }
-
- for (final SecretProvider provider : providers) {
- if (Objects.equals(provider.getProviderName(),
secretReference.getProviderName())) {
- return provider;
- }
- }
-
- // Try to find by Fully Qualified Name prefix
- final String fqn = secretReference.getFullyQualifiedName();
- if (fqn != null) {
- for (final SecretProvider provider : providers) {
- if (fqn.startsWith(provider.getProviderName() + ".")) {
- return provider;
- }
- }
- }
-
- return null;
- }
-
@Override
public ConfigurationUpdateResult replaceProperties(final String stepName,
final StepConfiguration configuration) {
writeLock.lock();
@@ -274,13 +241,20 @@ public class StandardConnectorConfigurationContext
implements MutableConnectorCo
public void resolvePropertyValues() {
writeLock.lock();
try {
- for (final Map.Entry<String, StepConfiguration> entry :
propertyConfigurations.entrySet()) {
- final String stepName = entry.getKey();
- final StepConfiguration stepConfig = entry.getValue();
- final Map<String, ConnectorValueReference> stepProperties =
stepConfig.getPropertyValues();
+ final Set<SecretReference> allSecretReferences = new HashSet<>();
+ for (final StepConfiguration stepConfig :
propertyConfigurations.values()) {
+ for (final ConnectorValueReference ref :
stepConfig.getPropertyValues().values()) {
+ if (ref instanceof SecretReference secretRef) {
+ allSecretReferences.add(secretRef);
+ }
+ }
+ }
- final StepConfiguration resolvedConfig =
resolvePropertyValues(stepProperties);
- this.resolvedPropertyConfigurations.put(stepName,
resolvedConfig);
+ final Map<SecretReference, Secret> resolvedSecrets =
allSecretReferences.isEmpty() ? Map.of() :
secretsManager.getSecrets(allSecretReferences);
+
+ for (final Map.Entry<String, StepConfiguration> entry :
propertyConfigurations.entrySet()) {
+ final StepConfiguration resolvedConfig =
resolvePropertyValues(entry.getValue().getPropertyValues(), resolvedSecrets);
+ this.resolvedPropertyConfigurations.put(entry.getKey(),
resolvedConfig);
}
} finally {
writeLock.unlock();
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorConfigurationContext.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorConfigurationContext.java
index 6a73137ac76..0265a70e698 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorConfigurationContext.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorConfigurationContext.java
@@ -17,24 +17,45 @@
package org.apache.nifi.components.connector;
+import org.apache.nifi.asset.Asset;
import org.apache.nifi.asset.AssetManager;
-import org.apache.nifi.components.connector.secrets.SecretProvider;
import org.apache.nifi.components.connector.secrets.SecretsManager;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
-import java.util.Collections;
+import java.io.File;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.ArgumentMatchers.anySet;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class TestStandardConnectorConfigurationContext {
+
+ private static final String PROVIDER_1_ID = "provider-1";
+ private static final String PROVIDER_1_NAME = "Provider1";
+ private static final String PROVIDER_2_ID = "provider-2";
+ private static final String PROVIDER_2_NAME = "Provider2";
+
+ private static final String SECRET_VALUE_1 = "resolved-value-1";
+ private static final String SECRET_VALUE_2 = "resolved-value-2";
+ private static final String SECRET_VALUE_3 = "resolved-value-3";
+ private static final String PLAIN_VALUE = "plainValue";
+ private static final String ASSET_PATH = "/path/to/asset";
+
+ private static final SecretReference SECRET_REF_1 = new
SecretReference(PROVIDER_1_ID, PROVIDER_1_NAME, "secret1",
"Provider1.group.secret1");
+ private static final SecretReference SECRET_REF_2 = new
SecretReference(PROVIDER_1_ID, PROVIDER_1_NAME, "secret2",
"Provider1.group.secret2");
+ private static final SecretReference SECRET_REF_3 = new
SecretReference(PROVIDER_2_ID, PROVIDER_2_NAME, "secret3",
"Provider2.group.secret3");
+
private StandardConnectorConfigurationContext context;
@BeforeEach
@@ -53,6 +74,12 @@ public class TestStandardConnectorConfigurationContext {
return new StepConfiguration(valueReferences);
}
+ private static Secret mockSecret(final String value) {
+ final Secret secret = mock(Secret.class);
+ when(secret.getValue()).thenReturn(value);
+ return secret;
+ }
+
@Test
public void testSetPropertiesWithNoExistingConfigurations() {
final Map<String, String> properties = new HashMap<>();
@@ -218,39 +245,161 @@ public class TestStandardConnectorConfigurationContext {
@Test
public void
testResolvePropertyValuesResolvesSecretsThatWereInitiallyUnresolvable() {
- final String providerId = "provider-1";
- final String providerName = "TestProvider";
- final String secretName = "mySecret";
- final String fullyQualifiedName = "TestProvider.mySecret";
- final String secretValue = "super-secret-value";
+ final Secret secret = mockSecret(SECRET_VALUE_1);
- final SecretProvider secretProvider = mock(SecretProvider.class);
- when(secretProvider.getProviderId()).thenReturn(providerId);
- when(secretProvider.getProviderName()).thenReturn(providerName);
+ final SecretsManager secretsManager = mock(SecretsManager.class);
+ when(secretsManager.getSecrets(anySet()))
+ .thenReturn(Map.of())
+ .thenReturn(Map.of(SECRET_REF_1, secret));
- final Secret secret = mock(Secret.class);
- when(secret.getValue()).thenReturn(secretValue);
-
when(secretProvider.getSecrets(List.of(fullyQualifiedName))).thenReturn(List.of(secret));
+ final StandardConnectorConfigurationContext testContext = new
StandardConnectorConfigurationContext(mock(AssetManager.class), secretsManager);
+
+ final Map<String, ConnectorValueReference> properties = new
HashMap<>();
+ properties.put("plain", new StringLiteralValue(PLAIN_VALUE));
+ properties.put("secret", SECRET_REF_1);
+ testContext.setProperties("step1", new StepConfiguration(properties));
+
+ assertEquals(PLAIN_VALUE, testContext.getProperty("step1",
"plain").getValue());
+ assertNull(testContext.getProperty("step1", "secret").getValue());
+
+ testContext.resolvePropertyValues();
+
+ assertEquals(PLAIN_VALUE, testContext.getProperty("step1",
"plain").getValue());
+ assertEquals(SECRET_VALUE_1, testContext.getProperty("step1",
"secret").getValue());
+ }
+
+ @Test
+ public void testSetPropertiesBatchesSecretResolution() {
+ final Secret secret1 = mockSecret(SECRET_VALUE_1);
+ final Secret secret2 = mockSecret(SECRET_VALUE_2);
+
+ final SecretsManager secretsManager = mock(SecretsManager.class);
+
when(secretsManager.getSecrets(anySet())).thenReturn(Map.of(SECRET_REF_1,
secret1, SECRET_REF_2, secret2));
+
+ final StandardConnectorConfigurationContext testContext = new
StandardConnectorConfigurationContext(mock(AssetManager.class), secretsManager);
+
+ final Map<String, ConnectorValueReference> properties = new
HashMap<>();
+ properties.put("plain", new StringLiteralValue(PLAIN_VALUE));
+ properties.put("secret1", SECRET_REF_1);
+ properties.put("secret2", SECRET_REF_2);
+ testContext.setProperties("step1", new StepConfiguration(properties));
+
+ assertEquals(PLAIN_VALUE, testContext.getProperty("step1",
"plain").getValue());
+ assertEquals(SECRET_VALUE_1, testContext.getProperty("step1",
"secret1").getValue());
+ assertEquals(SECRET_VALUE_2, testContext.getProperty("step1",
"secret2").getValue());
+ verify(secretsManager, times(1)).getSecrets(anySet());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testResolvePropertyValuesBatchesAcrossAllSteps() {
+ final Secret secret1 = mockSecret(SECRET_VALUE_1);
+ final Secret secret3 = mockSecret(SECRET_VALUE_3);
final SecretsManager secretsManager = mock(SecretsManager.class);
-
when(secretsManager.getSecretProviders()).thenReturn(Collections.emptySet());
+
when(secretsManager.getSecrets(anySet())).thenReturn(Map.of(SECRET_REF_1,
secret1, SECRET_REF_3, secret3));
+
+ final StandardConnectorConfigurationContext testContext = new
StandardConnectorConfigurationContext(mock(AssetManager.class), secretsManager);
+
+ final Map<String, ConnectorValueReference> step1Props = new
HashMap<>();
+ step1Props.put("secret1", SECRET_REF_1);
+ testContext.setProperties("step1", new StepConfiguration(step1Props));
+
+ final Map<String, ConnectorValueReference> step2Props = new
HashMap<>();
+ step2Props.put("secret3", SECRET_REF_3);
+ testContext.setProperties("step2", new StepConfiguration(step2Props));
+
+ verify(secretsManager, times(2)).getSecrets(anySet());
+
+ testContext.resolvePropertyValues();
+
+ final ArgumentCaptor<Set<SecretReference>> captor =
ArgumentCaptor.forClass(Set.class);
+ verify(secretsManager, times(3)).getSecrets(captor.capture());
+ assertEquals(Set.of(SECRET_REF_1, SECRET_REF_3),
captor.getAllValues().get(2));
+
+ assertEquals(SECRET_VALUE_1, testContext.getProperty("step1",
"secret1").getValue());
+ assertEquals(SECRET_VALUE_3, testContext.getProperty("step2",
"secret3").getValue());
+ }
+
+ @Test
+ public void testMixedPropertyTypesResolvedCorrectly() {
+ final AssetReference assetRef = new AssetReference(Set.of("asset-1"));
+
+ final Secret secret = mockSecret(SECRET_VALUE_1);
+
+ final SecretsManager secretsManager = mock(SecretsManager.class);
+
when(secretsManager.getSecrets(anySet())).thenReturn(Map.of(SECRET_REF_1,
secret));
+
+ final Asset asset = mock(Asset.class);
+ final File assetFile = mock(File.class);
+ when(assetFile.getAbsolutePath()).thenReturn(ASSET_PATH);
+ when(asset.getFile()).thenReturn(assetFile);
final AssetManager assetManager = mock(AssetManager.class);
+ when(assetManager.getAsset("asset-1")).thenReturn(Optional.of(asset));
+
final StandardConnectorConfigurationContext testContext = new
StandardConnectorConfigurationContext(assetManager, secretsManager);
- final SecretReference secretRef = new SecretReference(providerId,
providerName, secretName, fullyQualifiedName);
final Map<String, ConnectorValueReference> properties = new
HashMap<>();
- properties.put("plainProp", new StringLiteralValue("plainValue"));
- properties.put("secretProp", secretRef);
- testContext.setProperties("authStep", new
StepConfiguration(properties));
+ properties.put("plain", new StringLiteralValue(PLAIN_VALUE));
+ properties.put("secret", SECRET_REF_1);
+ properties.put("asset", assetRef);
+ testContext.setProperties("step1", new StepConfiguration(properties));
+
+ assertEquals(PLAIN_VALUE, testContext.getProperty("step1",
"plain").getValue());
+ assertEquals(SECRET_VALUE_1, testContext.getProperty("step1",
"secret").getValue());
+ assertEquals(ASSET_PATH, testContext.getProperty("step1",
"asset").getValue());
+ }
- assertEquals("plainValue", testContext.getProperty("authStep",
"plainProp").getValue());
- assertNull(testContext.getProperty("authStep",
"secretProp").getValue());
+ @Test
+ public void testUnresolvableSecretReferenceMapsToNull() {
+ final Map<SecretReference, Secret> resultWithNull = new HashMap<>();
+ resultWithNull.put(SECRET_REF_1, null);
-
when(secretsManager.getSecretProviders()).thenReturn(Set.of(secretProvider));
- testContext.resolvePropertyValues();
+ final SecretsManager secretsManager = mock(SecretsManager.class);
+ when(secretsManager.getSecrets(anySet())).thenReturn(resultWithNull);
+
+ final StandardConnectorConfigurationContext testContext = new
StandardConnectorConfigurationContext(mock(AssetManager.class), secretsManager);
+
+ final Map<String, ConnectorValueReference> properties = new
HashMap<>();
+ properties.put("secret", SECRET_REF_1);
+ testContext.setProperties("step1", new StepConfiguration(properties));
+
+ assertNull(testContext.getProperty("step1", "secret").getValue());
+ }
+
+ @Test
+ public void testNoSecretReferencesDoesNotCallGetSecrets() {
+ final SecretsManager secretsManager = mock(SecretsManager.class);
+ final StandardConnectorConfigurationContext testContext = new
StandardConnectorConfigurationContext(mock(AssetManager.class), secretsManager);
+
+ final Map<String, ConnectorValueReference> properties = new
HashMap<>();
+ properties.put("plain1", new StringLiteralValue("value1"));
+ properties.put("plain2", new StringLiteralValue("value2"));
+ testContext.setProperties("step1", new StepConfiguration(properties));
+
+ assertEquals("value1", testContext.getProperty("step1",
"plain1").getValue());
+ assertEquals("value2", testContext.getProperty("step1",
"plain2").getValue());
+ verify(secretsManager, never()).getSecrets(anySet());
+ }
+
+ @Test
+ public void testReplacePropertiesBatchesSecretResolution() {
+ final Secret secret1 = mockSecret(SECRET_VALUE_1);
+ final Secret secret2 = mockSecret(SECRET_VALUE_2);
+
+ final SecretsManager secretsManager = mock(SecretsManager.class);
+
when(secretsManager.getSecrets(anySet())).thenReturn(Map.of(SECRET_REF_1,
secret1, SECRET_REF_2, secret2));
+
+ final StandardConnectorConfigurationContext testContext = new
StandardConnectorConfigurationContext(mock(AssetManager.class), secretsManager);
+
+ final Map<String, ConnectorValueReference> properties = new
HashMap<>();
+ properties.put("secret1", SECRET_REF_1);
+ properties.put("secret2", SECRET_REF_2);
+ testContext.replaceProperties("step1", new
StepConfiguration(properties));
- assertEquals("plainValue", testContext.getProperty("authStep",
"plainProp").getValue());
- assertEquals(secretValue, testContext.getProperty("authStep",
"secretProp").getValue());
+ assertEquals(SECRET_VALUE_1, testContext.getProperty("step1",
"secret1").getValue());
+ assertEquals(SECRET_VALUE_2, testContext.getProperty("step1",
"secret2").getValue());
+ verify(secretsManager, times(1)).getSecrets(anySet());
}
}
