This is an automated email from the ASF dual-hosted git repository.
kevdoran pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 09c6d7765a8 NIFI-15992 - Add secret caching bypass in Connector
user-called verify (#11306)
09c6d7765a8 is described below
commit 09c6d7765a8b34c267cfb57ed8222a115ec86682
Author: Pierre Villard <[email protected]>
AuthorDate: Wed Jun 3 06:05:09 2026 -0700
NIFI-15992 - Add secret caching bypass in Connector user-called verify
(#11306)
Signed-off-by: Kevin Doran <[email protected]>
---
.../secrets/ParameterProviderSecretsManager.java | 14 +++--
.../TestParameterProviderSecretsManager.java | 61 ++++++++++++++++++++++
.../connector/secrets/SecretsManager.java | 13 +++++
.../connector/StandardConnectorNode.java | 12 +++--
.../connector/TestStandardConnectorNode.java | 49 ++++++++++++++++-
5 files changed, 140 insertions(+), 9 deletions(-)
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/components/connector/secrets/ParameterProviderSecretsManager.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/components/connector/secrets/ParameterProviderSecretsManager.java
index 5547170e979..6285ce9e16f 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/components/connector/secrets/ParameterProviderSecretsManager.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/components/connector/secrets/ParameterProviderSecretsManager.java
@@ -161,6 +161,11 @@ public class ParameterProviderSecretsManager implements
SecretsManager {
@Override
public Map<SecretReference, Secret> getSecrets(final Set<SecretReference>
secretReferences) {
+ return getSecrets(secretReferences, true);
+ }
+
+ @Override
+ public Map<SecretReference, Secret> getSecrets(final Set<SecretReference>
secretReferences, final boolean useCache) {
if (secretReferences.isEmpty()) {
return Map.of();
}
@@ -169,7 +174,7 @@ public class ParameterProviderSecretsManager implements
SecretsManager {
return fetchSecretsWithoutCache(secretReferences);
}
- return fetchSecretsWithCache(secretReferences);
+ return fetchSecretsWithCache(secretReferences, useCache);
}
private Map<SecretReference, Secret> fetchSecretsWithoutCache(final
Set<SecretReference> secretReferences) {
@@ -215,17 +220,18 @@ public class ParameterProviderSecretsManager implements
SecretsManager {
return secrets;
}
- private Map<SecretReference, Secret> fetchSecretsWithCache(final
Set<SecretReference> secretReferences) {
+ private Map<SecretReference, Secret> fetchSecretsWithCache(final
Set<SecretReference> secretReferences, final boolean useCache) {
final Set<SecretProvider> providers = getSecretProviders();
final Map<SecretReference, Secret> results = new HashMap<>();
- // Partition references into cache hits vs. misses that need fetching
+ // Partition references into cache hits vs. misses that need fetching.
When useCache is false, every
+ // reference is treated as a miss so fresh values are fetched and
written back, refreshing the cache.
final Map<SecretProvider, Set<SecretReference>> uncachedByProvider =
new HashMap<>();
for (final SecretReference secretReference : secretReferences) {
final String fqn = secretReference.getFullyQualifiedName();
- if (fqn != null) {
+ if (useCache && fqn != null) {
final CachedSecret cached = secretCache.get(fqn);
if (cached != null && !isExpired(cached)) {
logger.debug("Cached Secret found [{}]", fqn);
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/components/connector/secrets/TestParameterProviderSecretsManager.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/components/connector/secrets/TestParameterProviderSecretsManager.java
index 7ba784df077..b7dbd6895a6 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/components/connector/secrets/TestParameterProviderSecretsManager.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/components/connector/secrets/TestParameterProviderSecretsManager.java
@@ -127,6 +127,18 @@ public class TestParameterProviderSecretsManager {
return node;
}
+ private void updateSecretValue(final ParameterProviderNode node, final
String providerName, final String groupName, final String secretName, final
String newValue) {
+ when(node.fetchParameterValues(anyList())).thenAnswer(invocation -> {
+ final List<String> requestedNames = invocation.getArgument(0);
+ final String fullyQualifiedName = providerName + "." + groupName +
"." + secretName;
+ if (requestedNames.contains(fullyQualifiedName)) {
+ final Parameter updatedParameter = createParameter(secretName,
SECRET_1_DESCRIPTION, newValue);
+ return List.of(new ParameterGroup(groupName,
List.of(updatedParameter)));
+ }
+ return List.of();
+ });
+ }
+
private Parameter createParameter(final String name, final String
description, final String value) {
final ParameterDescriptor descriptor = new
ParameterDescriptor.Builder()
.name(name)
@@ -449,6 +461,55 @@ public class TestParameterProviderSecretsManager {
verify(providerNode, times(2)).fetchParameterValues(anyList());
}
+ @Test
+ public void testGetSecretsBypassingCacheRefetchesAndRefreshesCachedValue()
{
+ final ParameterProviderNode providerNode =
createMockedParameterProviderNode(PROVIDER_1_ID, PROVIDER_1_NAME, GROUP_1_NAME,
+ createParameter(SECRET_1_NAME, SECRET_1_DESCRIPTION,
SECRET_1_VALUE));
+ final ParameterProviderSecretsManager manager =
createManagerWithCacheDuration(DEFAULT_CACHE_DURATION, providerNode);
+
+ final SecretReference reference = createSecretReference(PROVIDER_1_ID,
null, SECRET_1_NAME);
+
+ final Map<SecretReference, Secret> cached =
manager.getSecrets(Set.of(reference));
+ assertEquals(SECRET_1_VALUE, cached.get(reference).getValue());
+ verify(providerNode, times(1)).fetchParameterValues(anyList());
+
+ final String updatedValue = "updated-secret-value-one";
+ updateSecretValue(providerNode, PROVIDER_1_NAME, GROUP_1_NAME,
SECRET_1_NAME, updatedValue);
+
+ // Bypassing the cache fetches the current value from the provider
rather than the cached value.
+ final Map<SecretReference, Secret> bypassed =
manager.getSecrets(Set.of(reference), false);
+ assertEquals(updatedValue, bypassed.get(reference).getValue());
+ verify(providerNode, times(2)).fetchParameterValues(anyList());
+
+ // The bypass refreshed the cache, so a subsequent cache-enabled read
returns the updated value without refetching.
+ final Map<SecretReference, Secret> afterRefresh =
manager.getSecrets(Set.of(reference), true);
+ assertEquals(updatedValue, afterRefresh.get(reference).getValue());
+ verify(providerNode, times(2)).fetchParameterValues(anyList());
+ }
+
+ @Test
+ public void testGetSecretsBypassingCacheLeavesOtherCachedEntriesIntact() {
+ final ParameterProviderNode providerNode =
createMockedParameterProviderNode(PROVIDER_1_ID, PROVIDER_1_NAME, GROUP_1_NAME,
+ createParameter(SECRET_1_NAME, SECRET_1_DESCRIPTION,
SECRET_1_VALUE),
+ createParameter(SECRET_2_NAME, SECRET_2_DESCRIPTION,
SECRET_2_VALUE));
+ final ParameterProviderSecretsManager manager =
createManagerWithCacheDuration(DEFAULT_CACHE_DURATION, providerNode);
+
+ final SecretReference reference1 =
createSecretReference(PROVIDER_1_ID, null, SECRET_1_NAME);
+ final SecretReference reference2 =
createSecretReference(PROVIDER_1_ID, null, SECRET_2_NAME);
+
+ manager.getSecrets(Set.of(reference1, reference2));
+ verify(providerNode, times(1)).fetchParameterValues(anyList());
+
+ // Bypassing the cache for reference1 only refetches reference1.
+ manager.getSecrets(Set.of(reference1), false);
+ verify(providerNode, times(2)).fetchParameterValues(anyList());
+
+ // reference2 remains cached and is served without an additional fetch.
+ final Map<SecretReference, Secret> reference2Result =
manager.getSecrets(Set.of(reference2), true);
+ assertEquals(SECRET_2_VALUE,
reference2Result.get(reference2).getValue());
+ verify(providerNode, times(2)).fetchParameterValues(anyList());
+ }
+
@Test
public void testGetSecretsBatchWithMixedCacheHitsAndMisses() {
final ParameterProviderNode providerNode =
createMockedParameterProviderNode(PROVIDER_1_ID, PROVIDER_1_NAME, GROUP_1_NAME,
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/secrets/SecretsManager.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/secrets/SecretsManager.java
index e3fbe297976..fccc59b3c45 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/secrets/SecretsManager.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/components/connector/secrets/SecretsManager.java
@@ -37,6 +37,19 @@ public interface SecretsManager {
Map<SecretReference, Secret> getSecrets(Set<SecretReference>
secretReferences);
+ /**
+ * Resolves the given Secret References, optionally bypassing any cached
values. When {@code useCache} is
+ * {@code false}, implementations that cache Secret values must fetch
fresh values from the underlying
+ * providers and refresh their cache with the retrieved values, rather
than serving previously cached values.
+ *
+ * @param secretReferences the Secret References to resolve
+ * @param useCache whether cached values may be returned; when {@code
false}, values are fetched fresh and the cache is refreshed
+ * @return a mapping of each Secret Reference to its resolved Secret, or
{@code null} for references that cannot be resolved
+ */
+ default Map<SecretReference, Secret> getSecrets(final Set<SecretReference>
secretReferences, final boolean useCache) {
+ return getSecrets(secretReferences);
+ }
+
/**
* Invalidates any cached secret data, forcing the next access to fetch
fresh data
* from the underlying secret providers.
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java
index b9659ebb486..2a82cfd962f 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java
@@ -931,7 +931,9 @@ public class StandardConnectorNode implements ConnectorNode
{
final ConfigurationStep configurationStep = optionalStep.get();
final List<SecretReference> invalidSecretRefs = new ArrayList<>();
final List<AssetReference> invalidAssetRefs = new ArrayList<>();
- final Map<String, String> resolvedPropertyOverrides =
resolvePropertyReferences(configurationStep, configurationOverrides,
invalidSecretRefs, invalidAssetRefs);
+ // Bypass the Secret value cache during verification so the user
sees results based on the current
+ // Secret values rather than potentially stale cached values
awaiting TTL expiration.
+ final Map<String, String> resolvedPropertyOverrides =
resolvePropertyReferences(configurationStep, configurationOverrides,
invalidSecretRefs, invalidAssetRefs, false);
final DescribedValueProvider allowableValueProvider = (step,
propertyName) -> fetchAllowableValues(step, propertyName, workingFlowContext);
@@ -986,7 +988,7 @@ public class StandardConnectorNode implements ConnectorNode
{
}
private Map<String, String> resolvePropertyReferences(final
ConfigurationStep configurationStep, final StepConfiguration
configurationOverrides,
- final
List<SecretReference> invalidSecretRefs, final List<AssetReference>
invalidAssetRefs) {
+ final
List<SecretReference> invalidSecretRefs, final List<AssetReference>
invalidAssetRefs, final boolean useCache) {
final Map<String, String> resolvedProperties = new HashMap<>();
final Map<String, ConnectorPropertyDescriptor> descriptorLookup =
buildPropertyDescriptorLookup(configurationStep);
@@ -1008,7 +1010,7 @@ public class StandardConnectorNode implements
ConnectorNode {
final Map<SecretReference, Secret> secretsByReference =
secretReferences.isEmpty()
? Map.of()
- :
initializationContext.getSecretsManager().getSecrets(secretReferences);
+ :
initializationContext.getSecretsManager().getSecrets(secretReferences,
useCache);
secretsByReference.forEach((ref, secret) -> {
if (secret == null) {
invalidSecretRefs.add(ref);
@@ -1637,7 +1639,9 @@ public class StandardConnectorNode implements
ConnectorNode {
// Check for invalid Secret and Asset references
final List<SecretReference> invalidSecrets = new ArrayList<>();
final List<AssetReference> invalidAssets = new ArrayList<>();
- resolvePropertyReferences(step, stepConfiguration, invalidSecrets,
invalidAssets);
+ // Regular validation may run frequently, so cached Secret values
are used here to avoid
+ // repeatedly fetching from the underlying Secret Providers on
every validation cycle.
+ resolvePropertyReferences(step, stepConfiguration, invalidSecrets,
invalidAssets, true);
addInvalidReferenceResults(allResults, invalidSecrets,
invalidAssets);
}
}
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java
index 00e136a6635..3c7c957cbbc 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java
@@ -62,8 +62,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anySet;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class TestStandardConnectorNode {
@@ -597,7 +601,7 @@ public class TestStandardConnectorNode {
// that stopped short-circuiting on unsatisfied dependencies would
surface the lookup attempt as a hard failure here
// instead of silently being masked by the empty-stub filter or by a
null-returning mock.
final SecretsManager strictSecretsManager = mock(SecretsManager.class);
- when(strictSecretsManager.getSecrets(anySet()))
+ when(strictSecretsManager.getSecrets(anySet(), anyBoolean()))
.thenThrow(new AssertionError("SecretsManager.getSecrets must not
be called when property dependencies are not satisfied"));
final DependentSecretVerifyConnector connector = new
DependentSecretVerifyConnector();
@@ -677,6 +681,48 @@ public class TestStandardConnectorNode {
}), () -> "Expected required-property failure in active validation,
got: " + errors);
}
+ @Test
+ public void testVerifyConfigurationStepResolvesSecretsBypassingCache()
throws FlowUpdateException {
+ final SecretsManager recordingSecretsManager =
mock(SecretsManager.class);
+ when(recordingSecretsManager.getSecrets(anySet(),
anyBoolean())).thenReturn(Collections.emptyMap());
+
+ final DependentSecretVerifyConnector connector = new
DependentSecretVerifyConnector();
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector, recordingSecretsManager);
+
+ connectorNode.transitionStateForUpdating();
+ connectorNode.prepareForUpdate();
+
+ final Map<String, ConnectorValueReference> propertyValues = new
HashMap<>();
+ propertyValues.put("Mode", new StringLiteralValue("WITH_SECRET"));
+ propertyValues.put("SecretKey", new SecretReference("pid", "My
Provider", "my-secret", "My Provider.my-secret"));
+
+ connectorNode.verifyConfigurationStep("authStep", new
StepConfiguration(propertyValues));
+
+ verify(recordingSecretsManager).getSecrets(anySet(), eq(false));
+ }
+
+ @Test
+ public void testPerformValidationResolvesSecretsUsingCache() throws
FlowUpdateException {
+ final SecretsManager recordingSecretsManager =
mock(SecretsManager.class);
+ when(recordingSecretsManager.getAllSecrets()).thenReturn(List.of());
+ when(recordingSecretsManager.getSecrets(anySet(),
anyBoolean())).thenReturn(Collections.emptyMap());
+
+ final DependentSecretVerifyConnector connector = new
DependentSecretVerifyConnector();
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector, recordingSecretsManager);
+
+ connectorNode.transitionStateForUpdating();
+ connectorNode.prepareForUpdate();
+ final Map<String, ConnectorValueReference> propertyValues = new
HashMap<>();
+ propertyValues.put("Mode", new StringLiteralValue("WITH_SECRET"));
+ propertyValues.put("SecretKey", new SecretReference("pid", "My
Provider", "my-secret", "My Provider.my-secret"));
+ connectorNode.setConfiguration("authStep", new
StepConfiguration(propertyValues));
+ connectorNode.applyUpdate();
+
+ connectorNode.performValidation();
+
+ verify(recordingSecretsManager, atLeastOnce()).getSecrets(anySet(),
eq(true));
+ }
+
@Test
@Timeout(value = 5, unit = TimeUnit.SECONDS)
public void testDrainFlowFilesTransitionsStateToDraining() throws
FlowUpdateException {
@@ -792,6 +838,7 @@ public class TestStandardConnectorNode {
final SecretsManager defaultSecretsManager =
mock(SecretsManager.class);
when(defaultSecretsManager.getAllSecrets()).thenReturn(List.of());
when(defaultSecretsManager.getSecrets(anySet())).thenReturn(Collections.emptyMap());
+ when(defaultSecretsManager.getSecrets(anySet(),
anyBoolean())).thenReturn(Collections.emptyMap());
return createConnectorNode(connector, defaultSecretsManager);
}
