This is an automated email from the ASF dual-hosted git repository.
kevdoran pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new e54dc5ec15d NIFI-16242: Inherit default values for newly added
Connector properties (#11581)
e54dc5ec15d is described below
commit e54dc5ec15dec122c30d38c7ddf8d59d1acbd6bf
Author: Mark Payne <[email protected]>
AuthorDate: Tue Aug 25 10:46:20 2026 -0400
NIFI-16242: Inherit default values for newly added Connector properties
(#11581)
---
.../connector/StandardConnectorNode.java | 48 ++-
.../connector/TestStandardConnectorNode.java | 409 ++++++++++++++++++++-
2 files changed, 452 insertions(+), 5 deletions(-)
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java
index d5c1e3fcf0d..552a95f90ea 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorNode.java
@@ -78,6 +78,7 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -419,11 +420,56 @@ public class StandardConnectorNode implements
ConnectorNode, GroupedComponent {
initial.put(versionedConfigStep.getName(), new
StepConfiguration(toValueReferenceMap(versionedConfigStep)));
}
+ final Set<String> persistedStepNames = new
LinkedHashSet<>(initial.keySet());
final StandardConnectorPropertyConfiguration propertyConfiguration =
new StandardConnectorPropertyConfiguration(initial, this.toString());
try (final NarCloseable ignored =
NarCloseable.withComponentNarLoader(extensionManager,
getConnector().getClass(), getIdentifier())) {
getConnector().migrateProperties(propertyConfiguration);
+ return
applyMissingRequiredPropertyDefaults(propertyConfiguration.getMutatedProperties(),
persistedStepNames, getConnector().getConfigurationSteps());
}
- return propertyConfiguration.getMutatedProperties();
+ }
+
+ /**
+ * Fills in the default value for any required property that has no value
in the migrated configuration, so a NAR
+ * upgrade that adds a required property with a default does not make the
Connector invalid. A step the Connector
+ * removed during migration (present in {@code persistedStepNames} but
absent from {@code migratedProperties}) is
+ * not re-created; a declared step in neither is newly added by this
version and is created, but only if at least
+ * one required default applies to it.
+ */
+ private Map<String, StepConfiguration>
applyMissingRequiredPropertyDefaults(final Map<String, StepConfiguration>
migratedProperties,
+ final Set<String> persistedStepNames, final
List<ConfigurationStep> configurationSteps) {
+ if (configurationSteps == null || configurationSteps.isEmpty()) {
+ return migratedProperties;
+ }
+
+ final Map<String, StepConfiguration> propertiesWithDefaults = new
LinkedHashMap<>(migratedProperties);
+ for (final ConfigurationStep configurationStep : configurationSteps) {
+ final String stepName = configurationStep.getName();
+ final StepConfiguration existingConfiguration =
propertiesWithDefaults.get(stepName);
+ if (existingConfiguration == null &&
persistedStepNames.contains(stepName)) {
+ continue;
+ }
+
+ final Map<String, ConnectorValueReference> existingValues =
existingConfiguration == null ? null :
existingConfiguration.getPropertyValues();
+ final Map<String, ConnectorValueReference> propertyValues =
existingValues == null ? new LinkedHashMap<>() : new
LinkedHashMap<>(existingValues);
+ boolean appliedMissingDefault = false;
+ for (final ConnectorPropertyGroup propertyGroup :
configurationStep.getPropertyGroups()) {
+ for (final ConnectorPropertyDescriptor descriptor :
propertyGroup.getProperties()) {
+ if (!descriptor.isRequired() ||
descriptor.getDefaultValue() == null ||
propertyValues.containsKey(descriptor.getName())) {
+ continue;
+ }
+
+ propertyValues.put(descriptor.getName(), new
StringLiteralValue(descriptor.getDefaultValue()));
+ appliedMissingDefault = true;
+ logger.debug("Applied default value for required property
[{}] of configuration step [{}] on {}", descriptor.getName(), stepName, this);
+ }
+ }
+
+ if (appliedMissingDefault) {
+ propertiesWithDefaults.put(stepName, new
StepConfiguration(propertyValues));
+ }
+ }
+
+ return propertiesWithDefaults;
}
private Map<String, ConnectorValueReference> toValueReferenceMap(final
VersionedConfigurationStep step) {
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java
index 545ba41a9b9..22a148bb709 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java
@@ -37,9 +37,12 @@ import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.controller.state.StandardStateMap;
import org.apache.nifi.engine.FlowEngine;
import org.apache.nifi.flow.Bundle;
+import org.apache.nifi.flow.VersionedConfigurationStep;
+import org.apache.nifi.flow.VersionedConnectorValueReference;
import org.apache.nifi.flow.VersionedExternalFlow;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.migration.ConnectorPropertyConfiguration;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.util.MockComponentLog;
import org.junit.jupiter.api.AfterEach;
@@ -1108,6 +1111,131 @@ public class TestStandardConnectorNode {
assertTrue(node.isModified());
}
+ @Test
+ public void
testVerifyCanStartAfterInheritingConfigurationMissingRequiredPropertyWithDefault()
throws FlowUpdateException {
+ final DefaultValueConnector connector = new DefaultValueConnector();
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector);
+
+ final VersionedConnectorValueReference greetingReference = new
VersionedConnectorValueReference();
+ greetingReference.setValueType("STRING_LITERAL");
+ greetingReference.setValue("Welcome");
+
+ final VersionedConfigurationStep persistedStep = new
VersionedConfigurationStep();
+ persistedStep.setName("settings");
+ persistedStep.setProperties(Map.of("Greeting", greetingReference));
+
+ connectorNode.transitionStateForUpdating();
+ connectorNode.prepareForUpdate();
+ connectorNode.inheritConfiguration(List.of(persistedStep),
List.of(persistedStep), createConnectorBundle());
+
+ connectorNode.verifyCanStart();
+ assertEquals("Welcome",
connectorNode.getActiveFlowContext().getConfigurationContext().getProperty("settings",
"Greeting").getValue());
+ assertEquals("1",
connectorNode.getActiveFlowContext().getConfigurationContext().getProperty("settings",
"Repeat Count").getValue());
+ }
+
+ @Test
+ public void
testInheritingConfigurationDoesNotApplyOptionalPropertyDefault() throws
FlowUpdateException {
+ final DependentDefaultValueConnector connector = new
DependentDefaultValueConnector();
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector);
+
+ final VersionedConfigurationStep persistedStep = new
VersionedConfigurationStep();
+ persistedStep.setName("settings");
+ persistedStep.setProperties(Map.of());
+
+ connectorNode.transitionStateForUpdating();
+ connectorNode.prepareForUpdate();
+ connectorNode.inheritConfiguration(List.of(persistedStep),
List.of(persistedStep), createConnectorBundle());
+
+ // "SSL Mode" is optional, so its default must not be inserted. If it
were, the "REQUIRED" default would
+ // satisfy the dependency of "Truststore Filename" and make that
required property report as missing.
+
assertFalse(connectorNode.getActiveFlowContext().getConfigurationContext().getPropertyNames("settings").contains("SSL
Mode"));
+ connectorNode.verifyCanStart();
+ }
+
+ @Test
+ public void
testInheritingConfigurationCreatesNewlyDeclaredStepWithRequiredDefaults()
throws FlowUpdateException {
+ final DeclaredStepRecordingConnector connector = new
DeclaredStepRecordingConnector();
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector);
+
+ final VersionedConnectorValueReference greetingReference = new
VersionedConnectorValueReference();
+ greetingReference.setValueType("STRING_LITERAL");
+ greetingReference.setValue("Welcome");
+
+ final VersionedConfigurationStep persistedStep = new
VersionedConfigurationStep();
+ persistedStep.setName("settings");
+ persistedStep.setProperties(Map.of("Greeting", greetingReference));
+
+ connectorNode.transitionStateForUpdating();
+ connectorNode.prepareForUpdate();
+ connectorNode.inheritConfiguration(List.of(persistedStep),
List.of(persistedStep), createConnectorBundle());
+
+ // The "extra" step is newly declared by this version of the Connector
and was never persisted, so it is
+ // created with its required default and its configuration callback
fires under its new name.
+ assertTrue(connector.getConfiguredStepNames().contains("extra"));
+ assertEquals("default",
connectorNode.getActiveFlowContext().getConfigurationContext().getProperty("extra",
"Extra Property").getValue());
+ connectorNode.verifyCanStart();
+ }
+
+ @Test
+ public void
testInheritingConfigurationDoesNotRecreateStepRemovedDuringMigration() throws
FlowUpdateException {
+ final LegacyStepRemovingConnector connector = new
LegacyStepRemovingConnector();
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector);
+
+ final VersionedConnectorValueReference greetingReference = new
VersionedConnectorValueReference();
+ greetingReference.setValueType("STRING_LITERAL");
+ greetingReference.setValue("Welcome");
+
+ final VersionedConnectorValueReference legacyReference = new
VersionedConnectorValueReference();
+ legacyReference.setValueType("STRING_LITERAL");
+ legacyReference.setValue("retained");
+
+ final VersionedConfigurationStep settingsStep = new
VersionedConfigurationStep();
+ settingsStep.setName("settings");
+ settingsStep.setProperties(Map.of("Greeting", greetingReference));
+
+ final VersionedConfigurationStep legacyStep = new
VersionedConfigurationStep();
+ legacyStep.setName("legacy");
+ legacyStep.setProperties(Map.of("Legacy Property", legacyReference));
+
+ connectorNode.transitionStateForUpdating();
+ connectorNode.prepareForUpdate();
+ connectorNode.inheritConfiguration(List.of(settingsStep, legacyStep),
List.of(settingsStep, legacyStep), createConnectorBundle());
+
+ // The Connector removed the persisted "legacy" step during migration.
Even though it still declares the step
+ // and the step's required property has a default, the step must not
be re-created and its callback must not fire.
+ assertFalse(connector.getConfiguredStepNames().contains("legacy"));
+
assertTrue(connectorNode.getActiveFlowContext().getConfigurationContext().getPropertyNames("legacy").isEmpty());
+ }
+
+ @Test
+ public void
testInheritingConfigurationKeepsTransitivelyGatedRequiredPropertyIrrelevant()
throws FlowUpdateException {
+ final TransitiveDependencyConnector connector = new
TransitiveDependencyConnector();
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector);
+
+ final VersionedConfigurationStep persistedStep = new
VersionedConfigurationStep();
+ persistedStep.setName("settings");
+ persistedStep.setProperties(Map.of());
+
+ connectorNode.transitionStateForUpdating();
+ connectorNode.prepareForUpdate();
+ connectorNode.inheritConfiguration(List.of(persistedStep),
List.of(persistedStep), createConnectorBundle());
+
+ // "Username" is required with a default, so the back-fill
materializes it even though it is gated off by the
+ // unset "Authentication". "Password" is required with no default and
depends on "Username". If dependency
+ // evaluation is transitive, "Password" stays gated off because
"Authentication" is unset, so materializing
+ // "Username" does not make "Password" required and the Connector
remains startable.
+ assertEquals("admin",
connectorNode.getActiveFlowContext().getConfigurationContext().getProperty("settings",
"Username").getValue());
+ connectorNode.verifyCanStart();
+ }
+
+ private static Bundle createConnectorBundle() {
+ final Bundle bundle = new Bundle();
+ bundle.setGroup("org.apache.nifi");
+ bundle.setArtifact("test-bundle");
+ bundle.setVersion("1.0.0");
+ return bundle;
+ }
+
private static void seedActiveConfiguration(final StandardConnectorNode
node, final String stepName, final Map<String, ConnectorValueReference>
properties) {
node.getActiveFlowContext().getConfigurationContext().setProperties(stepName,
new StepConfiguration(properties));
}
@@ -1187,6 +1315,7 @@ public class TestStandardConnectorNode {
final FrameworkConnectorInitializationContext initializationContext =
mock(FrameworkConnectorInitializationContext.class);
when(initializationContext.getSecretsManager()).thenReturn(initializedSecretsManager);
+ when(initializationContext.getAssetManager()).thenReturn(assetManager);
node.initializeConnector(initializationContext);
node.loadInitialFlow();
@@ -1502,10 +1631,6 @@ public class TestStandardConnectorNode {
}
}
- /**
- * Test connector declaring a single configuration step with two
properties that have default values. Used to
- * exercise the configuration-versus-default comparison performed by
{@code isModified()}.
- */
private static class DefaultValueConnector extends AbstractConnector {
@Override
public VersionedExternalFlow getInitialFlow() {
@@ -1565,6 +1690,282 @@ public class TestStandardConnectorNode {
}
}
+ private static class DependentDefaultValueConnector extends
AbstractConnector {
+ @Override
+ public VersionedExternalFlow getInitialFlow() {
+ return null;
+ }
+
+ @Override
+ public VersionedExternalFlow getActiveFlow(final FlowContext
activeFlowContext) {
+ return null;
+ }
+
+ @Override
+ public void prepareForUpdate(final FlowContext workingContext, final
FlowContext activeContext) {
+ }
+
+ @Override
+ public List<ConfigurationStep> getConfigurationSteps() {
+ final ConnectorPropertyDescriptor sslMode = new
ConnectorPropertyDescriptor.Builder()
+ .name("SSL Mode")
+ .description("Whether SSL is required")
+ .required(false)
+ .defaultValue("REQUIRED")
+ .build();
+
+ final ConnectorPropertyDescriptor truststoreFilename = new
ConnectorPropertyDescriptor.Builder()
+ .name("Truststore Filename")
+ .description("Location of the truststore")
+ .required(true)
+ .dependsOn(sslMode, "REQUIRED")
+ .build();
+
+ final ConnectorPropertyGroup propertyGroup =
ConnectorPropertyGroup.builder()
+ .name("Security")
+ .description("Security settings")
+ .properties(List.of(sslMode, truststoreFilename))
+ .build();
+
+ final ConfigurationStep step = new ConfigurationStep.Builder()
+ .name("settings")
+ .propertyGroups(List.of(propertyGroup))
+ .build();
+
+ return List.of(step);
+ }
+
+ @Override
+ public void applyUpdate(final FlowContext workingContext, final
FlowContext activeContext) {
+ }
+
+ @Override
+ protected void onStepConfigured(final String stepName, final
FlowContext workingContext) {
+ }
+
+ @Override
+ public List<ConfigVerificationResult> verifyConfigurationStep(final
String stepName, final Map<String, String> overrides, final FlowContext
flowContext) {
+ return List.of();
+ }
+ }
+
+ private static class DeclaredStepRecordingConnector extends
AbstractConnector {
+ private final Set<String> configuredStepNames = new HashSet<>();
+
+ @Override
+ public VersionedExternalFlow getInitialFlow() {
+ return null;
+ }
+
+ @Override
+ public VersionedExternalFlow getActiveFlow(final FlowContext
activeFlowContext) {
+ return null;
+ }
+
+ @Override
+ public void prepareForUpdate(final FlowContext workingContext, final
FlowContext activeContext) {
+ }
+
+ @Override
+ public List<ConfigurationStep> getConfigurationSteps() {
+ final ConnectorPropertyDescriptor greeting = new
ConnectorPropertyDescriptor.Builder()
+ .name("Greeting")
+ .description("Greeting text")
+ .required(true)
+ .defaultValue("Hello")
+ .build();
+
+ final ConfigurationStep settings = new ConfigurationStep.Builder()
+ .name("settings")
+ .propertyGroups(List.of(ConnectorPropertyGroup.builder()
+ .name("General")
+ .description("General settings")
+ .properties(List.of(greeting))
+ .build()))
+ .build();
+
+ final ConnectorPropertyDescriptor extraProperty = new
ConnectorPropertyDescriptor.Builder()
+ .name("Extra Property")
+ .description("Property added by a newer version of the
Connector")
+ .required(true)
+ .defaultValue("default")
+ .build();
+
+ final ConfigurationStep extra = new ConfigurationStep.Builder()
+ .name("extra")
+ .propertyGroups(List.of(ConnectorPropertyGroup.builder()
+ .name("Extra Group")
+ .description("Group added by a newer version of the
Connector")
+ .properties(List.of(extraProperty))
+ .build()))
+ .build();
+
+ return List.of(settings, extra);
+ }
+
+ @Override
+ public void applyUpdate(final FlowContext workingContext, final
FlowContext activeContext) {
+ }
+
+ @Override
+ protected void onStepConfigured(final String stepName, final
FlowContext workingContext) {
+ configuredStepNames.add(stepName);
+ }
+
+ @Override
+ public List<ConfigVerificationResult> verifyConfigurationStep(final
String stepName, final Map<String, String> overrides, final FlowContext
flowContext) {
+ return List.of();
+ }
+
+ Set<String> getConfiguredStepNames() {
+ return configuredStepNames;
+ }
+ }
+
+ private static class LegacyStepRemovingConnector extends AbstractConnector
{
+ private final Set<String> configuredStepNames = new HashSet<>();
+
+ @Override
+ public VersionedExternalFlow getInitialFlow() {
+ return null;
+ }
+
+ @Override
+ public VersionedExternalFlow getActiveFlow(final FlowContext
activeFlowContext) {
+ return null;
+ }
+
+ @Override
+ public void prepareForUpdate(final FlowContext workingContext, final
FlowContext activeContext) {
+ }
+
+ @Override
+ public void migrateProperties(final ConnectorPropertyConfiguration
configuration) {
+ configuration.removeStep("legacy");
+ }
+
+ @Override
+ public List<ConfigurationStep> getConfigurationSteps() {
+ final ConnectorPropertyDescriptor greeting = new
ConnectorPropertyDescriptor.Builder()
+ .name("Greeting")
+ .description("Greeting text")
+ .required(true)
+ .defaultValue("Hello")
+ .build();
+
+ final ConfigurationStep settings = new ConfigurationStep.Builder()
+ .name("settings")
+ .propertyGroups(List.of(ConnectorPropertyGroup.builder()
+ .name("General")
+ .description("General settings")
+ .properties(List.of(greeting))
+ .build()))
+ .build();
+
+ final ConnectorPropertyDescriptor legacyProperty = new
ConnectorPropertyDescriptor.Builder()
+ .name("Legacy Property")
+ .description("Property of a step removed during migration")
+ .required(true)
+ .defaultValue("old")
+ .build();
+
+ final ConfigurationStep legacy = new ConfigurationStep.Builder()
+ .name("legacy")
+ .propertyGroups(List.of(ConnectorPropertyGroup.builder()
+ .name("Legacy Group")
+ .description("Group of a step removed during migration")
+ .properties(List.of(legacyProperty))
+ .build()))
+ .build();
+
+ return List.of(settings, legacy);
+ }
+
+ @Override
+ public void applyUpdate(final FlowContext workingContext, final
FlowContext activeContext) {
+ }
+
+ @Override
+ protected void onStepConfigured(final String stepName, final
FlowContext workingContext) {
+ configuredStepNames.add(stepName);
+ }
+
+ @Override
+ public List<ConfigVerificationResult> verifyConfigurationStep(final
String stepName, final Map<String, String> overrides, final FlowContext
flowContext) {
+ return List.of();
+ }
+
+ Set<String> getConfiguredStepNames() {
+ return configuredStepNames;
+ }
+ }
+
+ private static class TransitiveDependencyConnector extends
AbstractConnector {
+ @Override
+ public VersionedExternalFlow getInitialFlow() {
+ return null;
+ }
+
+ @Override
+ public VersionedExternalFlow getActiveFlow(final FlowContext
activeFlowContext) {
+ return null;
+ }
+
+ @Override
+ public void prepareForUpdate(final FlowContext workingContext, final
FlowContext activeContext) {
+ }
+
+ @Override
+ public List<ConfigurationStep> getConfigurationSteps() {
+ final ConnectorPropertyDescriptor authentication = new
ConnectorPropertyDescriptor.Builder()
+ .name("Authentication")
+ .description("Authentication mode")
+ .required(false)
+ .build();
+
+ final ConnectorPropertyDescriptor username = new
ConnectorPropertyDescriptor.Builder()
+ .name("Username")
+ .description("Username used for authentication")
+ .required(true)
+ .defaultValue("admin")
+ .dependsOn(authentication, "Basic")
+ .build();
+
+ final ConnectorPropertyDescriptor password = new
ConnectorPropertyDescriptor.Builder()
+ .name("Password")
+ .description("Password used for authentication")
+ .required(true)
+ .dependsOn(username)
+ .build();
+
+ final ConnectorPropertyGroup propertyGroup =
ConnectorPropertyGroup.builder()
+ .name("Security")
+ .description("Security settings")
+ .properties(List.of(authentication, username, password))
+ .build();
+
+ final ConfigurationStep step = new ConfigurationStep.Builder()
+ .name("settings")
+ .propertyGroups(List.of(propertyGroup))
+ .build();
+
+ return List.of(step);
+ }
+
+ @Override
+ public void applyUpdate(final FlowContext workingContext, final
FlowContext activeContext) {
+ }
+
+ @Override
+ protected void onStepConfigured(final String stepName, final
FlowContext workingContext) {
+ }
+
+ @Override
+ public List<ConfigVerificationResult> verifyConfigurationStep(final
String stepName, final Map<String, String> overrides, final FlowContext
flowContext) {
+ return List.of();
+ }
+ }
+
/**
* Test connector that allows control over when drainFlowFiles completes
via a CompletableFuture
*/
